diff --git a/.buildkite/.env b/.buildkite/.env
deleted file mode 100644
index 85b102d07f..0000000000
--- a/.buildkite/.env
+++ /dev/null
@@ -1,13 +0,0 @@
-CI
-BUILDKITE
-BUILDKITE_BUILD_NUMBER
-BUILDKITE_BRANCH
-BUILDKITE_BUILD_NUMBER
-BUILDKITE_JOB_ID
-BUILDKITE_BUILD_URL
-BUILDKITE_PROJECT_SLUG
-BUILDKITE_COMMIT
-BUILDKITE_PULL_REQUEST
-BUILDKITE_TAG
-CODECOV_TOKEN
-TRIAL_FLAGS
diff --git a/.buildkite/merge_base_branch.sh b/.buildkite/merge_base_branch.sh
deleted file mode 100755
index d0a7aef8cb..0000000000
--- a/.buildkite/merge_base_branch.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
-    echo "Not merging forward, as this is a release branch"
-    exit 0
-fi
-
-if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
-    echo "Not a pull request, or hasn't had a PR opened yet..."
-
-    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
-    # can probably assume it's based on it and will be merged into it.
-    GITBASE="dinsic"
-else
-    # Get the reference, using the GitHub API
-    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
-fi
-
-echo "--- merge_base_branch $GITBASE"
-
-# Show what we are before
-git --no-pager show -s
-
-# Set up username so it can do a merge
-git config --global user.email bot@matrix.org
-git config --global user.name "A robot"
-
-# Fetch and merge. If it doesn't work, it will raise due to set -e.
-git fetch -u origin $GITBASE
-git merge --no-edit --no-commit origin/$GITBASE
-
-# Show what we are after.
-git --no-pager show -s
diff --git a/.buildkite/scripts/create_postgres_db.py b/.buildkite/scripts/create_postgres_db.py
deleted file mode 100755
index 956339de5c..0000000000
--- a/.buildkite/scripts/create_postgres_db.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from synapse.storage.engines import create_engine
-
-logger = logging.getLogger("create_postgres_db")
-
-if __name__ == "__main__":
-    # Create a PostgresEngine.
-    db_engine = create_engine({"name": "psycopg2", "args": {}})
-
-    # Connect to postgres to create the base database.
-    # We use "postgres" as a database because it's bound to exist and the "synapse" one
-    # doesn't exist yet.
-    db_conn = db_engine.module.connect(
-        user="postgres", host="postgres", password="postgres", dbname="postgres"
-    )
-    db_conn.autocommit = True
-    cur = db_conn.cursor()
-    cur.execute("CREATE DATABASE synapse;")
-    cur.close()
-    db_conn.close()
diff --git a/.buildkite/scripts/test_synapse_port_db.sh b/.buildkite/scripts/test_synapse_port_db.sh
deleted file mode 100755
index 8914319e38..0000000000
--- a/.buildkite/scripts/test_synapse_port_db.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-#
-# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
-# with additional dependencies needed for the test (such as coverage or the PostgreSQL
-# driver), update the schema of the test SQLite database and run background updates on it,
-# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
-# test porting the SQLite database to the PostgreSQL database (with coverage).
-
-set -xe
-cd `dirname $0`/../..
-
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2 coverage coverage-enable-subprocess
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
-echo "--- Generate the signing key"
-
-# Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
-
-echo "--- Prepare the databases"
-
-# Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
-
-# Create the PostgreSQL database.
-./.buildkite/scripts/create_postgres_db.py
-
-echo "+++ Run synapse_port_db"
-
-# Run the script
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
diff --git a/.buildkite/docker-compose-env b/.ci/docker-compose-env
similarity index 100%
rename from .buildkite/docker-compose-env
rename to .ci/docker-compose-env
diff --git a/.buildkite/docker-compose.yaml b/.ci/docker-compose.yaml
similarity index 100%
rename from .buildkite/docker-compose.yaml
rename to .ci/docker-compose.yaml
diff --git a/.buildkite/pipeline.yml b/.ci/pipeline.yml
similarity index 100%
rename from .buildkite/pipeline.yml
rename to .ci/pipeline.yml
diff --git a/.buildkite/postgres-config.yaml b/.ci/postgres-config.yaml
similarity index 72%
rename from .buildkite/postgres-config.yaml
rename to .ci/postgres-config.yaml
index 2acbe66f4c..f5a4aecd51 100644
--- a/.buildkite/postgres-config.yaml
+++ b/.ci/postgres-config.yaml
@@ -3,7 +3,7 @@
 # CI's Docker setup at the point where this file is considered.
 server_name: "localhost:8800"
 
-signing_key_path: "/src/.buildkite/test.signing.key"
+signing_key_path: ".ci/test.signing.key"
 
 report_stats: false
 
@@ -11,11 +11,9 @@ database:
   name: "psycopg2"
   args:
     user: postgres
-    host: postgres
+    host: localhost
     password: postgres
     database: synapse
 
 # Suppress the key server warning.
-trusted_key_servers:
-  - server_name: "matrix.org"
-suppress_key_server_warning: true
+trusted_key_servers: []
diff --git a/.ci/scripts/postgres_exec.py b/.ci/scripts/postgres_exec.py
new file mode 100755
index 0000000000..0f39a336d5
--- /dev/null
+++ b/.ci/scripts/postgres_exec.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import psycopg2
+
+# a very simple replacment for `psql`, to make up for the lack of the postgres client
+# libraries in the synapse docker image.
+
+# We use "postgres" as a database because it's bound to exist and the "synapse" one
+# doesn't exist yet.
+db_conn = psycopg2.connect(
+    user="postgres", host="localhost", password="postgres", dbname="postgres"
+)
+db_conn.autocommit = True
+cur = db_conn.cursor()
+for c in sys.argv[1:]:
+    cur.execute(c)
diff --git a/.buildkite/scripts/test_old_deps.sh b/.ci/scripts/test_old_deps.sh
similarity index 81%
rename from .buildkite/scripts/test_old_deps.sh
rename to .ci/scripts/test_old_deps.sh
index 3753f41a40..8b473936f8 100755
--- a/.buildkite/scripts/test_old_deps.sh
+++ b/.ci/scripts/test_old_deps.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# this script is run by buildkite in a plain `xenial` container; it installs the
+# this script is run by GitHub Actions in a plain `bionic` container; it installs the
 # minimal requirements for tox and hands over to the py3-old tox environment.
 
 set -ex
diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh
new file mode 100755
index 0000000000..2b4e5ec170
--- /dev/null
+++ b/.ci/scripts/test_synapse_port_db.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+# Test script for 'synapse_port_db'.
+#   - sets up synapse and deps
+#   - runs the port script on a prepopulated test sqlite db
+#   - also runs it against an new sqlite db
+
+
+set -xe
+cd `dirname $0`/../..
+
+echo "--- Install dependencies"
+
+# Install dependencies for this test.
+pip install psycopg2 coverage coverage-enable-subprocess
+
+# Install Synapse itself. This won't update any libraries.
+pip install -e .
+
+echo "--- Generate the signing key"
+
+# Generate the server's signing key.
+python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
+
+echo "--- Prepare test database"
+
+# Make sure the SQLite3 database is using the latest schema and has no pending background update.
+scripts-dev/update_database --database-config .ci/sqlite-config.yaml
+
+# Create the PostgreSQL database.
+.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+
+echo "+++ Run synapse_port_db against test database"
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+
+# We should be able to run twice against the same database.
+echo "+++ Run synapse_port_db a second time"
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+
+#####
+
+# Now do the same again, on an empty database.
+
+echo "--- Prepare empty SQLite database"
+
+# we do this by deleting the sqlite db, and then doing the same again.
+rm .ci/test_db.db
+
+scripts-dev/update_database --database-config .ci/sqlite-config.yaml
+
+# re-create the PostgreSQL database.
+.ci/scripts/postgres_exec.py \
+  "DROP DATABASE synapse" \
+  "CREATE DATABASE synapse"
+
+echo "+++ Run synapse_port_db against empty database"
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
diff --git a/.buildkite/sqlite-config.yaml b/.ci/sqlite-config.yaml
similarity index 65%
rename from .buildkite/sqlite-config.yaml
rename to .ci/sqlite-config.yaml
index 6d9bf80d84..3373743da3 100644
--- a/.buildkite/sqlite-config.yaml
+++ b/.ci/sqlite-config.yaml
@@ -3,16 +3,14 @@
 # schema and run background updates on it.
 server_name: "localhost:8800"
 
-signing_key_path: "/src/.buildkite/test.signing.key"
+signing_key_path: ".ci/test.signing.key"
 
 report_stats: false
 
 database:
   name: "sqlite3"
   args:
-    database: ".buildkite/test_db.db"
+    database: ".ci/test_db.db"
 
 # Suppress the key server warning.
-trusted_key_servers:
-  - server_name: "matrix.org"
-suppress_key_server_warning: true
+trusted_key_servers: []
diff --git a/.buildkite/test_db.db b/.ci/test_db.db
similarity index 100%
rename from .buildkite/test_db.db
rename to .ci/test_db.db
diff --git a/.buildkite/worker-blacklist b/.ci/worker-blacklist
similarity index 100%
rename from .buildkite/worker-blacklist
rename to .ci/worker-blacklist
diff --git a/.circleci/config.yml b/.circleci/config.yml
deleted file mode 100644
index 1ac48a71ba..0000000000
--- a/.circleci/config.yml
+++ /dev/null
@@ -1,78 +0,0 @@
-version: 2.1
-jobs:
-  dockerhubuploadrelease:
-    docker:
-      - image: docker:git
-    steps:
-      - checkout
-      - docker_prepare
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      # for release builds, we want to get the amd64 image out asap, so first
-      # we do an amd64-only build, before following up with a multiarch build.
-      - docker_build:
-          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
-          platforms: linux/amd64
-      - docker_build:
-          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
-          platforms: linux/amd64,linux/arm64
-
-  dockerhubuploadlatest:
-    docker:
-      - image: docker:git
-    steps:
-      - checkout
-      - docker_prepare
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      # for `latest`, we don't want the arm images to disappear, so don't update the tag
-      # until all of the platforms are built.
-      - docker_build:
-          tag: -t matrixdotorg/synapse:latest
-          platforms: linux/amd64,linux/arm64
-
-workflows:
-  build:
-    jobs:
-      - dockerhubuploadrelease:
-          filters:
-            tags:
-              only: /v[0-9].[0-9]+.[0-9]+.*/
-            branches:
-              ignore: /.*/
-      - dockerhubuploadlatest:
-          filters:
-            branches:
-              only: master
-
-commands:
-  docker_prepare:
-    description: Sets up a remote docker server, downloads the buildx cli plugin, and enables multiarch images
-    parameters:
-      buildx_version:
-        type: string
-        default: "v0.4.1"
-    steps:
-      - setup_remote_docker:
-          # 19.03.13 was the most recent available on circleci at the time of
-          # writing.
-          version: 19.03.13
-      - run: apk add --no-cache curl
-      - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
-      - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
-      - run: chmod a+x ~/.docker/cli-plugins/docker-buildx
-      # install qemu links in /proc/sys/fs/binfmt_misc on the docker instance running the circleci job
-      - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-      # create a context named `builder` for the builds
-      - run: docker context create builder
-      # create a buildx builder using the new context, and set it as the default
-      - run: docker buildx create builder --use
-
-  docker_build:
-    description: Builds and pushed images to dockerhub using buildx
-    parameters:
-      platforms:
-        type: string
-        default: linux/amd64
-      tag:
-        type: string
-    steps:
-      - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
new file mode 100644
index 0000000000..af7ed21fce
--- /dev/null
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,72 @@
+# GitHub actions workflow which builds and publishes the docker images.
+
+name: Build docker images
+
+on:
+  push:
+    tags: ["v*"]
+    branches: [ master, main ]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Inspect builder
+        run: docker buildx inspect
+          
+      - name: Log in to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Calculate docker image tag
+        id: set-tag
+        run: |
+          case "${GITHUB_REF}" in
+              refs/heads/master|refs/heads/main)
+                  tag=latest
+                  ;;
+              refs/tags/*)
+                  tag=${GITHUB_REF#refs/tags/}
+                  ;;
+              *)
+                  tag=${GITHUB_SHA}
+                  ;;
+          esac
+          echo "::set-output name=tag::$tag"
+
+        # for release builds, we want to get the amd64 image out asap, so first
+        # we do an amd64-only build, before following up with a multiarch build.
+      - name: Build and push amd64
+        uses: docker/build-push-action@v2
+        if: "${{ startsWith(github.ref, 'refs/tags/v') }}"
+        with:
+          push: true
+          labels: "gitsha1=${{ github.sha }}"
+          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
+          file: "docker/Dockerfile"
+          platforms: linux/amd64
+
+      - name: Build and push all platforms
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          labels: "gitsha1=${{ github.sha }}"
+          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
+          file: "docker/Dockerfile"
+          platforms: linux/amd64,linux/arm64
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
new file mode 100644
index 0000000000..808f825331
--- /dev/null
+++ b/.github/workflows/docs.yaml
@@ -0,0 +1,66 @@
+name: Deploy the documentation
+
+on:
+  push:
+    branches:
+      # For bleeding-edge documentation
+      - develop
+      # For documentation specific to a release
+      - 'release-v*'
+      # stable docs
+      - master
+
+  workflow_dispatch:
+
+jobs:
+  pages:
+    name: GitHub Pages
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup mdbook
+        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
+        with:
+          mdbook-version: '0.4.9'
+
+      - name: Build the documentation
+        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
+        # However, we're using docs/README.md for other purposes and need to pick a new page
+        # as the default. Let's opt for the welcome page instead.
+        run: |
+          mdbook build
+          cp book/welcome_and_overview.html book/index.html
+
+      # Figure out the target directory.
+      #
+      # The target directory depends on the name of the branch
+      #
+      - name: Get the target directory name
+        id: vars
+        run: |
+          # first strip the 'refs/heads/' prefix with some shell foo
+          branch="${GITHUB_REF#refs/heads/}"
+
+          case $branch in
+              release-*)
+                  # strip 'release-' from the name for release branches.
+                  branch="${branch#release-}"
+                  ;;
+              master)
+                  # deploy to "latest" for the master branch.
+                  branch="latest"
+                  ;;
+          esac
+
+          # finally, set the 'branch-version' var.
+          echo "::set-output name=branch-version::$branch"
+          
+      # Deploy to the target directory.
+      - name: Deploy to gh pages
+        uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          keep_files: true
+          publish_dir: ./book
+          destination_dir: ./${{ steps.vars.outputs.branch-version }}
diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml
new file mode 100644
index 0000000000..eb294f1619
--- /dev/null
+++ b/.github/workflows/release-artifacts.yml
@@ -0,0 +1,130 @@
+# GitHub actions workflow which builds the release artifacts.
+
+name: Build release artifacts
+
+on:
+  # we build on PRs and develop to (hopefully) get early warning
+  # of things breaking (but only build one set of debs)
+  pull_request:
+  push:
+    branches: ["develop"]
+
+    # we do the full build on tags.
+    tags: ["v*"]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+  
+permissions:
+  contents: write
+
+jobs:
+  get-distros:
+    name: "Calculate list of debian distros"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - id: set-distros
+        run: |
+          # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
+          dists='["debian:sid"]'
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+              dists=$(scripts-dev/build_debian_packages --show-dists-json)
+          fi
+          echo "::set-output name=distros::$dists"
+    # map the step outputs to job outputs
+    outputs:
+      distros: ${{ steps.set-distros.outputs.distros }}
+
+  # now build the packages with a matrix build.
+  build-debs:
+    needs: get-distros
+    name: "Build .deb packages"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: src
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+
+      - name: Set up docker layer caching
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Set up python
+        uses: actions/setup-python@v2
+
+      - name: Build the packages
+        # see https://github.com/docker/build-push-action/issues/252
+        # for the cache magic here
+        run: |
+          ./src/scripts-dev/build_debian_packages \
+            --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
+            --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
+            --docker-build-arg=--progress=plain \
+            --docker-build-arg=--load \
+            "${{ matrix.distro }}"
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Upload debs as artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: debs
+          path: debs/*
+
+  build-sdist:
+    name: "Build pypi distribution files"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install wheel
+      - run: |
+          python setup.py sdist bdist_wheel
+      - uses: actions/upload-artifact@v2
+        with:
+          name: python-dist
+          path: dist/*
+
+  # if it's a tag, create a release and attach the artifacts to it
+  attach-assets:
+    name: "Attach assets to release"
+    if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }}
+    needs:
+      - build-debs
+      - build-sdist
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download all workflow run artifacts
+        uses: actions/download-artifact@v2
+      - name: Build a tarball for the debs
+        run: tar -cvJf debs.tar.xz debs
+      - name: Attach to release
+        uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a  # PR#109
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          files: |
+            python-dist/*
+            debs.tar.xz
+          # if it's not already published, keep the release as a draft.
+          draft: true
+          # mark it as a prerelease if the tag contains 'rc'.
+          prerelease: ${{ contains(github.ref, 'rc') }}
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000000..8736699ad8
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,378 @@
+name: Tests
+
+on:
+  push:
+    branches: ["develop", "release-*"]
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        toxenv:
+          - "check-sampleconfig"
+          - "check_codestyle"
+          - "check_isort"
+          - "mypy"
+          - "packaging"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install tox
+      - run: tox -e ${{ matrix.toxenv }}
+
+  lint-crlf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check line endings
+        run: scripts-dev/check_line_terminators.sh
+
+  lint-newsfile:
+    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - uses: actions/setup-python@v2
+      - run: pip install tox
+      - run: scripts-dev/check-newsfragment
+        env:
+          PULL_REQUEST_NUMBER: ${{ github.event.number }}
+
+  lint-sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - run: pip install wheel
+      - run: python setup.py sdist bdist_wheel
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Python Distributions
+          path: dist/*
+
+  # Dummy step to gate other tests on without repeating the whole list
+  linting-done:
+    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
+    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
+    runs-on: ubuntu-latest
+    steps:
+      - run: "true"
+
+  trial:
+    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
+        database: ["sqlite"]
+        include:
+          # Newest Python without optional deps
+          - python-version: "3.9"
+            toxenv: "py-noextras,combine"
+
+          # Oldest Python with PostgreSQL
+          - python-version: "3.6"
+            database: "postgres"
+            postgres-version: "9.6"
+
+          # Newest Python with PostgreSQL
+          - python-version: "3.9"
+            database: "postgres"
+            postgres-version: "13"
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+        if: ${{ matrix.postgres-version }}
+        run: |
+          docker run -d -p 5432:5432 \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.postgres-version }}
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install tox
+      - name: Await PostgreSQL
+        if: ${{ matrix.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      - run: tox -e py,combine
+        env:
+          TRIAL_FLAGS: "--jobs=2"
+          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: localhost
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+  trial-olddeps:
+    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Test with old deps
+        uses: docker://ubuntu:bionic # For old python and sqlite
+        with:
+          workdir: /github/workspace
+          entrypoint: .ci/scripts/test_old_deps.sh
+        env:
+          TRIAL_FLAGS: "--jobs=2"
+      - name: Dump logs
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+  trial-pypy:
+    # Very slow; only run if the branch name includes 'pypy'
+    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
+    needs: linting-done
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["pypy-3.6"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install tox
+      - run: tox -e py,combine
+        env:
+          TRIAL_FLAGS: "--jobs=2"
+      - name: Dump logs
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+  sytest:
+    if: ${{ !failure() && !cancelled() }}
+    needs: linting-done
+    runs-on: ubuntu-latest
+    container:
+      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
+      volumes:
+        - ${{ github.workspace }}:/src
+      env:
+        POSTGRES: ${{ matrix.postgres && 1}}
+        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
+        WORKERS: ${{ matrix.workers && 1 }}
+        REDIS: ${{ matrix.redis && 1 }}
+        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+        TOP: ${{ github.workspace }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sytest-tag: bionic
+
+          - sytest-tag: bionic
+            postgres: postgres
+
+          - sytest-tag: testing
+            postgres: postgres
+
+          - sytest-tag: bionic
+            postgres: multi-postgres
+            workers: workers
+
+          - sytest-tag: buster
+            postgres: multi-postgres
+            workers: workers
+
+          - sytest-tag: buster
+            postgres: postgres
+            workers: workers
+            redis: redis
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Prepare test blacklist
+        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
+      - name: Run SyTest
+        run: /bootstrap.sh synapse
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+  portdb:
+    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    env:
+      TOP: ${{ github.workspace }}
+    strategy:
+      matrix:
+        include:
+          - python-version: "3.6"
+            postgres-version: "9.6"
+
+          - python-version: "3.9"
+            postgres-version: "13"
+
+    services:
+      postgres:
+        image: postgres:${{ matrix.postgres-version }}
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: "postgres"
+          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: .ci/scripts/test_synapse_port_db.sh
+
+  complement:
+    if: ${{ !failure() && !cancelled() }}
+    needs: linting-done
+    runs-on: ubuntu-latest
+    container:
+      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
+      image: matrixdotorg/complement:latest
+      env:
+        CI: true
+      ports:
+        - 8448:8448
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
+
+    steps:
+      - name: Run actions/checkout@v2 for synapse
+        uses: actions/checkout@v2
+        with:
+          path: synapse
+
+      # Attempt to check out the same branch of Complement as the PR. If it
+      # doesn't exist, fallback to master.
+      - name: Checkout complement
+        shell: bash
+        run: |
+          mkdir -p complement
+          # Attempt to use the version of complement which best matches the current
+          # build. Depending on whether this is a PR or release, etc. we need to
+          # use different fallbacks.
+          #
+          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
+          #    for pull requests, otherwise GITHUB_REF).
+          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
+          #    (GITHUB_BASE_REF for pull requests).
+          # 3. Use the default complement branch ("master").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
+            # Skip empty branch names and merge commits.
+            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+              continue
+            fi
+
+            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
+          done
+
+      # Build initial Synapse image
+      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
+        working-directory: synapse
+
+      # Build a ready-to-run Synapse image based on the initial image above.
+      # This new image includes a config file, keys for signing and TLS, and
+      # other settings to make it suitable for testing under Complement.
+      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
+        working-directory: complement/dockerfiles
+
+      # Run Complement
+      - run: go test -v -tags synapse_blacklist,msc2403,msc2946,msc3083 ./tests/...
+        env:
+          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
+        working-directory: complement
+
+  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
+  tests-done:
+    if: ${{ always() }}
+    needs:
+      - lint
+      - lint-crlf
+      - lint-newsfile
+      - lint-sdist
+      - trial
+      - trial-olddeps
+      - sytest
+      - portdb
+      - complement
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set build result
+        env:
+          NEEDS_CONTEXT: ${{ toJSON(needs) }}
+        # the `jq` incantation dumps out a series of "<job> <result>" lines.
+        # we set it to an intermediate variable to avoid a pipe, which makes it
+        # hard to set $rc.
+        run: |
+          rc=0
+          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
+          while read job result ; do
+              # The newsfile lint may be skipped on non PR builds
+              if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
+                continue
+              fi
+
+              if [ "$result" != "success" ]; then
+                  echo "::set-failed ::Job $job returned $result"
+                  rc=1
+              fi
+          done <<< $results
+          exit $rc
diff --git a/.gitignore b/.gitignore
index 295a18b539..6b9257b5c9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,3 +46,6 @@ __pycache__/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
+
+# docs
+book/
diff --git a/CHANGES.md b/CHANGES.md
index 27483532d0..fab27b874e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,1000 @@
+Synapse 1.41.1 (2021-08-31)
+===========================
+
+Due to the two security issues highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild.
+
+Security advisory
+-----------------
+
+The following issues are fixed in v1.41.1.
+
+- **[GHSA-3x4c-pq33-4w3q](https://github.com/matrix-org/synapse/security/advisories/GHSA-3x4c-pq33-4w3q) / [CVE-2021-39164](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-39164): Enumerating a private room's list of members and their display names.**
+
+  If an unauthorized user both knows the Room ID of a private room *and* that room's history visibility is set to `shared`, then they may be able to enumerate the room's members, including their display names.
+
+  The unauthorized user must be on the same homeserver as a user who is a member of the target room.
+
+  Fixed by [52c7a51cf](https://github.com/matrix-org/synapse/commit/52c7a51cf).
+
+- **[GHSA-jj53-8fmw-f2w2](https://github.com/matrix-org/synapse/security/advisories/GHSA-jj53-8fmw-f2w2) / [CVE-2021-39163](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-39163): Disclosing a private room's name, avatar, topic, and number of members.**
+
+  If an unauthorized user knows the Room ID of a private room, then its name, avatar, topic, and number of members may be disclosed through Group / Community features.
+
+  The unauthorized user must be on the same homeserver as a user who is a member of the target room, and their homeserver must allow non-administrators to create groups (`enable_group_creation` in the Synapse configuration; off by default).
+
+  Fixed by [cb35df940a](https://github.com/matrix-org/synapse/commit/cb35df940a), [\#10723](https://github.com/matrix-org/synapse/issues/10723).
+
+Bugfixes
+--------
+
+- Fix a regression introduced in Synapse 1.41 which broke email transmission on systems using older versions of the Twisted library. ([\#10713](https://github.com/matrix-org/synapse/issues/10713))
+
+
+Synapse 1.41.0 (2021-08-24)
+===========================
+
+This release adds support for Debian 12 (Bookworm), but **removes support for Ubuntu 20.10 (Groovy Gorilla)**, which reached End of Life last month.
+
+Note that when using workers the `/_synapse/admin/v1/users/{userId}/media` must now be handled by media workers. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html) for more information.
+
+
+Features
+--------
+
+- Enable room capabilities ([MSC3244](https://github.com/matrix-org/matrix-doc/pull/3244)) by default and set room version 8 as the preferred room version when creating restricted rooms. ([\#10571](https://github.com/matrix-org/synapse/issues/10571))
+
+
+Synapse 1.41.0rc1 (2021-08-18)
+==============================
+
+Features
+--------
+
+- Add `get_userinfo_by_id` method to ModuleApi. ([\#9581](https://github.com/matrix-org/synapse/issues/9581))
+- Initial local support for [MSC3266](https://github.com/matrix-org/synapse/pull/10394), Room Summary over the unstable `/rooms/{roomIdOrAlias}/summary` API. ([\#10394](https://github.com/matrix-org/synapse/issues/10394))
+- Experimental support for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288), sending `room_type` to the identity server for 3pid invites over the `/store-invite` API. ([\#10435](https://github.com/matrix-org/synapse/issues/10435))
+- Add support for sending federation requests through a proxy. Contributed by @Bubu and @dklimpel. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html) for more information. ([\#10596](https://github.com/matrix-org/synapse/issues/10596)). ([\#10475](https://github.com/matrix-org/synapse/issues/10475))
+- Add support for "marker" events which makes historical events discoverable for servers that already have all of the scrollback history (part of [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). ([\#10498](https://github.com/matrix-org/synapse/issues/10498))
+- Add a configuration setting for the time a `/sync` response is cached for. ([\#10513](https://github.com/matrix-org/synapse/issues/10513))
+- The default logging handler for new installations is now `PeriodicallyFlushingMemoryHandler`, a buffered logging handler which periodically flushes itself. ([\#10518](https://github.com/matrix-org/synapse/issues/10518))
+- Add support for new redaction rules for historical events specified in [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716). ([\#10538](https://github.com/matrix-org/synapse/issues/10538))
+- Add a setting to disable TLS when sending email. ([\#10546](https://github.com/matrix-org/synapse/issues/10546))
+- Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10549](https://github.com/matrix-org/synapse/issues/10549), [\#10560](https://github.com/matrix-org/synapse/issues/10560), [\#10569](https://github.com/matrix-org/synapse/issues/10569), [\#10574](https://github.com/matrix-org/synapse/issues/10574), [\#10575](https://github.com/matrix-org/synapse/issues/10575), [\#10579](https://github.com/matrix-org/synapse/issues/10579), [\#10583](https://github.com/matrix-org/synapse/issues/10583))
+- Admin API to delete several media for a specific user. Contributed by @dklimpel. ([\#10558](https://github.com/matrix-org/synapse/issues/10558), [\#10628](https://github.com/matrix-org/synapse/issues/10628))
+- Add support for routing `/createRoom` to workers. ([\#10564](https://github.com/matrix-org/synapse/issues/10564))
+- Update the Synapse Grafana dashboard. ([\#10570](https://github.com/matrix-org/synapse/issues/10570))
+- Add an admin API (`GET /_synapse/admin/username_available`) to check if a username is available (regardless of registration settings). ([\#10578](https://github.com/matrix-org/synapse/issues/10578))
+- Allow editing a user's `external_ids` via the "Edit User" admin API. Contributed by @dklimpel. ([\#10598](https://github.com/matrix-org/synapse/issues/10598))
+- The Synapse manhole no longer needs coroutines to be wrapped in `defer.ensureDeferred`. ([\#10602](https://github.com/matrix-org/synapse/issues/10602))
+- Add option to allow modules to run periodic tasks on all instances, rather than just the one configured to run background tasks. ([\#10638](https://github.com/matrix-org/synapse/issues/10638))
+
+
+Bugfixes
+--------
+
+- Add some clarification to the sample config file. Contributed by @Kentokamoto. ([\#10129](https://github.com/matrix-org/synapse/issues/10129))
+- Fix a long-standing bug where protocols which are not implemented by any appservices were incorrectly returned via `GET /_matrix/client/r0/thirdparty/protocols`. ([\#10532](https://github.com/matrix-org/synapse/issues/10532))
+- Fix exceptions in logs when failing to get remote room list. ([\#10541](https://github.com/matrix-org/synapse/issues/10541))
+- Fix longstanding bug which caused the user's presence "status message" to be reset when the user went offline. Contributed by @dklimpel. ([\#10550](https://github.com/matrix-org/synapse/issues/10550))
+- Allow public rooms to be previewed in the spaces summary APIs from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10580](https://github.com/matrix-org/synapse/issues/10580))
+- Fix a bug introduced in v1.37.1 where an error could occur in the asynchronous processing of PDUs when the queue was empty. ([\#10592](https://github.com/matrix-org/synapse/issues/10592))
+- Fix errors on /sync when read receipt data is a string. Only affects homeservers with the experimental flag for [MSC2285](https://github.com/matrix-org/matrix-doc/pull/2285) enabled. Contributed by @SimonBrandner. ([\#10606](https://github.com/matrix-org/synapse/issues/10606))
+- Additional validation for the spaces summary API to avoid errors like `ValueError: Stop argument for islice() must be None or an integer`. The missing validation has existed since v1.31.0. ([\#10611](https://github.com/matrix-org/synapse/issues/10611))
+- Revert behaviour introduced in v1.38.0 that strips `org.matrix.msc2732.device_unused_fallback_key_types` from `/sync` when its value is empty. This field should instead always be present according to [MSC2732](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2732-olm-fallback-keys.md). ([\#10623](https://github.com/matrix-org/synapse/issues/10623))
+
+
+Improved Documentation
+----------------------
+
+- Add documentation for configuring a forward proxy. ([\#10443](https://github.com/matrix-org/synapse/issues/10443))
+- Updated the reverse proxy documentation to highlight the homserver configuration that is needed to make Synapse aware that is is intentionally reverse proxied. ([\#10551](https://github.com/matrix-org/synapse/issues/10551))
+- Update CONTRIBUTING.md to fix index links and the instructions for SyTest in docker. ([\#10599](https://github.com/matrix-org/synapse/issues/10599))
+
+
+Deprecations and Removals
+-------------------------
+
+- No longer build `.deb` packages for Ubuntu 20.10 Groovy Gorilla, which has now EOLed. ([\#10588](https://github.com/matrix-org/synapse/issues/10588))
+- The `template_dir` configuration settings in the `sso`, `account_validity` and `email` sections of the configuration file are now deprecated in favour of the global `templates.custom_template_directory` setting. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html) for more information. ([\#10596](https://github.com/matrix-org/synapse/issues/10596))
+
+
+Internal Changes
+----------------
+
+- Improve event caching mechanism to avoid having multiple copies of an event in memory at a time. ([\#10119](https://github.com/matrix-org/synapse/issues/10119))
+- Reduce errors in PostgreSQL logs due to concurrent serialization errors. ([\#10504](https://github.com/matrix-org/synapse/issues/10504))
+- Include room ID in ignored EDU log messages. Contributed by @ilmari. ([\#10507](https://github.com/matrix-org/synapse/issues/10507))
+- Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946). ([\#10527](https://github.com/matrix-org/synapse/issues/10527), [\#10530](https://github.com/matrix-org/synapse/issues/10530))
+- Fix CI to not break when run against branches rather than pull requests. ([\#10529](https://github.com/matrix-org/synapse/issues/10529))
+- Mark all events stemming from the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint as historical. ([\#10537](https://github.com/matrix-org/synapse/issues/10537))
+- Clean up some of the federation event authentication code for clarity. ([\#10539](https://github.com/matrix-org/synapse/issues/10539), [\#10591](https://github.com/matrix-org/synapse/issues/10591))
+- Convert `Transaction` and `Edu` objects to attrs. ([\#10542](https://github.com/matrix-org/synapse/issues/10542))
+- Update `/batch_send` endpoint to only return `state_events` created by the `state_events_from_before` passed in. ([\#10552](https://github.com/matrix-org/synapse/issues/10552))
+- Update contributing.md to warn against rebasing an open PR. ([\#10563](https://github.com/matrix-org/synapse/issues/10563))
+- Remove the unused public rooms replication stream. ([\#10565](https://github.com/matrix-org/synapse/issues/10565))
+- Clarify error message when failing to join a restricted room. ([\#10572](https://github.com/matrix-org/synapse/issues/10572))
+- Remove references to BuildKite in favour of GitHub Actions. ([\#10573](https://github.com/matrix-org/synapse/issues/10573))
+- Move `/batch_send` endpoint defined by [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) to the `/v2_alpha` directory. ([\#10576](https://github.com/matrix-org/synapse/issues/10576))
+- Allow multiple custom directories in `read_templates`. ([\#10587](https://github.com/matrix-org/synapse/issues/10587))
+- Re-organize the `synapse.federation.transport.server` module to create smaller files. ([\#10590](https://github.com/matrix-org/synapse/issues/10590))
+- Flatten the `synapse.rest.client` package by moving the contents of `v1` and `v2_alpha` into the parent. ([\#10600](https://github.com/matrix-org/synapse/issues/10600))
+- Build Debian packages for Debian 12 (Bookworm). ([\#10612](https://github.com/matrix-org/synapse/issues/10612))
+- Fix up a couple of links to the database schema documentation. ([\#10620](https://github.com/matrix-org/synapse/issues/10620))
+- Fix a broken link to the upgrade notes. ([\#10631](https://github.com/matrix-org/synapse/issues/10631))
+
+
+Synapse 1.40.0 (2021-08-10)
+===========================
+
+No significant changes.
+
+
+Synapse 1.40.0rc3 (2021-08-09)
+==============================
+
+Features
+--------
+
+- Support [MSC3289: room version 8](https://github.com/matrix-org/matrix-doc/pull/3289). ([\#10449](https://github.com/matrix-org/synapse/issues/10449))
+
+
+Bugfixes
+--------
+
+- Mark the experimental room version from [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) as unstable. ([\#10449](https://github.com/matrix-org/synapse/issues/10449))
+
+
+Improved Documentation
+----------------------
+
+- Fix broken links in `upgrade.md`. Contributed by @dklimpel. ([\#10543](https://github.com/matrix-org/synapse/issues/10543))
+
+
+Synapse 1.40.0rc2 (2021-08-04)
+==============================
+
+Bugfixes
+--------
+
+- Fix the `PeriodicallyFlushingMemoryHandler` inhibiting application shutdown because of its background thread. ([\#10517](https://github.com/matrix-org/synapse/issues/10517))
+- Fix a bug introduced in Synapse v1.40.0rc1 that could cause Synapse to respond with an error when clients would update read receipts. ([\#10531](https://github.com/matrix-org/synapse/issues/10531))
+
+
+Internal Changes
+----------------
+
+- Fix release script to open the correct URL for the release. ([\#10516](https://github.com/matrix-org/synapse/issues/10516))
+
+
+Synapse 1.40.0rc1 (2021-08-03)
+==============================
+
+Features
+--------
+
+- Add support for [MSC2033](https://github.com/matrix-org/matrix-doc/pull/2033): `device_id` on `/account/whoami`. ([\#9918](https://github.com/matrix-org/synapse/issues/9918))
+- Update support for [MSC2716 - Incrementally importing history into existing rooms](https://github.com/matrix-org/matrix-doc/pull/2716). ([\#10245](https://github.com/matrix-org/synapse/issues/10245), [\#10432](https://github.com/matrix-org/synapse/issues/10432), [\#10463](https://github.com/matrix-org/synapse/issues/10463))
+- Update support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) to consider changes in the MSC around which servers can issue join events. ([\#10254](https://github.com/matrix-org/synapse/issues/10254), [\#10447](https://github.com/matrix-org/synapse/issues/10447), [\#10489](https://github.com/matrix-org/synapse/issues/10489))
+- Initial support for [MSC3244](https://github.com/matrix-org/matrix-doc/pull/3244), Room version capabilities over the /capabilities API. ([\#10283](https://github.com/matrix-org/synapse/issues/10283))
+- Add a buffered logging handler which periodically flushes itself. ([\#10407](https://github.com/matrix-org/synapse/issues/10407), [\#10515](https://github.com/matrix-org/synapse/issues/10515))
+- Add support for https connections to a proxy server. Contributed by @Bubu and @dklimpel. ([\#10411](https://github.com/matrix-org/synapse/issues/10411))
+- Support for [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-doc/pull/2285). Contributed by @SimonBrandner. ([\#10413](https://github.com/matrix-org/synapse/issues/10413))
+- Email notifications now state whether an invitation is to a room or a space. ([\#10426](https://github.com/matrix-org/synapse/issues/10426))
+- Allow setting transaction limit for database connections. ([\#10440](https://github.com/matrix-org/synapse/issues/10440), [\#10511](https://github.com/matrix-org/synapse/issues/10511))
+- Add `creation_ts` to "list users" admin API. ([\#10448](https://github.com/matrix-org/synapse/issues/10448))
+
+
+Bugfixes
+--------
+
+- Improve character set detection in URL previews by supporting underscores (in addition to hyphens). Contributed by @srividyut. ([\#10410](https://github.com/matrix-org/synapse/issues/10410))
+- Fix events being incorrectly rejected over federation if they reference auth events that the server needed to fetch. ([\#10439](https://github.com/matrix-org/synapse/issues/10439))
+- Fix `synapse_federation_server_oldest_inbound_pdu_in_staging` Prometheus metric to not report a max age of 51 years when the queue is empty. ([\#10455](https://github.com/matrix-org/synapse/issues/10455))
+- Fix a bug which caused an explicit assignment of power-level 0 to a user to be misinterpreted in rare circumstances. ([\#10499](https://github.com/matrix-org/synapse/issues/10499))
+
+
+Improved Documentation
+----------------------
+
+- Fix hierarchy of providers on the OpenID page. ([\#10445](https://github.com/matrix-org/synapse/issues/10445))
+- Consolidate development documentation to `docs/development/`. ([\#10453](https://github.com/matrix-org/synapse/issues/10453))
+- Add some developer docs to explain room DAG concepts like `outliers`, `state_groups`, `depth`, etc. ([\#10464](https://github.com/matrix-org/synapse/issues/10464))
+- Document how to use Complement while developing a new Synapse feature. ([\#10483](https://github.com/matrix-org/synapse/issues/10483))
+
+
+Internal Changes
+----------------
+
+- Prune inbound federation queues for a room if they get too large. ([\#10390](https://github.com/matrix-org/synapse/issues/10390))
+- Add type hints to `synapse.federation.transport.client` module. ([\#10408](https://github.com/matrix-org/synapse/issues/10408))
+- Remove shebang line from module files. ([\#10415](https://github.com/matrix-org/synapse/issues/10415))
+- Drop backwards-compatibility code that was required to support Ubuntu Xenial. ([\#10429](https://github.com/matrix-org/synapse/issues/10429))
+- Use a docker image cache for the prerequisites for the debian package build. ([\#10431](https://github.com/matrix-org/synapse/issues/10431))
+- Improve servlet type hints. ([\#10437](https://github.com/matrix-org/synapse/issues/10437), [\#10438](https://github.com/matrix-org/synapse/issues/10438))
+- Replace usage of `or_ignore` in `simple_insert` with `simple_upsert` usage, to stop spamming postgres logs with spurious ERROR messages. ([\#10442](https://github.com/matrix-org/synapse/issues/10442))
+- Update the `tests-done` Github Actions status. ([\#10444](https://github.com/matrix-org/synapse/issues/10444), [\#10512](https://github.com/matrix-org/synapse/issues/10512))
+- Update type annotations to work with forthcoming Twisted 21.7.0 release. ([\#10446](https://github.com/matrix-org/synapse/issues/10446), [\#10450](https://github.com/matrix-org/synapse/issues/10450))
+- Cancel redundant GHA workflows when a new commit is pushed. ([\#10451](https://github.com/matrix-org/synapse/issues/10451))
+- Mitigate media repo XSS attacks on IE11 via the non-standard X-Content-Security-Policy header. ([\#10468](https://github.com/matrix-org/synapse/issues/10468))
+- Additional type hints in the state handler. ([\#10482](https://github.com/matrix-org/synapse/issues/10482))
+- Update syntax used to run complement tests. ([\#10488](https://github.com/matrix-org/synapse/issues/10488))
+- Fix up type annotations to work with Twisted 21.7. ([\#10490](https://github.com/matrix-org/synapse/issues/10490))
+- Improve type annotations for `ObservableDeferred`. ([\#10491](https://github.com/matrix-org/synapse/issues/10491))
+- Extend release script to also tag and create GitHub releases. ([\#10496](https://github.com/matrix-org/synapse/issues/10496))
+- Fix a bug which caused production debian packages to be incorrectly marked as 'prerelease'. ([\#10500](https://github.com/matrix-org/synapse/issues/10500))
+
+
+Synapse 1.39.0 (2021-07-29)
+===========================
+
+No significant changes.
+
+
+Synapse 1.39.0rc3 (2021-07-28)
+==============================
+
+Bugfixes
+--------
+
+- Fix a bug introduced in Synapse 1.38 which caused an exception at startup when SAML authentication was enabled. ([\#10477](https://github.com/matrix-org/synapse/issues/10477))
+- Fix a long-standing bug where Synapse would not inform clients that a device had exhausted its one-time-key pool, potentially causing problems decrypting events. ([\#10485](https://github.com/matrix-org/synapse/issues/10485))
+- Fix reporting old R30 stats as R30v2 stats. Introduced in v1.39.0rc1. ([\#10486](https://github.com/matrix-org/synapse/issues/10486))
+
+
+Internal Changes
+----------------
+
+- Fix an error which prevented the Github Actions workflow to build the docker images from running. ([\#10461](https://github.com/matrix-org/synapse/issues/10461))
+- Fix release script to correctly version debian changelog when doing RCs. ([\#10465](https://github.com/matrix-org/synapse/issues/10465))
+
+
+Synapse 1.39.0rc2 (2021-07-22)
+==============================
+
+This release also includes the changes in v1.38.1.
+
+
+Internal Changes
+----------------
+
+- Move docker image build to Github Actions. ([\#10416](https://github.com/matrix-org/synapse/issues/10416))
+
+
+Synapse 1.38.1 (2021-07-22)
+===========================
+
+Bugfixes
+--------
+
+- Always include `device_one_time_keys_count` key in `/sync` response to work around a bug in Element Android that broke encryption for new devices. ([\#10457](https://github.com/matrix-org/synapse/issues/10457))
+
+
+Synapse 1.39.0rc1 (2021-07-20)
+==============================
+
+The Third-Party Event Rules module interface has been deprecated in favour of the generic module interface introduced in Synapse v1.37.0. Support for the old interface is planned to be removed in September 2021. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html#upgrading-to-v1390) for more information.
+
+Features
+--------
+
+- Add the ability to override the account validity feature with a module. ([\#9884](https://github.com/matrix-org/synapse/issues/9884))
+- The spaces summary API now returns any joinable rooms, not only rooms which are world-readable. ([\#10298](https://github.com/matrix-org/synapse/issues/10298), [\#10305](https://github.com/matrix-org/synapse/issues/10305))
+- Add a new version of the R30 phone-home metric, which removes a false impression of retention given by the old R30 metric. ([\#10332](https://github.com/matrix-org/synapse/issues/10332), [\#10427](https://github.com/matrix-org/synapse/issues/10427))
+- Allow providing credentials to `http_proxy`. ([\#10360](https://github.com/matrix-org/synapse/issues/10360))
+
+
+Bugfixes
+--------
+
+- Fix error while dropping locks on shutdown. Introduced in v1.38.0. ([\#10433](https://github.com/matrix-org/synapse/issues/10433))
+- Add base starting insertion event when no chunk ID is specified in the historical batch send API. ([\#10250](https://github.com/matrix-org/synapse/issues/10250))
+- Fix historical batch send endpoint (MSC2716) rejecting batches with messages from multiple senders. ([\#10276](https://github.com/matrix-org/synapse/issues/10276))
+- Fix purging rooms that other homeservers are still sending events for. Contributed by @ilmari. ([\#10317](https://github.com/matrix-org/synapse/issues/10317))
+- Fix errors during backfill caused by previously purged redaction events. Contributed by Andreas Rammhold (@andir). ([\#10343](https://github.com/matrix-org/synapse/issues/10343))
+- Fix the user directory becoming broken (and noisy errors being logged) when knocking and room statistics are in use. ([\#10344](https://github.com/matrix-org/synapse/issues/10344))
+- Fix newly added `synapse_federation_server_oldest_inbound_pdu_in_staging` prometheus metric to measure age rather than timestamp. ([\#10355](https://github.com/matrix-org/synapse/issues/10355))
+- Fix PostgreSQL sometimes using table scans for queries against `state_groups_state` table, taking a long time and a large amount of IO. ([\#10359](https://github.com/matrix-org/synapse/issues/10359))
+- Fix `make_room_admin` failing for users that have left a private room. ([\#10367](https://github.com/matrix-org/synapse/issues/10367))
+- Fix a number of logged errors caused by remote servers being down. ([\#10400](https://github.com/matrix-org/synapse/issues/10400), [\#10414](https://github.com/matrix-org/synapse/issues/10414))
+- Responses from `/make_{join,leave,knock}` no longer include signatures, which will turn out to be invalid after events are returned to `/send_{join,leave,knock}`. ([\#10404](https://github.com/matrix-org/synapse/issues/10404))
+
+
+Improved Documentation
+----------------------
+
+- Updated installation dependencies for newer macOS versions and ARM Macs. Contributed by Luke Walsh. ([\#9971](https://github.com/matrix-org/synapse/issues/9971))
+- Simplify structure of room admin API. ([\#10313](https://github.com/matrix-org/synapse/issues/10313))
+- Refresh the logcontext dev documentation. ([\#10353](https://github.com/matrix-org/synapse/issues/10353)), ([\#10337](https://github.com/matrix-org/synapse/issues/10337))
+- Add delegation example for caddy in the reverse proxy documentation. Contributed by @moritzdietz. ([\#10368](https://github.com/matrix-org/synapse/issues/10368))
+- Fix and clarify some links in `docs` and `contrib`. ([\#10370](https://github.com/matrix-org/synapse/issues/10370)), ([\#10322](https://github.com/matrix-org/synapse/issues/10322)), ([\#10399](https://github.com/matrix-org/synapse/issues/10399))
+- Make deprecation notice of the spam checker doc more obvious. ([\#10395](https://github.com/matrix-org/synapse/issues/10395))
+- Add instructions on installing Debian packages for release candidates. ([\#10396](https://github.com/matrix-org/synapse/issues/10396))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove functionality associated with the unused `room_stats_historical` and `user_stats_historical` tables. Contributed by @xmunoz. ([\#9721](https://github.com/matrix-org/synapse/issues/9721))
+- The third-party event rules module interface is deprecated in favour of the generic module interface introduced in Synapse v1.37.0. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html#upgrading-to-v1390) for more information. ([\#10386](https://github.com/matrix-org/synapse/issues/10386))
+
+
+Internal Changes
+----------------
+
+- Convert `room_depth.min_depth` column to a `BIGINT`. ([\#10289](https://github.com/matrix-org/synapse/issues/10289))
+- Add tests to characterise the current behaviour of R30 phone-home metrics. ([\#10315](https://github.com/matrix-org/synapse/issues/10315))
+- Rebuild event context and auth when processing specific results from `ThirdPartyEventRules` modules. ([\#10316](https://github.com/matrix-org/synapse/issues/10316))
+- Minor change to the code that populates `user_daily_visits`. ([\#10324](https://github.com/matrix-org/synapse/issues/10324))
+- Re-enable Sytests that were disabled for the 1.37.1 release. ([\#10345](https://github.com/matrix-org/synapse/issues/10345), [\#10357](https://github.com/matrix-org/synapse/issues/10357))
+- Run `pyupgrade` on the codebase. ([\#10347](https://github.com/matrix-org/synapse/issues/10347), [\#10348](https://github.com/matrix-org/synapse/issues/10348))
+- Switch `application_services_txns.txn_id` database column to `BIGINT`. ([\#10349](https://github.com/matrix-org/synapse/issues/10349))
+- Convert internal type variable syntax to reflect wider ecosystem use. ([\#10350](https://github.com/matrix-org/synapse/issues/10350), [\#10380](https://github.com/matrix-org/synapse/issues/10380), [\#10381](https://github.com/matrix-org/synapse/issues/10381), [\#10382](https://github.com/matrix-org/synapse/issues/10382), [\#10418](https://github.com/matrix-org/synapse/issues/10418))
+- Make the Github Actions workflow configuration more efficient. ([\#10383](https://github.com/matrix-org/synapse/issues/10383))
+- Add type hints to `get_{domain,localpart}_from_id`. ([\#10385](https://github.com/matrix-org/synapse/issues/10385))
+- When building Debian packages for prerelease versions, set the Section accordingly. ([\#10391](https://github.com/matrix-org/synapse/issues/10391))
+- Add type hints and comments to event auth code. ([\#10393](https://github.com/matrix-org/synapse/issues/10393))
+- Stagger sending of presence update to remote servers, reducing CPU spikes caused by starting many connections to remote servers at once. ([\#10398](https://github.com/matrix-org/synapse/issues/10398))
+- Remove unused `events_by_room` code (tech debt). ([\#10421](https://github.com/matrix-org/synapse/issues/10421))
+- Add a github actions job which records success of other jobs. ([\#10430](https://github.com/matrix-org/synapse/issues/10430))
+
+
+Synapse 1.38.0 (2021-07-13)
+===========================
+
+This release includes a database schema update which could result in elevated disk usage. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1380) for more information.
+
+No significant changes since 1.38.0rc3.
+
+
+Synapse 1.38.0rc3 (2021-07-13)
+==============================
+
+Internal Changes
+----------------
+
+- Build the Debian packages in CI. ([\#10247](https://github.com/matrix-org/synapse/issues/10247), [\#10379](https://github.com/matrix-org/synapse/issues/10379))
+
+
+Synapse 1.38.0rc2 (2021-07-09)
+==============================
+
+Bugfixes
+--------
+
+- Fix bug where inbound federation in a room could be delayed due to not correctly dropping a lock. Introduced in v1.37.1. ([\#10336](https://github.com/matrix-org/synapse/issues/10336))
+
+
+Improved Documentation
+----------------------
+
+- Update links to documentation in the sample config. Contributed by @dklimpel. ([\#10287](https://github.com/matrix-org/synapse/issues/10287))
+- Fix broken links in [INSTALL.md](INSTALL.md). Contributed by @dklimpel. ([\#10331](https://github.com/matrix-org/synapse/issues/10331))
+
+
+Synapse 1.38.0rc1 (2021-07-06)
+==============================
+
+Features
+--------
+
+- Implement refresh tokens as specified by [MSC2918](https://github.com/matrix-org/matrix-doc/pull/2918). ([\#9450](https://github.com/matrix-org/synapse/issues/9450))
+- Add support for evicting cache entries based on last access time. ([\#10205](https://github.com/matrix-org/synapse/issues/10205))
+- Omit empty fields from the `/sync` response. Contributed by @deepbluev7. ([\#10214](https://github.com/matrix-org/synapse/issues/10214))
+- Improve validation on federation `send_{join,leave,knock}` endpoints. ([\#10225](https://github.com/matrix-org/synapse/issues/10225), [\#10243](https://github.com/matrix-org/synapse/issues/10243))
+- Add SSO `external_ids` to the Query User Account admin API. ([\#10261](https://github.com/matrix-org/synapse/issues/10261))
+- Mark events received over federation which fail a spam check as "soft-failed". ([\#10263](https://github.com/matrix-org/synapse/issues/10263))
+- Add metrics for new inbound federation staging area. ([\#10284](https://github.com/matrix-org/synapse/issues/10284))
+- Add script to print information about recently registered users. ([\#10290](https://github.com/matrix-org/synapse/issues/10290))
+
+
+Bugfixes
+--------
+
+- Fix a long-standing bug which meant that invite rejections and knocks were not sent out over federation in a timely manner. ([\#10223](https://github.com/matrix-org/synapse/issues/10223))
+- Fix a bug introduced in v1.26.0 where only users who have set profile information could be deactivated with erasure enabled. ([\#10252](https://github.com/matrix-org/synapse/issues/10252))
+- Fix a long-standing bug where Synapse would return errors after 2<sup>31</sup> events were handled by the server. ([\#10264](https://github.com/matrix-org/synapse/issues/10264), [\#10267](https://github.com/matrix-org/synapse/issues/10267), [\#10282](https://github.com/matrix-org/synapse/issues/10282), [\#10286](https://github.com/matrix-org/synapse/issues/10286), [\#10291](https://github.com/matrix-org/synapse/issues/10291), [\#10314](https://github.com/matrix-org/synapse/issues/10314), [\#10326](https://github.com/matrix-org/synapse/issues/10326))
+- Fix the prometheus `synapse_federation_server_pdu_process_time` metric. Broke in v1.37.1. ([\#10279](https://github.com/matrix-org/synapse/issues/10279))
+- Ensure that inbound events from federation that were being processed when Synapse was restarted get promptly processed on start up. ([\#10303](https://github.com/matrix-org/synapse/issues/10303))
+
+
+Improved Documentation
+----------------------
+
+- Move the upgrade notes to [docs/upgrade.md](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md) and convert them to markdown. ([\#10166](https://github.com/matrix-org/synapse/issues/10166))
+- Choose Welcome & Overview as the default page for synapse documentation website. ([\#10242](https://github.com/matrix-org/synapse/issues/10242))
+- Adjust the URL in the README.rst file to point to irc.libera.chat. ([\#10258](https://github.com/matrix-org/synapse/issues/10258))
+- Fix homeserver config option name in presence router documentation. ([\#10288](https://github.com/matrix-org/synapse/issues/10288))
+- Fix link pointing at the wrong section in the modules documentation page. ([\#10302](https://github.com/matrix-org/synapse/issues/10302))
+
+
+Internal Changes
+----------------
+
+- Drop `Origin` and `Accept` from the value of the `Access-Control-Allow-Headers` response header. ([\#10114](https://github.com/matrix-org/synapse/issues/10114))
+- Add type hints to the federation servlets. ([\#10213](https://github.com/matrix-org/synapse/issues/10213))
+- Improve the reliability of auto-joining remote rooms. ([\#10237](https://github.com/matrix-org/synapse/issues/10237))
+- Update the release script to use the semver terminology and determine the release branch based on the next version. ([\#10239](https://github.com/matrix-org/synapse/issues/10239))
+- Fix type hints for computing auth events. ([\#10253](https://github.com/matrix-org/synapse/issues/10253))
+- Improve the performance of the spaces summary endpoint by only recursing into spaces (and not rooms in general). ([\#10256](https://github.com/matrix-org/synapse/issues/10256))
+- Move event authentication methods from `Auth` to `EventAuthHandler`. ([\#10268](https://github.com/matrix-org/synapse/issues/10268))
+- Re-enable a SyTest after it has been fixed. ([\#10292](https://github.com/matrix-org/synapse/issues/10292))
+
+
+Synapse 1.37.1 (2021-06-30)
+===========================
+
+This release resolves issues (such as [#9490](https://github.com/matrix-org/synapse/issues/9490)) where one busy room could cause head-of-line blocking, starving Synapse from processing events in other rooms, and causing all federated traffic to fall behind. Synapse 1.37.1 processes inbound federation traffic asynchronously, ensuring that one busy room won't impact others. Please upgrade to Synapse 1.37.1 as soon as possible, in order to increase resilience to other traffic spikes.
+
+No significant changes since v1.37.1rc1.
+
+
+Synapse 1.37.1rc1 (2021-06-29)
+==============================
+
+Features
+--------
+
+- Handle inbound events from federation asynchronously. ([\#10269](https://github.com/matrix-org/synapse/issues/10269), [\#10272](https://github.com/matrix-org/synapse/issues/10272))
+
+
+Synapse 1.37.0 (2021-06-29)
+===========================
+
+This release deprecates the current spam checker interface. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new generic module interface.
+
+This release also removes support for fetching and renewing TLS certificates using the ACME v1 protocol, which has been fully decommissioned by Let's Encrypt on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings.
+
+Synapse 1.37.0rc1 (2021-06-24)
+==============================
+
+Features
+--------
+
+- Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by @Sorunome and anoa. ([\#6739](https://github.com/matrix-org/synapse/issues/6739), [\#9359](https://github.com/matrix-org/synapse/issues/9359), [\#10167](https://github.com/matrix-org/synapse/issues/10167), [\#10212](https://github.com/matrix-org/synapse/issues/10212), [\#10227](https://github.com/matrix-org/synapse/issues/10227))
+- Add experimental support for backfilling history into rooms ([MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). ([\#9247](https://github.com/matrix-org/synapse/issues/9247))
+- Implement a generic interface for third-party plugin modules. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10206](https://github.com/matrix-org/synapse/issues/10206))
+- Implement config option `sso.update_profile_information` to sync SSO users' profile information with the identity provider each time they login. Currently only displayname is supported. ([\#10108](https://github.com/matrix-org/synapse/issues/10108))
+- Ensure that errors during startup are written to the logs and the console. ([\#10191](https://github.com/matrix-org/synapse/issues/10191))
+
+
+Bugfixes
+--------
+
+- Fix a bug introduced in Synapse v1.25.0 that prevented the `ip_range_whitelist` configuration option from working for federation and identity servers. Contributed by @mikure. ([\#10115](https://github.com/matrix-org/synapse/issues/10115))
+- Remove a broken import line in Synapse's `admin_cmd` worker. Broke in Synapse v1.33.0. ([\#10154](https://github.com/matrix-org/synapse/issues/10154))
+- Fix a bug introduced in Synapse v1.21.0 which could cause `/sync` to return immediately with an empty response. ([\#10157](https://github.com/matrix-org/synapse/issues/10157), [\#10158](https://github.com/matrix-org/synapse/issues/10158))
+- Fix a minor bug in the response to `/_matrix/client/r0/user/{user}/openid/request_token` causing `expires_in` to be a float instead of an integer. Contributed by @lukaslihotzki. ([\#10175](https://github.com/matrix-org/synapse/issues/10175))
+- Always require users to re-authenticate for dangerous operations: deactivating an account, modifying an account password, and adding 3PIDs. ([\#10184](https://github.com/matrix-org/synapse/issues/10184))
+- Fix a bug introduced in Synpase v1.7.2 where remote server count metrics collection would be incorrectly delayed on startup. Found by @heftig. ([\#10195](https://github.com/matrix-org/synapse/issues/10195))
+- Fix a bug introduced in Synapse v1.35.1 where an `allow` key of a `m.room.join_rules` event could be applied for incorrect room versions and configurations. ([\#10208](https://github.com/matrix-org/synapse/issues/10208))
+- Fix performance regression in responding to user key requests over federation. Introduced in Synapse v1.34.0rc1. ([\#10221](https://github.com/matrix-org/synapse/issues/10221))
+
+
+Improved Documentation
+----------------------
+
+- Add a new guide to decoding request logs. ([\#8436](https://github.com/matrix-org/synapse/issues/8436))
+- Mention in the sample homeserver config that you may need to configure max upload size in your reverse proxy. Contributed by @aaronraimist. ([\#10122](https://github.com/matrix-org/synapse/issues/10122))
+- Fix broken links in documentation. ([\#10180](https://github.com/matrix-org/synapse/issues/10180))
+- Deploy a snapshot of the documentation website upon each new Synapse release. ([\#10198](https://github.com/matrix-org/synapse/issues/10198))
+
+
+Deprecations and Removals
+-------------------------
+
+- The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10210](https://github.com/matrix-org/synapse/issues/10210), [\#10238](https://github.com/matrix-org/synapse/issues/10238))
+- Stop supporting the unstable spaces prefixes from MSC1772. ([\#10161](https://github.com/matrix-org/synapse/issues/10161))
+- Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing installations on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. ([\#10194](https://github.com/matrix-org/synapse/issues/10194))
+
+
+Internal Changes
+----------------
+
+- Update the database schema versioning to support gradual migration away from legacy tables. ([\#9933](https://github.com/matrix-org/synapse/issues/9933))
+- Add type hints to the federation servlets. ([\#10080](https://github.com/matrix-org/synapse/issues/10080))
+- Improve OpenTracing for event persistence. ([\#10134](https://github.com/matrix-org/synapse/issues/10134), [\#10193](https://github.com/matrix-org/synapse/issues/10193))
+- Clean up the interface for injecting OpenTracing over HTTP. ([\#10143](https://github.com/matrix-org/synapse/issues/10143))
+- Limit the number of in-flight `/keys/query` requests from a single device. ([\#10144](https://github.com/matrix-org/synapse/issues/10144))
+- Refactor EventPersistenceQueue. ([\#10145](https://github.com/matrix-org/synapse/issues/10145))
+- Document `SYNAPSE_TEST_LOG_LEVEL` to see the logger output when running tests. ([\#10148](https://github.com/matrix-org/synapse/issues/10148))
+- Update the Complement build tags in GitHub Actions to test currently experimental features. ([\#10155](https://github.com/matrix-org/synapse/issues/10155))
+- Add a `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. ([\#10156](https://github.com/matrix-org/synapse/issues/10156))
+- Fetch the corresponding complement branch when performing CI. ([\#10160](https://github.com/matrix-org/synapse/issues/10160))
+- Add some developer documentation about boolean columns in database schemas. ([\#10164](https://github.com/matrix-org/synapse/issues/10164))
+- Add extra logging fields to better debug where events are being soft failed. ([\#10168](https://github.com/matrix-org/synapse/issues/10168))
+- Add debug logging for when we enter and exit `Measure` blocks. ([\#10183](https://github.com/matrix-org/synapse/issues/10183))
+- Improve comments in structured logging code. ([\#10188](https://github.com/matrix-org/synapse/issues/10188))
+- Update [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) support with modifications from the MSC. ([\#10189](https://github.com/matrix-org/synapse/issues/10189))
+- Remove redundant DNS lookup limiter. ([\#10190](https://github.com/matrix-org/synapse/issues/10190))
+- Upgrade `black` linting tool to 21.6b0. ([\#10197](https://github.com/matrix-org/synapse/issues/10197))
+- Expose OpenTracing trace id in response headers. ([\#10199](https://github.com/matrix-org/synapse/issues/10199))
+
+
+Synapse 1.36.0 (2021-06-15)
+===========================
+
+No significant changes.
+
+
+Synapse 1.36.0rc2 (2021-06-11)
+==============================
+
+Bugfixes
+--------
+
+- Fix a bug which caused  presence updates to stop working some time after a restart, when using a presence writer worker. Broke in v1.33.0. ([\#10149](https://github.com/matrix-org/synapse/issues/10149))
+- Fix a bug when using federation sender worker where it would send out more presence updates than necessary, leading to high resource usage. Broke in v1.33.0. ([\#10163](https://github.com/matrix-org/synapse/issues/10163))
+- Fix a bug where Synapse could send the same presence update to a remote twice. ([\#10165](https://github.com/matrix-org/synapse/issues/10165))
+
+
+Synapse 1.36.0rc1 (2021-06-08)
+==============================
+
+Features
+--------
+
+- Add new endpoint `/_matrix/client/r0/rooms/{roomId}/aliases` from Client-Server API r0.6.1 (previously [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432)). ([\#9224](https://github.com/matrix-org/synapse/issues/9224))
+- Improve performance of incoming federation transactions in large rooms. ([\#9953](https://github.com/matrix-org/synapse/issues/9953), [\#9973](https://github.com/matrix-org/synapse/issues/9973))
+- Rewrite logic around verifying JSON object and fetching server keys to be more performant and use less memory. ([\#10035](https://github.com/matrix-org/synapse/issues/10035))
+- Add new admin APIs for unprotecting local media from quarantine. Contributed by @dklimpel. ([\#10040](https://github.com/matrix-org/synapse/issues/10040))
+- Add new admin APIs to remove media by media ID from quarantine. Contributed by @dklimpel. ([\#10044](https://github.com/matrix-org/synapse/issues/10044))
+- Make reason and score parameters optional for reporting content. Implements [MSC2414](https://github.com/matrix-org/matrix-doc/pull/2414). Contributed by Callum Brown. ([\#10077](https://github.com/matrix-org/synapse/issues/10077))
+- Add support for routing more requests to workers. ([\#10084](https://github.com/matrix-org/synapse/issues/10084))
+- Report OpenTracing spans for database activity. ([\#10113](https://github.com/matrix-org/synapse/issues/10113), [\#10136](https://github.com/matrix-org/synapse/issues/10136), [\#10141](https://github.com/matrix-org/synapse/issues/10141))
+- Significantly reduce memory usage of joining large remote rooms. ([\#10117](https://github.com/matrix-org/synapse/issues/10117))
+
+
+Bugfixes
+--------
+
+- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. ([\#10082](https://github.com/matrix-org/synapse/issues/10082))
+- Fix a bug in the `force_tracing_for_users` option introduced in Synapse v1.35 which meant that the OpenTracing spans produced were missing most tags. ([\#10092](https://github.com/matrix-org/synapse/issues/10092))
+- Fixed a bug that could cause Synapse to stop notifying application services. Contributed by Willem Mulder. ([\#10107](https://github.com/matrix-org/synapse/issues/10107))
+- Fix bug where the server would attempt to fetch the same history in the room from a remote server multiple times in parallel. ([\#10116](https://github.com/matrix-org/synapse/issues/10116))
+- Fix a bug introduced in Synapse 1.33.0 which caused replication requests to fail when receiving a lot of very large events via federation. ([\#10118](https://github.com/matrix-org/synapse/issues/10118))
+- Fix bug when using workers where pagination requests failed if a remote server returned zero events from `/backfill`. Introduced in 1.35.0. ([\#10133](https://github.com/matrix-org/synapse/issues/10133))
+
+
+Improved Documentation
+----------------------
+
+- Clarify security note regarding hosting Synapse on the same domain as other web applications. ([\#9221](https://github.com/matrix-org/synapse/issues/9221))
+- Update CAPTCHA documentation to mention turning off the verify origin feature. Contributed by @aaronraimist. ([\#10046](https://github.com/matrix-org/synapse/issues/10046))
+- Tweak wording of database recommendation in `INSTALL.md`. Contributed by @aaronraimist. ([\#10057](https://github.com/matrix-org/synapse/issues/10057))
+- Add initial infrastructure for rendering Synapse documentation with mdbook. ([\#10086](https://github.com/matrix-org/synapse/issues/10086))
+- Convert the remaining Admin API documentation files to markdown. ([\#10089](https://github.com/matrix-org/synapse/issues/10089))
+- Make a link in docs use HTTPS. Contributed by @RhnSharma. ([\#10130](https://github.com/matrix-org/synapse/issues/10130))
+- Fix broken link in Docker docs. ([\#10132](https://github.com/matrix-org/synapse/issues/10132))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove the experimental `spaces_enabled` flag. The spaces features are always available now. ([\#10063](https://github.com/matrix-org/synapse/issues/10063))
+
+
+Internal Changes
+----------------
+
+- Tell CircleCI to build Docker images from `main` branch. ([\#9906](https://github.com/matrix-org/synapse/issues/9906))
+- Simplify naming convention for release branches to only include the major and minor version numbers. ([\#10013](https://github.com/matrix-org/synapse/issues/10013))
+- Add `parse_strings_from_args` for parsing an array from query parameters. ([\#10048](https://github.com/matrix-org/synapse/issues/10048), [\#10137](https://github.com/matrix-org/synapse/issues/10137))
+- Remove some dead code regarding TLS certificate handling. ([\#10054](https://github.com/matrix-org/synapse/issues/10054))
+- Remove redundant, unmaintained `convert_server_keys` script. ([\#10055](https://github.com/matrix-org/synapse/issues/10055))
+- Improve the error message printed by synctl when synapse fails to start. ([\#10059](https://github.com/matrix-org/synapse/issues/10059))
+- Fix GitHub Actions lint for newsfragments. ([\#10069](https://github.com/matrix-org/synapse/issues/10069))
+- Update opentracing to inject the right context into the carrier. ([\#10074](https://github.com/matrix-org/synapse/issues/10074))
+- Fix up `BatchingQueue` implementation. ([\#10078](https://github.com/matrix-org/synapse/issues/10078))
+- Log method and path when dropping request due to size limit. ([\#10091](https://github.com/matrix-org/synapse/issues/10091))
+- In Github Actions workflows, summarize the Sytest results in an easy-to-read format. ([\#10094](https://github.com/matrix-org/synapse/issues/10094))
+- Make `/sync` do fewer state resolutions. ([\#10102](https://github.com/matrix-org/synapse/issues/10102))
+- Add missing type hints to the admin API servlets. ([\#10105](https://github.com/matrix-org/synapse/issues/10105))
+- Improve opentracing annotations for `Notifier`. ([\#10111](https://github.com/matrix-org/synapse/issues/10111))
+- Enable Prometheus metrics for the jaeger client library. ([\#10112](https://github.com/matrix-org/synapse/issues/10112))
+- Work to improve the responsiveness of `/sync` requests. ([\#10124](https://github.com/matrix-org/synapse/issues/10124))
+- OpenTracing: use a consistent name for background processes. ([\#10135](https://github.com/matrix-org/synapse/issues/10135))
+
+
+Synapse 1.35.1 (2021-06-03)
+===========================
+
+Bugfixes
+--------
+
+- Fix a bug introduced in v1.35.0 where invite-only rooms would be shown to all users in a space, regardless of if the user had access to it. ([\#10109](https://github.com/matrix-org/synapse/issues/10109))
+
+
+Synapse 1.35.0 (2021-06-01)
+===========================
+
+Note that [the tag](https://github.com/matrix-org/synapse/releases/tag/v1.35.0rc3) and [docker images](https://hub.docker.com/layers/matrixdotorg/synapse/v1.35.0rc3/images/sha256-34ccc87bd99a17e2cbc0902e678b5937d16bdc1991ead097eee6096481ecf2c4?context=explore) for `v1.35.0rc3` were incorrectly built. If you are experiencing issues with either, it is recommended to upgrade to the equivalent tag or docker image for the `v1.35.0` release.
+
+Deprecations and Removals
+-------------------------
+
+- The core Synapse development team plan to drop support for the [unstable API of MSC2858](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), including the undocumented `experimental.msc2858_enabled` config option, in August 2021. Client authors should ensure that their clients are updated to use the stable API (which has been supported since Synapse 1.30) well before that time, to give their users time to upgrade. ([\#10101](https://github.com/matrix-org/synapse/issues/10101))
+
+Bugfixes
+--------
+
+- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. Introduced in v1.33.0. ([\#10082](https://github.com/matrix-org/synapse/issues/10082))
+- Fix HTTP response size limit to allow joining very large rooms over federation. Introduced in v1.33.0. ([\#10093](https://github.com/matrix-org/synapse/issues/10093))
+
+
+Internal Changes
+----------------
+
+- Log method and path when dropping request due to size limit. ([\#10091](https://github.com/matrix-org/synapse/issues/10091))
+
+
+Synapse 1.35.0rc2 (2021-05-27)
+==============================
+
+Bugfixes
+--------
+
+- Fix a bug introduced in v1.35.0rc1 when calling the spaces summary API via a GET request. ([\#10079](https://github.com/matrix-org/synapse/issues/10079))
+
+
+Synapse 1.35.0rc1 (2021-05-25)
+==============================
+
+Features
+--------
+
+- Add experimental support to allow a user who could join a restricted room to view it in the spaces summary. ([\#9922](https://github.com/matrix-org/synapse/issues/9922), [\#10007](https://github.com/matrix-org/synapse/issues/10007), [\#10038](https://github.com/matrix-org/synapse/issues/10038))
+- Reduce memory usage when joining very large rooms over federation. ([\#9958](https://github.com/matrix-org/synapse/issues/9958))
+- Add a configuration option which allows enabling opentracing by user id. ([\#9978](https://github.com/matrix-org/synapse/issues/9978))
+- Enable experimental support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) (spaces summary API) and [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) (restricted join rules) by default. ([\#10011](https://github.com/matrix-org/synapse/issues/10011))
+
+
+Bugfixes
+--------
+
+- Fix a bug introduced in v1.26.0 which meant that `synapse_port_db` would not correctly initialise some postgres sequences, requiring manual updates afterwards. ([\#9991](https://github.com/matrix-org/synapse/issues/9991))
+- Fix `synctl`'s `--no-daemonize` parameter to work correctly with worker processes. ([\#9995](https://github.com/matrix-org/synapse/issues/9995))
+- Fix a validation bug introduced in v1.34.0 in the ordering of spaces in the space summary API. ([\#10002](https://github.com/matrix-org/synapse/issues/10002))
+- Fixed deletion of new presence stream states from database. ([\#10014](https://github.com/matrix-org/synapse/issues/10014), [\#10033](https://github.com/matrix-org/synapse/issues/10033))
+- Fixed a bug with very high resolution image uploads throwing internal server errors. ([\#10029](https://github.com/matrix-org/synapse/issues/10029))
+
+
+Updates to the Docker image
+---------------------------
+
+- Fix bug introduced in Synapse 1.33.0 which caused a `Permission denied: '/homeserver.log'` error when starting Synapse with the generated log configuration. Contributed by Sergio Miguéns Iglesias. ([\#10045](https://github.com/matrix-org/synapse/issues/10045))
+
+
+Improved Documentation
+----------------------
+
+- Add hardened systemd files as proposed in [#9760](https://github.com/matrix-org/synapse/issues/9760) and added them to `contrib/`. Change the docs to reflect the presence of these files. ([\#9803](https://github.com/matrix-org/synapse/issues/9803))
+- Clarify documentation around SSO mapping providers generating unique IDs and localparts. ([\#9980](https://github.com/matrix-org/synapse/issues/9980))
+- Updates to the PostgreSQL documentation (`postgres.md`). ([\#9988](https://github.com/matrix-org/synapse/issues/9988), [\#9989](https://github.com/matrix-org/synapse/issues/9989))
+- Fix broken link in user directory documentation. Contributed by @junquera. ([\#10016](https://github.com/matrix-org/synapse/issues/10016))
+- Add missing room state entry to the table of contents of room admin API. ([\#10043](https://github.com/matrix-org/synapse/issues/10043))
+
+
+Deprecations and Removals
+-------------------------
+
+- Removed support for the deprecated `tls_fingerprints` configuration setting. Contributed by Jerin J Titus. ([\#9280](https://github.com/matrix-org/synapse/issues/9280))
+
+
+Internal Changes
+----------------
+
+- Allow sending full presence to users via workers other than the one that called `ModuleApi.send_local_online_presence_to`. ([\#9823](https://github.com/matrix-org/synapse/issues/9823))
+- Update comments in the space summary handler. ([\#9974](https://github.com/matrix-org/synapse/issues/9974))
+- Minor enhancements to the `@cachedList` descriptor. ([\#9975](https://github.com/matrix-org/synapse/issues/9975))
+- Split multipart email sending into a dedicated handler. ([\#9977](https://github.com/matrix-org/synapse/issues/9977))
+- Run `black` on files in the `scripts` directory. ([\#9981](https://github.com/matrix-org/synapse/issues/9981))
+- Add missing type hints to `synapse.util` module. ([\#9982](https://github.com/matrix-org/synapse/issues/9982))
+- Simplify a few helper functions. ([\#9984](https://github.com/matrix-org/synapse/issues/9984), [\#9985](https://github.com/matrix-org/synapse/issues/9985), [\#9986](https://github.com/matrix-org/synapse/issues/9986))
+- Remove unnecessary property from SQLBaseStore. ([\#9987](https://github.com/matrix-org/synapse/issues/9987))
+- Remove `keylen` param on `LruCache`. ([\#9993](https://github.com/matrix-org/synapse/issues/9993))
+- Update the Grafana dashboard in `contrib/`. ([\#10001](https://github.com/matrix-org/synapse/issues/10001))
+- Add a batching queue implementation. ([\#10017](https://github.com/matrix-org/synapse/issues/10017))
+- Reduce memory usage when verifying signatures on large numbers of events at once. ([\#10018](https://github.com/matrix-org/synapse/issues/10018))
+- Properly invalidate caches for destination retry timings every (instead of expiring entries every 5 minutes). ([\#10036](https://github.com/matrix-org/synapse/issues/10036))
+- Fix running complement tests with Synapse workers. ([\#10039](https://github.com/matrix-org/synapse/issues/10039))
+- Fix typo in `get_state_ids_for_event` docstring where the return type was incorrect. ([\#10050](https://github.com/matrix-org/synapse/issues/10050))
+
+
+Synapse 1.34.0 (2021-05-17)
+===========================
+
+This release deprecates the `room_invite_state_types` configuration setting. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.34.0/UPGRADE.rst#upgrading-to-v1340) for instructions on updating your configuration file to use the new `room_prejoin_state` setting.
+
+This release also deprecates the `POST /_synapse/admin/v1/rooms/<room_id>/delete` admin API route. Server administrators are encouraged to update their scripts to use the new `DELETE /_synapse/admin/v1/rooms/<room_id>` route instead.
+
+
+No significant changes since v1.34.0rc1.
+
+
+Synapse 1.34.0rc1 (2021-05-12)
+==============================
+
+Features
+--------
+
+- Add experimental option to track memory usage of the caches. ([\#9881](https://github.com/matrix-org/synapse/issues/9881))
+- Add support for `DELETE /_synapse/admin/v1/rooms/<room_id>`. ([\#9889](https://github.com/matrix-org/synapse/issues/9889))
+- Add limits to how often Synapse will GC, ensuring that large servers do not end up GC thrashing if `gc_thresholds` has not been correctly set. ([\#9902](https://github.com/matrix-org/synapse/issues/9902))
+- Improve performance of sending events for worker-based deployments using Redis. ([\#9905](https://github.com/matrix-org/synapse/issues/9905), [\#9950](https://github.com/matrix-org/synapse/issues/9950), [\#9951](https://github.com/matrix-org/synapse/issues/9951))
+- Improve performance after joining a large room when presence is enabled. ([\#9910](https://github.com/matrix-org/synapse/issues/9910), [\#9916](https://github.com/matrix-org/synapse/issues/9916))
+- Support stable identifiers for [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772) Spaces. `m.space.child` events will now be taken into account when populating the experimental spaces summary response. Please see [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.34.0/UPGRADE.rst#upgrading-to-v1340) if you have customised `room_invite_state_types` in your configuration. ([\#9915](https://github.com/matrix-org/synapse/issues/9915), [\#9966](https://github.com/matrix-org/synapse/issues/9966))
+- Improve performance of backfilling in large rooms. ([\#9935](https://github.com/matrix-org/synapse/issues/9935))
+- Add a config option to allow you to prevent device display names from being shared over federation. Contributed by @aaronraimist. ([\#9945](https://github.com/matrix-org/synapse/issues/9945))
+- Update support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary. ([\#9947](https://github.com/matrix-org/synapse/issues/9947), [\#9954](https://github.com/matrix-org/synapse/issues/9954))
+
+
+Bugfixes
+--------
+
+- Fix a bug introduced in v1.32.0 where the associated connection was improperly logged for SQL logging statements. ([\#9895](https://github.com/matrix-org/synapse/issues/9895))
+- Correct the type hint for the `user_may_create_room_alias` method of spam checkers. It is provided a `RoomAlias`, not a `str`. ([\#9896](https://github.com/matrix-org/synapse/issues/9896))
+- Fix bug where user directory could get out of sync if room visibility and membership changed in quick succession. ([\#9910](https://github.com/matrix-org/synapse/issues/9910))
+- Include the `origin_server_ts` property in the experimental [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) support to allow clients to properly sort rooms. ([\#9928](https://github.com/matrix-org/synapse/issues/9928))
+- Fix bugs introduced in v1.23.0 which made the PostgreSQL port script fail when run with a newly-created SQLite database. ([\#9930](https://github.com/matrix-org/synapse/issues/9930))
+- Fix a bug introduced in Synapse 1.29.0 which caused `m.room_key_request` to-device messages sent from one user to another to be dropped. ([\#9961](https://github.com/matrix-org/synapse/issues/9961), [\#9965](https://github.com/matrix-org/synapse/issues/9965))
+- Fix a bug introduced in v1.27.0 preventing users and appservices exempt from ratelimiting from creating rooms with many invitees. ([\#9968](https://github.com/matrix-org/synapse/issues/9968))
+
+
+Updates to the Docker image
+---------------------------
+
+- Add `startup_delay` to docker healthcheck to reduce waiting time for coming online and update the documentation with extra options. Contributed by @Maquis196. ([\#9913](https://github.com/matrix-org/synapse/issues/9913))
+
+
+Improved Documentation
+----------------------
+
+- Add `port` argument to the Postgres database sample config section. ([\#9911](https://github.com/matrix-org/synapse/issues/9911))
+
+
+Deprecations and Removals
+-------------------------
+
+- Mark as deprecated `POST /_synapse/admin/v1/rooms/<room_id>/delete`. ([\#9889](https://github.com/matrix-org/synapse/issues/9889))
+
+
+Internal Changes
+----------------
+
+- Reduce the length of Synapse's access tokens. ([\#5588](https://github.com/matrix-org/synapse/issues/5588))
+- Export jemalloc stats to Prometheus if it is being used. ([\#9882](https://github.com/matrix-org/synapse/issues/9882))
+- Add type hints to presence handler. ([\#9885](https://github.com/matrix-org/synapse/issues/9885))
+- Reduce memory usage of the LRU caches. ([\#9886](https://github.com/matrix-org/synapse/issues/9886))
+- Add type hints to the `synapse.handlers` module. ([\#9896](https://github.com/matrix-org/synapse/issues/9896))
+- Time response time for external cache requests. ([\#9904](https://github.com/matrix-org/synapse/issues/9904))
+- Minor fixes to the `make_full_schema.sh` script. ([\#9931](https://github.com/matrix-org/synapse/issues/9931))
+- Move database schema files into a common directory. ([\#9932](https://github.com/matrix-org/synapse/issues/9932))
+- Add debug logging for lost/delayed to-device messages. ([\#9959](https://github.com/matrix-org/synapse/issues/9959))
+
+
+Synapse 1.33.2 (2021-05-11)
+===========================
+
+Due to the security issue highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild.
+
+Security advisory
+-----------------
+
+This release fixes a denial of service attack ([CVE-2021-29471](https://github.com/matrix-org/synapse/security/advisories/GHSA-x345-32rc-8h85)) against Synapse's push rules implementation. Server admins are encouraged to upgrade.
+
+Internal Changes
+----------------
+
+- Unpin attrs dependency. ([\#9946](https://github.com/matrix-org/synapse/issues/9946))
+
+
+Synapse 1.33.1 (2021-05-06)
+===========================
+
+Bugfixes
+--------
+
+- Fix bug where `/sync` would break if using the latest version of `attrs` dependency, by pinning to a previous version. ([\#9937](https://github.com/matrix-org/synapse/issues/9937))
+
+
+Synapse 1.33.0 (2021-05-05)
+===========================
+
+Features
+--------
+
+- Build Debian packages for Ubuntu 21.04 (Hirsute Hippo). ([\#9909](https://github.com/matrix-org/synapse/issues/9909))
+
+
+Synapse 1.33.0rc2 (2021-04-29)
+==============================
+
+Bugfixes
+--------
+
+- Fix tight loop when handling presence replication when using workers. Introduced in v1.33.0rc1. ([\#9900](https://github.com/matrix-org/synapse/issues/9900))
+
+
+Synapse 1.33.0rc1 (2021-04-28)
+==============================
+
+Features
+--------
+
+- Update experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership. ([\#9800](https://github.com/matrix-org/synapse/issues/9800), [\#9814](https://github.com/matrix-org/synapse/issues/9814))
+- Add experimental support for handling presence on a worker. ([\#9819](https://github.com/matrix-org/synapse/issues/9819), [\#9820](https://github.com/matrix-org/synapse/issues/9820), [\#9828](https://github.com/matrix-org/synapse/issues/9828), [\#9850](https://github.com/matrix-org/synapse/issues/9850))
+- Return a new template when an user attempts to renew their account multiple times with the same token, stating that their account is set to expire. This replaces the invalid token template that would previously be shown in this case. This change concerns the optional account validity feature. ([\#9832](https://github.com/matrix-org/synapse/issues/9832))
+
+
+Bugfixes
+--------
+
+- Fixes the OIDC SSO flow when using a `public_baseurl` value including a non-root URL path. ([\#9726](https://github.com/matrix-org/synapse/issues/9726))
+- Fix thumbnail generation for some sites with non-standard content types. Contributed by @rkfg. ([\#9788](https://github.com/matrix-org/synapse/issues/9788))
+- Add some sanity checks to identity server passed to 3PID bind/unbind endpoints. ([\#9802](https://github.com/matrix-org/synapse/issues/9802))
+- Limit the size of HTTP responses read over federation. ([\#9833](https://github.com/matrix-org/synapse/issues/9833))
+- Fix a bug which could cause Synapse to get stuck in a loop of resyncing device lists. ([\#9867](https://github.com/matrix-org/synapse/issues/9867))
+- Fix a long-standing bug where errors from federation did not propagate to the client. ([\#9868](https://github.com/matrix-org/synapse/issues/9868))
+
+
+Improved Documentation
+----------------------
+
+- Add a note to the docker docs mentioning that we mirror upstream's supported Docker platforms. ([\#9801](https://github.com/matrix-org/synapse/issues/9801))
+
+
+Internal Changes
+----------------
+
+- Add a dockerfile for running Synapse in worker-mode under Complement. ([\#9162](https://github.com/matrix-org/synapse/issues/9162))
+- Apply `pyupgrade` across the codebase. ([\#9786](https://github.com/matrix-org/synapse/issues/9786))
+- Move some replication processing out of `generic_worker`. ([\#9796](https://github.com/matrix-org/synapse/issues/9796))
+- Replace `HomeServer.get_config()` with inline references. ([\#9815](https://github.com/matrix-org/synapse/issues/9815))
+- Rename some handlers and config modules to not duplicate the top-level module. ([\#9816](https://github.com/matrix-org/synapse/issues/9816))
+- Fix a long-standing bug which caused `max_upload_size` to not be correctly enforced. ([\#9817](https://github.com/matrix-org/synapse/issues/9817))
+- Reduce CPU usage of the user directory by reusing existing calculated room membership. ([\#9821](https://github.com/matrix-org/synapse/issues/9821))
+- Small speed up for joining large remote rooms. ([\#9825](https://github.com/matrix-org/synapse/issues/9825))
+- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9838](https://github.com/matrix-org/synapse/issues/9838))
+- Only store the raw data in the in-memory caches, rather than objects that include references to e.g. the data stores. ([\#9845](https://github.com/matrix-org/synapse/issues/9845))
+- Limit length of accepted email addresses. ([\#9855](https://github.com/matrix-org/synapse/issues/9855))
+- Remove redundant `synapse.types.Collection` type definition. ([\#9856](https://github.com/matrix-org/synapse/issues/9856))
+- Handle recently added rate limits correctly when using `--no-rate-limit` with the demo scripts. ([\#9858](https://github.com/matrix-org/synapse/issues/9858))
+- Disable invite rate-limiting by default when running the unit tests. ([\#9871](https://github.com/matrix-org/synapse/issues/9871))
+- Pass a reactor into `SynapseSite` to make testing easier. ([\#9874](https://github.com/matrix-org/synapse/issues/9874))
+- Make `DomainSpecificString` an `attrs` class. ([\#9875](https://github.com/matrix-org/synapse/issues/9875))
+- Add type hints to `synapse.api.auth` and `synapse.api.auth_blocking` modules. ([\#9876](https://github.com/matrix-org/synapse/issues/9876))
+- Remove redundant `_PushHTTPChannel` test class. ([\#9878](https://github.com/matrix-org/synapse/issues/9878))
+- Remove backwards-compatibility code for Python versions < 3.6. ([\#9879](https://github.com/matrix-org/synapse/issues/9879))
+- Small performance improvement around handling new local presence updates. ([\#9887](https://github.com/matrix-org/synapse/issues/9887))
+
+
+Synapse 1.32.2 (2021-04-22)
+===========================
+
+This release includes a fix for a regression introduced in 1.32.0.
+
+Bugfixes
+--------
+
+- Fix a regression in Synapse 1.32.0 and 1.32.1 which caused `LoggingContext` errors in plugins. ([\#9857](https://github.com/matrix-org/synapse/issues/9857))
+
+
+Synapse 1.32.1 (2021-04-21)
+===========================
+
+This release fixes [a regression](https://github.com/matrix-org/synapse/issues/9853)
+in Synapse 1.32.0 that caused connected Prometheus instances to become unstable.
+
+However, as this release is still subject to the `LoggingContext` change in 1.32.0,
+it is recommended to remain on or downgrade to 1.31.0.
+
+Bugfixes
+--------
+
+- Fix a regression in Synapse 1.32.0 which caused Synapse to report large numbers of Prometheus time series, potentially overwhelming Prometheus instances. ([\#9854](https://github.com/matrix-org/synapse/issues/9854))
+
+
+Synapse 1.32.0 (2021-04-20)
+===========================
+
+**Note:** This release introduces [a regression](https://github.com/matrix-org/synapse/issues/9853)
+that can overwhelm connected Prometheus instances. This issue was not present in
+1.32.0rc1. If affected, it is recommended to downgrade to 1.31.0 in the meantime, and
+follow [these instructions](https://github.com/matrix-org/synapse/pull/9854#issuecomment-823472183)
+to clean up any excess writeahead logs.
+
+**Note:** This release also mistakenly included a change that may affected Synapse
+modules that import `synapse.logging.context.LoggingContext`, such as
+[synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider).
+This will be fixed in a later Synapse version.
+
+**Note:** This release requires Python 3.6+ and Postgres 9.6+ or SQLite 3.22+.
+
+This release removes the deprecated `GET /_synapse/admin/v1/users/<user_id>` admin API. Please use the [v2 API](https://github.com/matrix-org/synapse/blob/develop/docs/admin_api/user_admin_api.rst#query-user-account) instead, which has improved capabilities.
+
+This release requires Application Services to use type `m.login.application_service` when registering users via the `/_matrix/client/r0/register` endpoint to comply with the spec. Please ensure your Application Services are up to date.
+
+If you are using the `packages.matrix.org` Debian repository for Synapse packages,
+note that we have recently updated the expiry date on the gpg signing key. If you see an
+error similar to `The following signatures were invalid: EXPKEYSIG F473DD4473365DE1`, you
+will need to get a fresh copy of the keys. You can do so with:
+
+```sh
+sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+```
+
+Bugfixes
+--------
+
+- Fix the log lines of nested logging contexts. Broke in 1.32.0rc1. ([\#9829](https://github.com/matrix-org/synapse/issues/9829))
+
+
+Synapse 1.32.0rc1 (2021-04-13)
+==============================
+
+Features
+--------
+
+- Add a Synapse module for routing presence updates between users. ([\#9491](https://github.com/matrix-org/synapse/issues/9491))
+- Add an admin API to manage ratelimit for a specific user. ([\#9648](https://github.com/matrix-org/synapse/issues/9648))
+- Include request information in structured logging output. ([\#9654](https://github.com/matrix-org/synapse/issues/9654))
+- Add `order_by` to the admin API `GET /_synapse/admin/v2/users`. Contributed by @dklimpel. ([\#9691](https://github.com/matrix-org/synapse/issues/9691))
+- Replace the `room_invite_state_types` configuration setting with `room_prejoin_state`. ([\#9700](https://github.com/matrix-org/synapse/issues/9700))
+- Add experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership. ([\#9717](https://github.com/matrix-org/synapse/issues/9717), [\#9735](https://github.com/matrix-org/synapse/issues/9735))
+- Update experimental support for Spaces: include `m.room.create` in the room state sent with room-invites. ([\#9710](https://github.com/matrix-org/synapse/issues/9710))
+- Synapse now requires Python 3.6 or later. It also requires Postgres 9.6 or later or SQLite 3.22 or later. ([\#9766](https://github.com/matrix-org/synapse/issues/9766))
+
+
+Bugfixes
+--------
+
+- Prevent `synapse_forward_extremities` and `synapse_excess_extremity_events` Prometheus metrics from initially reporting zero-values after startup. ([\#8926](https://github.com/matrix-org/synapse/issues/8926))
+- Fix recently added ratelimits to correctly honour the application service `rate_limited` flag. ([\#9711](https://github.com/matrix-org/synapse/issues/9711))
+- Fix longstanding bug which caused `duplicate key value violates unique constraint "remote_media_cache_thumbnails_media_origin_media_id_thumbna_key"` errors. ([\#9725](https://github.com/matrix-org/synapse/issues/9725))
+- Fix bug where sharded federation senders could get stuck repeatedly querying the DB in a loop, using lots of CPU. ([\#9770](https://github.com/matrix-org/synapse/issues/9770))
+- Fix duplicate logging of exceptions thrown during federation transaction processing. ([\#9780](https://github.com/matrix-org/synapse/issues/9780))
+
+
+Updates to the Docker image
+---------------------------
+
+- Move opencontainers labels to the final Docker image such that users can inspect them. ([\#9765](https://github.com/matrix-org/synapse/issues/9765))
+
+
+Improved Documentation
+----------------------
+
+- Make the `allowed_local_3pids` regex example in the sample config stricter. ([\#9719](https://github.com/matrix-org/synapse/issues/9719))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove old admin API `GET /_synapse/admin/v1/users/<user_id>`. ([\#9401](https://github.com/matrix-org/synapse/issues/9401))
+- Make `/_matrix/client/r0/register` expect a type of `m.login.application_service` when an Application Service registers a user, to align with [the relevant spec](https://spec.matrix.org/unstable/application-service-api/#server-admin-style-permissions). ([\#9548](https://github.com/matrix-org/synapse/issues/9548))
+
+
+Internal Changes
+----------------
+
+- Replace deprecated `imp` module with successor `importlib`. Contributed by Cristina Muñoz. ([\#9718](https://github.com/matrix-org/synapse/issues/9718))
+- Experiment with GitHub Actions for CI. ([\#9661](https://github.com/matrix-org/synapse/issues/9661))
+- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9682](https://github.com/matrix-org/synapse/issues/9682))
+- Update `scripts-dev/complement.sh` to use a local checkout of Complement, allow running a subset of tests and have it use Synapse's Complement test blacklist. ([\#9685](https://github.com/matrix-org/synapse/issues/9685))
+- Improve Jaeger tracing for `to_device` messages. ([\#9686](https://github.com/matrix-org/synapse/issues/9686))
+- Add release helper script for automating part of the Synapse release process. ([\#9713](https://github.com/matrix-org/synapse/issues/9713))
+- Add type hints to expiring cache. ([\#9730](https://github.com/matrix-org/synapse/issues/9730))
+- Convert various testcases to `HomeserverTestCase`. ([\#9736](https://github.com/matrix-org/synapse/issues/9736))
+- Start linting mypy with `no_implicit_optional`. ([\#9742](https://github.com/matrix-org/synapse/issues/9742))
+- Add missing type hints to federation handler and server. ([\#9743](https://github.com/matrix-org/synapse/issues/9743))
+- Check that a `ConfigError` is raised, rather than simply `Exception`, when appropriate in homeserver config file generation tests. ([\#9753](https://github.com/matrix-org/synapse/issues/9753))
+- Fix incompatibility with `tox` 2.5. ([\#9769](https://github.com/matrix-org/synapse/issues/9769))
+- Enable Complement tests for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary API. ([\#9771](https://github.com/matrix-org/synapse/issues/9771))
+- Use mock from the standard library instead of a separate package. ([\#9772](https://github.com/matrix-org/synapse/issues/9772))
+- Update Black configuration to target Python 3.6. ([\#9781](https://github.com/matrix-org/synapse/issues/9781))
+- Add option to skip unit tests when building Debian packages. ([\#9793](https://github.com/matrix-org/synapse/issues/9793))
+
+
 Synapse 1.31.0 (2021-04-06)
 ===========================
 
@@ -200,7 +1197,7 @@ Internal Changes
 Synapse 1.29.0 (2021-03-08)
 ===========================
 
-Note that synapse now expects an `X-Forwarded-Proto` header when used with a reverse proxy. Please see [UPGRADE.rst](UPGRADE.rst#upgrading-to-v1290) for more details on this change.
+Note that synapse now expects an `X-Forwarded-Proto` header when used with a reverse proxy. Please see the [upgrade notes](docs/upgrade.md#upgrading-to-v1290) for more details on this change.
 
 
 No significant changes.
@@ -265,7 +1262,7 @@ Synapse 1.28.0 (2021-02-25)
 
 Note that this release drops support for ARMv7 in the official Docker images, due to repeated problems building for ARMv7 (and the associated maintenance burden this entails).
 
-This release also fixes the documentation included in v1.27.0 around the callback URI for SAML2 identity providers. If your server is configured to use single sign-on via a SAML2 IdP, you may need to make configuration changes. Please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes.
+This release also fixes the documentation included in v1.27.0 around the callback URI for SAML2 identity providers. If your server is configured to use single sign-on via a SAML2 IdP, you may need to make configuration changes. Please review the [upgrade notes](docs/upgrade.md) for more details on these changes.
 
 
 Internal Changes
@@ -364,9 +1361,9 @@ Synapse 1.27.0 (2021-02-16)
 
 Note that this release includes a change in Synapse to use Redis as a cache ─ as well as a pub/sub mechanism ─ if Redis support is enabled for workers. No action is needed by server administrators, and we do not expect resource usage of the Redis instance to change dramatically.
 
-This release also changes the callback URI for OpenID Connect (OIDC) and SAML2 identity providers. If your server is configured to use single sign-on via an OIDC/OAuth2 or SAML2 IdP, you may need to make configuration changes. Please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes.
+This release also changes the callback URI for OpenID Connect (OIDC) and SAML2 identity providers. If your server is configured to use single sign-on via an OIDC/OAuth2 or SAML2 IdP, you may need to make configuration changes. Please review the [upgrade notes](docs/upgrade.md) for more details on these changes.
 
-This release also changes escaping of variables in the HTML templates for SSO or email notifications. If you have customised these templates, please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes.
+This release also changes escaping of variables in the HTML templates for SSO or email notifications. If you have customised these templates, please review the [upgrade notes](docs/upgrade.md) for more details on these changes.
 
 
 Bugfixes
@@ -470,7 +1467,7 @@ Synapse 1.26.0 (2021-01-27)
 ===========================
 
 This release brings a new schema version for Synapse and rolling back to a previous
-version is not trivial. Please review [UPGRADE.rst](UPGRADE.rst) for more details
+version is not trivial. Please review the [upgrade notes](docs/upgrade.md) for more details
 on these changes and for general upgrade guidance.
 
 No significant changes since 1.26.0rc2.
@@ -497,7 +1494,7 @@ Synapse 1.26.0rc1 (2021-01-20)
 ==============================
 
 This release brings a new schema version for Synapse and rolling back to a previous
-version is not trivial. Please review [UPGRADE.rst](UPGRADE.rst) for more details
+version is not trivial. Please review the [upgrade notes](docs/upgrade.md) for more details
 on these changes and for general upgrade guidance.
 
 Features
@@ -600,7 +1597,10 @@ Crucially, this means __we will not produce .deb packages for Debian 9 (Stretch)
 
 The website https://endoflife.date/ has convenient summaries of the support schedules for projects like [Python](https://endoflife.date/python) and [PostgreSQL](https://endoflife.date/postgresql).
 
-If you are unable to upgrade your environment to a supported version of Python or Postgres, we encourage you to consider using the [Synapse Docker images](./INSTALL.md#docker-images-and-ansible-playbooks) instead.
+If you are unable to upgrade your environment to a supported version of Python or
+Postgres, we encourage you to consider using the
+[Synapse Docker images](https://matrix-org.github.io/synapse/latest/setup/installation.html#docker-images-and-ansible-playbooks)
+instead.
 
 ### Transition Period
 
@@ -743,11 +1743,11 @@ To upgrade Synapse along with the cryptography package:
 * Administrators using the [`matrix.org` Docker
   image](https://hub.docker.com/r/matrixdotorg/synapse/) or the [Debian/Ubuntu
   packages from
-  `matrix.org`](https://github.com/matrix-org/synapse/blob/master/INSTALL.md#matrixorg-packages)
+  `matrix.org`](https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages)
   should ensure that they have version 1.24.0 or 1.23.1 installed: these images include
   the updated packages.
 * Administrators who have [installed Synapse from
-  source](https://github.com/matrix-org/synapse/blob/master/INSTALL.md#installing-from-source)
+  source](https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source)
   should upgrade the cryptography package within their virtualenv by running:
   ```sh
   <path_to_virtualenv>/bin/pip install 'cryptography>=3.3'
@@ -789,11 +1789,11 @@ To upgrade Synapse along with the cryptography package:
 * Administrators using the [`matrix.org` Docker
   image](https://hub.docker.com/r/matrixdotorg/synapse/) or the [Debian/Ubuntu
   packages from
-  `matrix.org`](https://github.com/matrix-org/synapse/blob/master/INSTALL.md#matrixorg-packages)
+  `matrix.org`](https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages)
   should ensure that they have version 1.24.0 or 1.23.1 installed: these images include
   the updated packages.
 * Administrators who have [installed Synapse from
-  source](https://github.com/matrix-org/synapse/blob/master/INSTALL.md#installing-from-source)
+  source](https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source)
   should upgrade the cryptography package within their virtualenv by running:
   ```sh
   <path_to_virtualenv>/bin/pip install 'cryptography>=3.3'
@@ -903,7 +1903,7 @@ Internal Changes
 Synapse 1.23.0 (2020-11-18)
 ===========================
 
-This release changes the way structured logging is configured. See the [upgrade notes](UPGRADE.rst#upgrading-to-v1230) for details.
+This release changes the way structured logging is configured. See the [upgrade notes](docs/upgrade.md#upgrading-to-v1230) for details.
 
 **Note**: We are aware of a trivially exploitable denial of service vulnerability in versions of Synapse prior to 1.20.0. Complete details will be disclosed on Monday, November 23rd. If you have not upgraded recently, please do so.
 
@@ -1506,7 +2506,10 @@ No significant changes since 1.19.0rc1.
 Removal warning
 ---------------
 
-As outlined in the [previous release](https://github.com/matrix-org/synapse/releases/tag/v1.18.0), we are no longer publishing Docker images with the `-py3` tag suffix. On top of that, we have also removed the `latest-py3` tag. Please see [the announcement in the upgrade notes for 1.18.0](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#upgrading-to-v1180).
+As outlined in the [previous release](https://github.com/matrix-org/synapse/releases/tag/v1.18.0),
+we are no longer publishing Docker images with the `-py3` tag suffix. On top of that, we have also removed the
+`latest-py3` tag. Please see
+[the announcement in the upgrade notes for 1.18.0](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1180).
 
 
 Synapse 1.19.0rc1 (2020-08-13)
@@ -1537,7 +2540,7 @@ Bugfixes
 Updates to the Docker image
 ---------------------------
 
-- We no longer publish Docker images with the `-py3` tag suffix, as [announced in the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#upgrading-to-v1180). ([\#8056](https://github.com/matrix-org/synapse/issues/8056))
+- We no longer publish Docker images with the `-py3` tag suffix, as [announced in the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1180). ([\#8056](https://github.com/matrix-org/synapse/issues/8056))
 
 
 Improved Documentation
@@ -2095,7 +3098,7 @@ configurations of Synapse:
   to be incomplete or empty if Synapse was upgraded directly from v1.2.1 or
   earlier, to versions between v1.4.0 and v1.12.x.
 
-Please review [UPGRADE.rst](UPGRADE.rst) for more details on these changes
+Please review the [upgrade notes](docs/upgrade.md) for more details on these changes
 and for general upgrade guidance.
 
 
@@ -2196,7 +3199,7 @@ Bugfixes
 - Fix bad error handling that would cause Synapse to crash if it's provided with a YAML configuration file that's either empty or doesn't parse into a key-value map. ([\#7341](https://github.com/matrix-org/synapse/issues/7341))
 - Fix incorrect metrics reporting for `renew_attestations` background task. ([\#7344](https://github.com/matrix-org/synapse/issues/7344))
 - Prevent non-federating rooms from appearing in responses to federated `POST /publicRoom` requests when a filter was included. ([\#7367](https://github.com/matrix-org/synapse/issues/7367))
-- Fix a bug which would cause the room durectory to be incorrectly populated if Synapse was upgraded directly from v1.2.1 or earlier to v1.4.0 or later. Note that this fix does not apply retrospectively; see the [upgrade notes](UPGRADE.rst#upgrading-to-v1130) for more information. ([\#7387](https://github.com/matrix-org/synapse/issues/7387))
+- Fix a bug which would cause the room durectory to be incorrectly populated if Synapse was upgraded directly from v1.2.1 or earlier to v1.4.0 or later. Note that this fix does not apply retrospectively; see the [upgrade notes](docs/upgrade.md#upgrading-to-v1130) for more information. ([\#7387](https://github.com/matrix-org/synapse/issues/7387))
 - Fix bug in `EventContext.deserialize`. ([\#7393](https://github.com/matrix-org/synapse/issues/7393))
 
 
@@ -2346,7 +3349,7 @@ Synapse 1.12.0 includes a database update which is run as part of the upgrade,
 and which may take some time (several hours in the case of a large
 server). Synapse will not respond to HTTP requests while this update is taking
 place. For imformation on seeing if you are affected, and workaround if you
-are, see the [upgrade notes](UPGRADE.rst#upgrading-to-v1120).
+are, see the [upgrade notes](docs/upgrade.md#upgrading-to-v1120).
 
 Security advisory
 -----------------
@@ -2369,11 +3372,11 @@ installation remains secure.
 * Administrators using the [`matrix.org` Docker
   image](https://hub.docker.com/r/matrixdotorg/synapse/) or the [Debian/Ubuntu
   packages from
-  `matrix.org`](https://github.com/matrix-org/synapse/blob/master/INSTALL.md#matrixorg-packages)
+  `matrix.org`](https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages)
   should ensure that they have version 1.12.0 installed: these images include
   Twisted 20.3.0.
 * Administrators who have [installed Synapse from
-  source](https://github.com/matrix-org/synapse/blob/master/INSTALL.md#installing-from-source)
+  source](https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source)
   should upgrade Twisted within their virtualenv by running:
   ```sh
   <path_to_virtualenv>/bin/pip install 'Twisted>=20.3.0'
@@ -2899,7 +3902,7 @@ Bugfixes
 Synapse 1.7.0 (2019-12-13)
 ==========================
 
-This release changes the default settings so that only local authenticated users can query the server's room directory. See the [upgrade notes](UPGRADE.rst#upgrading-to-v170) for details.
+This release changes the default settings so that only local authenticated users can query the server's room directory. See the [upgrade notes](docs/upgrade.md#upgrading-to-v170) for details.
 
 Support for SQLite versions before 3.11 is now deprecated. A future release will refuse to start if used with an SQLite version before 3.11.
 
@@ -3263,7 +4266,7 @@ Synapse 1.4.0rc1 (2019-09-26)
 =============================
 
 Note that this release includes significant changes around 3pid
-verification. Administrators are reminded to review the [upgrade notes](UPGRADE.rst#upgrading-to-v140).
+verification. Administrators are reminded to review the [upgrade notes](docs/upgrade.md#upgrading-to-v140).
 
 Features
 --------
@@ -3639,7 +4642,7 @@ Synapse 1.1.0 (2019-07-04)
 ==========================
 
 As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4.
-See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details.
+See the [upgrade notes](docs/upgrade.md#upgrading-to-v110) for more details.
 
 This release also deprecates the use of environment variables to configure the
 docker image. See the [docker README](https://github.com/matrix-org/synapse/blob/release-v1.1.0/docker/README.md#legacy-dynamic-configuration-file-support)
@@ -3669,7 +4672,7 @@ Synapse 1.1.0rc1 (2019-07-02)
 =============================
 
 As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4.
-See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details.
+See the [upgrade notes](docs/upgrade.md#upgrading-to-v110) for more details.
 
 Features
 --------
@@ -4441,7 +5444,7 @@ run on Python versions 3.5 or 3.6 (as well as 2.7). Support for Python 3.7
 remains experimental.
 
 We recommend upgrading to Python 3, but make sure to read the [upgrade
-notes](UPGRADE.rst#upgrading-to-v0340) when doing so.
+notes](docs/upgrade.md#upgrading-to-v0340) when doing so.
 
 Features
 --------
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b6a70f7ffe..cd6c34df85 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -13,8 +13,9 @@ This document aims to get you started with contributing to this repo!
 - [7. Turn coffee and documentation into code and documentation!](#7-turn-coffee-and-documentation-into-code-and-documentation)
 - [8. Test, test, test!](#8-test-test-test)
   * [Run the linters.](#run-the-linters)
-  * [Run the unit tests.](#run-the-unit-tests)
-  * [Run the integration tests.](#run-the-integration-tests)
+  * [Run the unit tests.](#run-the-unit-tests-twisted-trial)
+  * [Run the integration tests (SyTest).](#run-the-integration-tests-sytest)
+  * [Run the integration tests (Complement).](#run-the-integration-tests-complement)
 - [9. Submit your patch.](#9-submit-your-patch)
   * [Changelog](#changelog)
     + [How do I know what to call the changelog file before I create the PR?](#how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr)
@@ -155,7 +156,7 @@ source ./env/bin/activate
 ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```
 
-## Run the unit tests.
+## Run the unit tests (Twisted trial).
 
 The unit tests run parts of Synapse, including your changes, to see if anything
 was broken. They are slower than the linters but will typically catch more errors.
@@ -173,13 +174,20 @@ source ./env/bin/activate
 trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
 ```
 
-If your tests fail, you may wish to look at the logs:
+If your tests fail, you may wish to look at the logs (the default log level is `ERROR`):
 
 ```sh
 less _trial_temp/test.log
 ```
 
-## Run the integration tests.
+To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`:
+
+```sh
+SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
+```
+
+
+## Run the integration tests ([Sytest](https://github.com/matrix-org/sytest)).
 
 The integration tests are a more comprehensive suite of tests. They
 run a full version of Synapse, including your changes, to check if
@@ -190,12 +198,49 @@ The following command will let you run the integration test with the most common
 configuration:
 
 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py37
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
 ```
 
 This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
 
 
+## Run the integration tests ([Complement](https://github.com/matrix-org/complement)).
+
+[Complement](https://github.com/matrix-org/complement) is a suite of black box tests that can be run on any homeserver implementation. It can also be thought of as end-to-end (e2e) tests.
+
+It's often nice to develop on Synapse and write Complement tests at the same time.
+Here is how to run your local Synapse checkout against your local Complement checkout.
+
+(checkout [`complement`](https://github.com/matrix-org/complement) alongside your `synapse` checkout)
+```sh
+COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh
+```
+
+To run a specific test file, you can pass the test name at the end of the command. The name passed comes from the naming structure in your Complement tests. If you're unsure of the name, you can do a full run and copy it from the test output:
+
+```sh
+COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory
+```
+
+To run a specific test, you can specify the whole name structure:
+
+```sh
+COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory/parallel/Backfilled_historical_events_resolve_with_proper_state_in_correct_order
+```
+
+
+### Access database for homeserver after Complement test runs.
+
+If you're curious what the database looks like after you run some tests, here are some steps to get you going in Synapse:
+
+ 1. In your Complement test comment out `defer deployment.Destroy(t)` and replace with `defer time.Sleep(2 * time.Hour)` to keep the homeserver running after the tests complete
+ 1. Start the Complement tests
+ 1. Find the name of the container, `docker ps -f name=complement_` (this will filter for just the Compelement related Docker containers)
+ 1. Access the container replacing the name with what you found in the previous step: `docker exec -it complement_1_hs_with_application_service.hs1_2 /bin/bash`
+ 1. Install sqlite (database driver), `apt-get update && apt-get install -y sqlite3`
+ 1. Then run `sqlite3` and open the database `.open /conf/homeserver.db` (this db path comes from the Synapse homeserver.yaml)
+
+
 # 9. Submit your patch.
 
 Once you're happy with your patch, it's time to prepare a Pull Request.
@@ -208,6 +253,7 @@ To prepare a Pull Request, please:
 4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
 5. add a [changelog entry](#changelog) and push it to your Pull Request;
 6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
+7. if you need to update your PR, please avoid rebasing and just add new commits to your branch.
 
 
 ## Changelog
@@ -385,7 +431,7 @@ By now, you know the drill!
 # Notes for maintainers on merging PRs etc
 
 There are some notes for those with commit access to the project on how we
-manage git [here](docs/dev/git.md).
+manage git [here](docs/development/git.md).
 
 # Conclusion
 
diff --git a/INSTALL.md b/INSTALL.md
index 7b40689234..f199b233b9 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -1,594 +1,7 @@
 # Installation Instructions
 
-There are 3 steps to follow under **Installation Instructions**.
+This document has moved to the
+[Synapse documentation website](https://matrix-org.github.io/synapse/latest/setup/installation.html).
+Please update your links.
 
-- [Installation Instructions](#installation-instructions)
-  - [Choosing your server name](#choosing-your-server-name)
-  - [Installing Synapse](#installing-synapse)
-    - [Installing from source](#installing-from-source)
-      - [Platform-specific prerequisites](#platform-specific-prerequisites)
-        - [Debian/Ubuntu/Raspbian](#debianubunturaspbian)
-        - [ArchLinux](#archlinux)
-        - [CentOS/Fedora](#centosfedora)
-        - [macOS](#macos)
-        - [OpenSUSE](#opensuse)
-        - [OpenBSD](#openbsd)
-        - [Windows](#windows)
-    - [Prebuilt packages](#prebuilt-packages)
-      - [Docker images and Ansible playbooks](#docker-images-and-ansible-playbooks)
-      - [Debian/Ubuntu](#debianubuntu)
-        - [Matrix.org packages](#matrixorg-packages)
-        - [Downstream Debian packages](#downstream-debian-packages)
-        - [Downstream Ubuntu packages](#downstream-ubuntu-packages)
-      - [Fedora](#fedora)
-      - [OpenSUSE](#opensuse-1)
-      - [SUSE Linux Enterprise Server](#suse-linux-enterprise-server)
-      - [ArchLinux](#archlinux-1)
-      - [Void Linux](#void-linux)
-      - [FreeBSD](#freebsd)
-      - [OpenBSD](#openbsd-1)
-      - [NixOS](#nixos)
-  - [Setting up Synapse](#setting-up-synapse)
-    - [Using PostgreSQL](#using-postgresql)
-    - [TLS certificates](#tls-certificates)
-    - [Client Well-Known URI](#client-well-known-uri)
-    - [Email](#email)
-    - [Registering a user](#registering-a-user)
-    - [Setting up a TURN server](#setting-up-a-turn-server)
-    - [URL previews](#url-previews)
-    - [Troubleshooting Installation](#troubleshooting-installation)
-
-
-## Choosing your server name
-
-It is important to choose the name for your server before you install Synapse,
-because it cannot be changed later.
-
-The server name determines the "domain" part of user-ids for users on your
-server: these will all be of the format `@user:my.domain.name`. It also
-determines how other matrix servers will reach yours for federation.
-
-For a test configuration, set this to the hostname of your server. For a more
-production-ready setup, you will probably want to specify your domain
-(`example.com`) rather than a matrix-specific hostname here (in the same way
-that your email address is probably `user@example.com` rather than
-`user@email.example.com`) - but doing so may require more advanced setup: see
-[Setting up Federation](docs/federate.md).
-
-## Installing Synapse
-
-### Installing from source
-
-(Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
-
-When installing from source please make sure that the [Platform-specific prerequisites](#platform-specific-prerequisites) are already installed.
-
-System requirements:
-
-- POSIX-compliant system (tested on Linux & OS X)
-- Python 3.5.2 or later, up to Python 3.9.
-- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
-
-
-To install the Synapse homeserver run:
-
-```sh
-mkdir -p ~/synapse
-virtualenv -p python3 ~/synapse/env
-source ~/synapse/env/bin/activate
-pip install --upgrade pip
-pip install --upgrade setuptools
-pip install matrix-synapse
-```
-
-This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
-and install it, along with the python libraries it uses, into a virtual environment
-under `~/synapse/env`.  Feel free to pick a different directory if you
-prefer.
-
-This Synapse installation can then be later upgraded by using pip again with the
-update flag:
-
-```sh
-source ~/synapse/env/bin/activate
-pip install -U matrix-synapse
-```
-
-Before you can start Synapse, you will need to generate a configuration
-file. To do this, run (in your virtualenv, as before):
-
-```sh
-cd ~/synapse
-python -m synapse.app.homeserver \
-    --server-name my.domain.name \
-    --config-path homeserver.yaml \
-    --generate-config \
-    --report-stats=[yes|no]
-```
-
-... substituting an appropriate value for `--server-name`.
-
-This command will generate you a config file that you can then customise, but it will
-also generate a set of keys for you. These keys will allow your homeserver to
-identify itself to other homeserver, so don't lose or delete them. It would be
-wise to back them up somewhere safe. (If, for whatever reason, you do need to
-change your homeserver's keys, you may find that other homeserver have the
-old key cached. If you update the signing key, you should change the name of the
-key in the `<server name>.signing.key` file (the second word) to something
-different. See the [spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys) for more information on key management).
-
-To actually run your new homeserver, pick a working directory for Synapse to
-run (e.g. `~/synapse`), and:
-
-```sh
-cd ~/synapse
-source env/bin/activate
-synctl start
-```
-
-#### Platform-specific prerequisites
-
-Synapse is written in Python but some of the libraries it uses are written in
-C. So before we can install Synapse itself we need a working C compiler and the
-header files for Python C extensions.
-
-##### Debian/Ubuntu/Raspbian
-
-Installing prerequisites on Ubuntu or Debian:
-
-```sh
-sudo apt install build-essential python3-dev libffi-dev \
-                     python3-pip python3-setuptools sqlite3 \
-                     libssl-dev virtualenv libjpeg-dev libxslt1-dev
-```
-
-##### ArchLinux
-
-Installing prerequisites on ArchLinux:
-
-```sh
-sudo pacman -S base-devel python python-pip \
-               python-setuptools python-virtualenv sqlite3
-```
-
-##### CentOS/Fedora
-
-Installing prerequisites on CentOS or Fedora Linux:
-
-```sh
-sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
-                 libwebp-devel libxml2-devel libxslt-devel libpq-devel \
-                 python3-virtualenv libffi-devel openssl-devel python3-devel
-sudo dnf groupinstall "Development Tools"
-```
-
-##### macOS
-
-Installing prerequisites on macOS:
-
-```sh
-xcode-select --install
-sudo easy_install pip
-sudo pip install virtualenv
-brew install pkg-config libffi
-```
-
-On macOS Catalina (10.15) you may need to explicitly install OpenSSL
-via brew and inform `pip` about it so that `psycopg2` builds:
-
-```sh
-brew install openssl@1.1
-export LDFLAGS="-L/usr/local/opt/openssl/lib"
-export CPPFLAGS="-I/usr/local/opt/openssl/include"
-```
-
-##### OpenSUSE
-
-Installing prerequisites on openSUSE:
-
-```sh
-sudo zypper in -t pattern devel_basis
-sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
-               python-devel libffi-devel libopenssl-devel libjpeg62-devel
-```
-
-##### OpenBSD
-
-A port of Synapse is available under `net/synapse`. The filesystem
-underlying the homeserver directory (defaults to `/var/synapse`) has to be
-mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
-and mounting it to `/var/synapse` should be taken into consideration.
-
-To be able to build Synapse's dependency on python the `WRKOBJDIR`
-(cf. `bsd.port.mk(5)`) for building python, too, needs to be on a filesystem
-mounted with `wxallowed` (cf. `mount(8)`).
-
-Creating a `WRKOBJDIR` for building python under `/usr/local` (which on a
-default OpenBSD installation is mounted with `wxallowed`):
-
-```sh
-doas mkdir /usr/local/pobj_wxallowed
-```
-
-Assuming `PORTS_PRIVSEP=Yes` (cf. `bsd.port.mk(5)`) and `SUDO=doas` are
-configured in `/etc/mk.conf`:
-
-```sh
-doas chown _pbuild:_pbuild /usr/local/pobj_wxallowed
-```
-
-Setting the `WRKOBJDIR` for building python:
-
-```sh
-echo WRKOBJDIR_lang/python/3.7=/usr/local/pobj_wxallowed  \\nWRKOBJDIR_lang/python/2.7=/usr/local/pobj_wxallowed >> /etc/mk.conf
-```
-
-Building Synapse:
-
-```sh
-cd /usr/ports/net/synapse
-make install
-```
-
-##### Windows
-
-If you wish to run or develop Synapse on Windows, the Windows Subsystem For
-Linux provides a Linux environment on Windows 10 which is capable of using the
-Debian, Fedora, or source installation methods. More information about WSL can
-be found at <https://docs.microsoft.com/en-us/windows/wsl/install-win10> for
-Windows 10 and <https://docs.microsoft.com/en-us/windows/wsl/install-on-server>
-for Windows Server.
-
-### Prebuilt packages
-
-As an alternative to installing from source, prebuilt packages are available
-for a number of platforms.
-
-#### Docker images and Ansible playbooks
-
-There is an official synapse image available at
-<https://hub.docker.com/r/matrixdotorg/synapse> which can be used with
-the docker-compose file available at [contrib/docker](contrib/docker). Further
-information on this including configuration options is available in the README
-on hub.docker.com.
-
-Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
-Dockerfile to automate a synapse server in a single Docker image, at
-<https://hub.docker.com/r/avhost/docker-matrix/tags/>
-
-Slavi Pantaleev has created an Ansible playbook,
-which installs the offical Docker image of Matrix Synapse
-along with many other Matrix-related services (Postgres database, Element, coturn,
-ma1sd, SSL support, etc.).
-For more details, see
-<https://github.com/spantaleev/matrix-docker-ansible-deploy>
-
-#### Debian/Ubuntu
-
-##### Matrix.org packages
-
-Matrix.org provides Debian/Ubuntu packages of the latest stable version of
-Synapse via <https://packages.matrix.org/debian/>. They are available for Debian
-9 (Stretch), Ubuntu 16.04 (Xenial), and later. To use them:
-
-```sh
-sudo apt install -y lsb-release wget apt-transport-https
-sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
-echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
-    sudo tee /etc/apt/sources.list.d/matrix-org.list
-sudo apt update
-sudo apt install matrix-synapse-py3
-```
-
-**Note**: if you followed a previous version of these instructions which
-recommended using `apt-key add` to add an old key from
-`https://matrix.org/packages/debian/`, you should note that this key has been
-revoked. You should remove the old key with `sudo apt-key remove
-C35EB17E1EAE708E6603A9B3AD0592FE47F0DF61`, and follow the above instructions to
-update your configuration.
-
-The fingerprint of the repository signing key (as shown by `gpg
-/usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
-`AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
-
-##### Downstream Debian packages
-
-We do not recommend using the packages from the default Debian `buster`
-repository at this time, as they are old and suffer from known security
-vulnerabilities. You can install the latest version of Synapse from
-[our repository](#matrixorg-packages) or from `buster-backports`. Please
-see the [Debian documentation](https://backports.debian.org/Instructions/)
-for information on how to use backports.
-
-If you are using Debian `sid` or testing, Synapse is available in the default
-repositories and it should be possible to install it simply with:
-
-```sh
-sudo apt install matrix-synapse
-```
-
-##### Downstream Ubuntu packages
-
-We do not recommend using the packages in the default Ubuntu repository
-at this time, as they are old and suffer from known security vulnerabilities.
-The latest version of Synapse can be installed from [our repository](#matrixorg-packages).
-
-#### Fedora
-
-Synapse is in the Fedora repositories as `matrix-synapse`:
-
-```sh
-sudo dnf install matrix-synapse
-```
-
-Oleg Girko provides Fedora RPMs at
-<https://obs.infoserver.lv/project/monitor/matrix-synapse>
-
-#### OpenSUSE
-
-Synapse is in the OpenSUSE repositories as `matrix-synapse`:
-
-```sh
-sudo zypper install matrix-synapse
-```
-
-#### SUSE Linux Enterprise Server
-
-Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
-<https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/>
-
-#### ArchLinux
-
-The quickest way to get up and running with ArchLinux is probably with the community package
-<https://www.archlinux.org/packages/community/any/matrix-synapse/>, which should pull in most of
-the necessary dependencies.
-
-pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
-
-```sh
-sudo pip install --upgrade pip
-```
-
-If you encounter an error with lib bcrypt causing an Wrong ELF Class:
-ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
-compile it under the right architecture. (This should not be needed if
-installing under virtualenv):
-
-```sh
-sudo pip uninstall py-bcrypt
-sudo pip install py-bcrypt
-```
-
-#### Void Linux
-
-Synapse can be found in the void repositories as 'synapse':
-
-```sh
-xbps-install -Su
-xbps-install -S synapse
-```
-
-#### FreeBSD
-
-Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
-
-- Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
-- Packages: `pkg install py37-matrix-synapse`
-
-#### OpenBSD
-
-As of OpenBSD 6.7 Synapse is available as a pre-compiled binary. The filesystem
-underlying the homeserver directory (defaults to `/var/synapse`) has to be
-mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
-and mounting it to `/var/synapse` should be taken into consideration.
-
-Installing Synapse:
-
-```sh
-doas pkg_add synapse
-```
-
-#### NixOS
-
-Robin Lambertz has packaged Synapse for NixOS at:
-<https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix>
-
-## Setting up Synapse
-
-Once you have installed synapse as above, you will need to configure it.
-
-### Using PostgreSQL
-
-By default Synapse uses [SQLite](https://sqlite.org/) and in doing so trades performance for convenience.
-SQLite is only recommended in Synapse for testing purposes or for servers with
-very light workloads.
-
-Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org). Advantages include:
-
-- significant performance improvements due to the superior threading and
-  caching model, smarter query optimiser
-- allowing the DB to be run on separate hardware
-
-For information on how to install and use PostgreSQL in Synapse, please see
-[docs/postgres.md](docs/postgres.md)
-
-### TLS certificates
-
-The default configuration exposes a single HTTP port on the local
-interface: `http://localhost:8008`. It is suitable for local testing,
-but for any practical use, you will need Synapse's APIs to be served
-over HTTPS.
-
-The recommended way to do so is to set up a reverse proxy on port
-`8448`. You can find documentation on doing so in
-[docs/reverse_proxy.md](docs/reverse_proxy.md).
-
-Alternatively, you can configure Synapse to expose an HTTPS port. To do
-so, you will need to edit `homeserver.yaml`, as follows:
-
-- First, under the `listeners` section, uncomment the configuration for the
-  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
-  each line). The relevant lines are like this:
-
-```yaml
-  - port: 8448
-    type: http
-    tls: true
-    resources:
-      - names: [client, federation]
-  ```
-
-- You will also need to uncomment the `tls_certificate_path` and
-  `tls_private_key_path` lines under the `TLS` section. You will need to manage
-  provisioning of these certificates yourself — Synapse had built-in ACME
-  support, but the ACMEv1 protocol Synapse implements is deprecated, not
-  allowed by LetsEncrypt for new sites, and will break for existing sites in
-  late 2020. See [ACME.md](docs/ACME.md).
-
-  If you are using your own certificate, be sure to use a `.pem` file that
-  includes the full certificate chain including any intermediate certificates
-  (for instance, if using certbot, use `fullchain.pem` as your certificate, not
-  `cert.pem`).
-
-For a more detailed guide to configuring your server for federation, see
-[federate.md](docs/federate.md).
-
-### Client Well-Known URI
-
-Setting up the client Well-Known URI is optional but if you set it up, it will
-allow users to enter their full username (e.g. `@user:<server_name>`) into clients
-which support well-known lookup to automatically configure the homeserver and
-identity server URLs. This is useful so that users don't have to memorize or think
-about the actual homeserver URL you are using.
-
-The URL `https://<server_name>/.well-known/matrix/client` should return JSON in
-the following format.
-
-```json
-{
-  "m.homeserver": {
-    "base_url": "https://<matrix.example.com>"
-  }
-}
-```
-
-It can optionally contain identity server information as well.
-
-```json
-{
-  "m.homeserver": {
-    "base_url": "https://<matrix.example.com>"
-  },
-  "m.identity_server": {
-    "base_url": "https://<identity.example.com>"
-  }
-}
-```
-
-To work in browser based clients, the file must be served with the appropriate
-Cross-Origin Resource Sharing (CORS) headers. A recommended value would be
-`Access-Control-Allow-Origin: *` which would allow all browser based clients to
-view it.
-
-In nginx this would be something like:
-
-```nginx
-location /.well-known/matrix/client {
-    return 200 '{"m.homeserver": {"base_url": "https://<matrix.example.com>"}}';
-    default_type application/json;
-    add_header Access-Control-Allow-Origin *;
-}
-```
-
-You should also ensure the `public_baseurl` option in `homeserver.yaml` is set
-correctly. `public_baseurl` should be set to the URL that clients will use to
-connect to your server. This is the same URL you put for the `m.homeserver`
-`base_url` above.
-
-```yaml
-public_baseurl: "https://<matrix.example.com>"
-```
-
-### Email
-
-It is desirable for Synapse to have the capability to send email. This allows
-Synapse to send password reset emails, send verifications when an email address
-is added to a user's account, and send email notifications to users when they
-receive new messages.
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed `email`, and be sure to have at least the `smtp_host`, `smtp_port`
-and `notif_from` fields filled out.  You may also need to set `smtp_user`,
-`smtp_pass`, and `require_transport_security`.
-
-If email is not configured, password reset, registration and notifications via
-email will be disabled.
-
-### Registering a user
-
-The easiest way to create a new user is to do so from a client like [Element](https://element.io/).
-
-Alternatively, you can do so from the command line. This can be done as follows:
-
- 1. If synapse was installed via pip, activate the virtualenv as follows (if Synapse was
-    installed via a prebuilt package, `register_new_matrix_user` should already be
-    on the search path):
-    ```sh
-    cd ~/synapse
-    source env/bin/activate
-    synctl start # if not already running
-    ```
- 2. Run the following command:
-    ```sh
-    register_new_matrix_user -c homeserver.yaml http://localhost:8008
-    ```
-
-This will prompt you to add details for the new user, and will then connect to
-the running Synapse to create the new user. For example:
-```
-New user localpart: erikj
-Password:
-Confirm password:
-Make admin [no]:
-Success!
-```
-
-This process uses a setting `registration_shared_secret` in
-`homeserver.yaml`, which is shared between Synapse itself and the
-`register_new_matrix_user` script. It doesn't matter what it is (a random
-value is generated by `--generate-config`), but it should be kept secret, as
-anyone with knowledge of it can register users, including admin accounts,
-on your server even if `enable_registration` is `false`.
-
-### Setting up a TURN server
-
-For reliable VoIP calls to be routed via this homeserver, you MUST configure
-a TURN server. See [docs/turn-howto.md](docs/turn-howto.md) for details.
-
-### URL previews
-
-Synapse includes support for previewing URLs, which is disabled by default.  To
-turn it on you must enable the `url_preview_enabled: True` config parameter
-and explicitly specify the IP ranges that Synapse is not allowed to spider for
-previewing in the `url_preview_ip_range_blacklist` configuration parameter.
-This is critical from a security perspective to stop arbitrary Matrix users
-spidering 'internal' URLs on your network. At the very least we recommend that
-your loopback and RFC1918 IP addresses are blacklisted.
-
-This also requires the optional `lxml` python dependency to be  installed. This
-in turn requires the `libxml2` library to be available - on  Debian/Ubuntu this
-means `apt-get install libxml2-dev`, or equivalent for your OS.
-
-### Troubleshooting Installation
-
-`pip` seems to leak *lots* of memory during installation. For instance, a Linux
-host with 512MB of RAM may run out of memory whilst installing Twisted. If this
-happens, you will have to individually install the dependencies which are
-failing, e.g.:
-
-```sh
-pip install twisted
-```
-
-If you have any other problems, feel free to ask in
-[#synapse:matrix.org](https://matrix.to/#/#synapse:matrix.org).
+The markdown source is available in [docs/setup/installation.md](docs/setup/installation.md).
diff --git a/MANIFEST.in b/MANIFEST.in
index 09bef29705..a615c2a083 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -41,12 +41,13 @@ exclude mypy.ini
 exclude sytest-blacklist
 exclude test_postgresql.sh
 
+include book.toml
 include pyproject.toml
 recursive-include changelog.d *
 
-prune .buildkite
 prune .circleci
 prune .github
+prune .ci
 prune contrib
 prune debian
 prune demo/etc
diff --git a/README.rst b/README.rst
index 655a2bf3be..0ae05616e7 100644
--- a/README.rst
+++ b/README.rst
@@ -25,7 +25,7 @@ The overall architecture is::
 
 ``#matrix:matrix.org`` is the official support room for Matrix, and can be
 accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html or
-via IRC bridge at irc://irc.freenode.net/matrix.
+via IRC bridge at irc://irc.libera.chat/matrix.
 
 Synapse is currently in rapid development, but as of version 0.5 we believe it
 is sufficiently stable to be run as an internet-facing service for real usage!
@@ -94,7 +94,8 @@ Synapse Installation
 
 .. _federation:
 
-* For details on how to install synapse, see `<INSTALL.md>`_.
+* For details on how to install synapse, see
+  `Installation Instructions <https://matrix-org.github.io/synapse/latest/setup/installation.html>`_.
 * For specific details on how to configure Synapse for federation see `docs/federate.md <docs/federate.md>`_
 
 
@@ -106,7 +107,8 @@ from a web client.
 
 Unless you are running a test instance of Synapse on your local machine, in
 general, you will need to enable TLS support before you can successfully
-connect from a client: see `<INSTALL.md#tls-certificates>`_.
+connect from a client: see
+`TLS certificates <https://matrix-org.github.io/synapse/latest/setup/installation.html#tls-certificates>`_.
 
 An easy way to get started is to login or register via Element at
 https://app.element.io/#/login or https://app.element.io/#/register respectively.
@@ -142,38 +144,55 @@ the form of::
 As when logging in, you will need to specify a "Custom server".  Specify your
 desired ``localpart`` in the 'User name' box.
 
-ACME setup
-==========
+Security note
+=============
 
-For details on having Synapse manage your federation TLS certificates
-automatically, please see `<docs/ACME.md>`_.
+Matrix serves raw, user-supplied data in some APIs -- specifically the `content
+repository endpoints`_.
 
+.. _content repository endpoints: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid
 
-Security Note
-=============
+Whilst we make a reasonable effort to mitigate against XSS attacks (for
+instance, by using `CSP`_), a Matrix homeserver should not be hosted on a
+domain hosting other web applications. This especially applies to sharing
+the domain with Matrix web clients and other sensitive applications like
+webmail. See
+https://developer.github.com/changes/2014-04-25-user-content-security for more
+information.
+
+.. _CSP: https://github.com/matrix-org/synapse/pull/1021
+
+Ideally, the homeserver should not simply be on a different subdomain, but on
+a completely different `registered domain`_ (also known as top-level site or
+eTLD+1). This is because `some attacks`_ are still possible as long as the two
+applications share the same registered domain.
 
-Matrix serves raw user generated data in some APIs - specifically the `content
-repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
+.. _registered domain: https://tools.ietf.org/html/draft-ietf-httpbis-rfc6265bis-03#section-2.3
 
-Whilst we have tried to mitigate against possible XSS attacks (e.g.
-https://github.com/matrix-org/synapse/pull/1021) we recommend running
-matrix homeservers on a dedicated domain name, to limit any malicious user generated
-content served to web browsers a matrix API from being able to attack webapps hosted
-on the same domain.  This is particularly true of sharing a matrix webclient and
-server on the same domain.
+.. _some attacks: https://en.wikipedia.org/wiki/Session_fixation#Attacks_using_cross-subdomain_cookie
 
-See https://github.com/vector-im/riot-web/issues/1977 and
-https://developer.github.com/changes/2014-04-25-user-content-security for more details.
+To illustrate this with an example, if your Element Web or other sensitive web
+application is hosted on ``A.example1.com``, you should ideally host Synapse on
+``example2.com``. Some amount of protection is offered by hosting on
+``B.example1.com`` instead, so this is also acceptable in some scenarios.
+However, you should *not* host your Synapse on ``A.example1.com``.
+
+Note that all of the above refers exclusively to the domain used in Synapse's
+``public_baseurl`` setting. In particular, it has no bearing on the domain
+mentioned in MXIDs hosted on that server.
+
+Following this advice ensures that even if an XSS is found in Synapse, the
+impact to other applications will be minimal.
 
 
 Upgrading an existing Synapse
 =============================
 
-The instructions for upgrading synapse are in `UPGRADE.rst`_.
+The instructions for upgrading synapse are in `the upgrade notes`_.
 Please check these instructions as upgrading may require extra steps for some
 versions of synapse.
 
-.. _UPGRADE.rst: UPGRADE.rst
+.. _the upgrade notes: https://matrix-org.github.io/synapse/develop/upgrade.html
 
 .. _reverse-proxy:
 
@@ -248,7 +267,7 @@ Join our developer community on Matrix: `#synapse-dev:matrix.org <https://matrix
 
 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see
-`Installing from source <INSTALL.md#installing-from-source>`_.
+`Installing from source <https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source>`_.
 
 To check out a synapse for development, clone the git repo into a working
 directory of your choice::
@@ -269,18 +288,6 @@ try installing the failing modules individually::
 
     pip install -e "module-name"
 
-Once this is done, you may wish to run Synapse's unit tests to
-check that everything is installed correctly::
-
-    python -m twisted.trial tests
-
-This should end with a 'PASSED' result (note that exact numbers will
-differ)::
-
-    Ran 1337 tests in 716.064s
-
-    PASSED (skips=15, successes=1322)
-
 We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`
 
     ./demo/start.sh
@@ -300,6 +307,23 @@ If you just want to start a single instance of the app and run it directly::
     python -m synapse.app.homeserver --config-path homeserver.yaml
 
 
+Running the unit tests
+======================
+
+After getting up and running, you may wish to run Synapse's unit tests to
+check that everything is installed correctly::
+
+    trial tests
+
+This should end with a 'PASSED' result (note that exact numbers will
+differ)::
+
+    Ran 1337 tests in 716.064s
+
+    PASSED (skips=15, successes=1322)
+
+For more tips on running the unit tests, like running a specific test or
+to see the logging output, see the `CONTRIBUTING doc <CONTRIBUTING.md#run-the-unit-tests>`_.
 
 
 Running the Integration Tests
@@ -311,8 +335,8 @@ access the API as a Matrix client would. It is able to run Synapse directly from
 the source tree, so installation of the server is not required.
 
 Testing with SyTest is recommended for verifying that changes related to the
-Client-Server API are functioning correctly. See the `installation instructions
-<https://github.com/matrix-org/sytest#installing>`_ for details.
+Client-Server API are functioning correctly. See the `SyTest installation
+instructions <https://github.com/matrix-org/sytest#installing>`_ for details.
 
 
 Platform dependencies
@@ -393,7 +417,12 @@ massive excess of outgoing federation requests (see `discussion
 indicate that your server is also issuing far more outgoing federation
 requests than can be accounted for by your users' activity, this is a
 likely cause. The misbehavior can be worked around by setting
-``use_presence: false`` in the Synapse config file.
+the following in the Synapse config file:
+
+.. code-block:: yaml
+
+   presence:
+       enabled: false
 
 People can't accept room invitations from me
 --------------------------------------------
diff --git a/UPGRADE.rst b/UPGRADE.rst
index ba488e1041..6c7f9cb18e 100644
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -1,1232 +1,7 @@
 Upgrading Synapse
 =================
 
-Before upgrading check if any special steps are required to upgrade from the
-version you currently have installed to the current version of Synapse. The extra
-instructions that may be required are listed later in this document.
+This document has moved to the `Synapse documentation website <https://matrix-org.github.io/synapse/latest/upgrade>`_.
+Please update your links.
 
-* Check that your versions of Python and PostgreSQL are still supported.
-
-  Synapse follows upstream lifecycles for `Python`_ and `PostgreSQL`_, and
-  removes support for versions which are no longer maintained.
-
-  The website https://endoflife.date also offers convenient summaries.
-
-  .. _Python: https://devguide.python.org/devcycle/#end-of-life-branches
-  .. _PostgreSQL: https://www.postgresql.org/support/versioning/
-
-* If Synapse was installed using `prebuilt packages
-  <INSTALL.md#prebuilt-packages>`_, you will need to follow the normal process
-  for upgrading those packages.
-
-* If Synapse was installed from source, then:
-
-  1. Activate the virtualenv before upgrading. For example, if Synapse is
-     installed in a virtualenv in ``~/synapse/env`` then run:
-
-     .. code:: bash
-
-       source ~/synapse/env/bin/activate
-
-  2. If Synapse was installed using pip then upgrade to the latest version by
-     running:
-
-     .. code:: bash
-
-       pip install --upgrade matrix-synapse
-
-     If Synapse was installed using git then upgrade to the latest version by
-     running:
-
-     .. code:: bash
-
-       git pull
-       pip install --upgrade .
-
-  3. Restart Synapse:
-
-     .. code:: bash
-
-       ./synctl restart
-
-To check whether your update was successful, you can check the running server
-version with:
-
-.. code:: bash
-
-    # you may need to replace 'localhost:8008' if synapse is not configured
-    # to listen on port 8008.
-
-    curl http://localhost:8008/_synapse/admin/v1/server_version
-
-Rolling back to older versions
-------------------------------
-
-Rolling back to previous releases can be difficult, due to database schema
-changes between releases. Where we have been able to test the rollback process,
-this will be noted below.
-
-In general, you will need to undo any changes made during the upgrade process,
-for example:
-
-* pip:
-
-  .. code:: bash
-
-     source env/bin/activate
-     # replace `1.3.0` accordingly:
-     pip install matrix-synapse==1.3.0
-
-* Debian:
-
-  .. code:: bash
-
-     # replace `1.3.0` and `stretch` accordingly:
-     wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
-     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
-
-Upgrading to v1.29.0
-====================
-
-Requirement for X-Forwarded-Proto header
-----------------------------------------
-
-When using Synapse with a reverse proxy (in particular, when using the
-`x_forwarded` option on an HTTP listener), Synapse now expects to receive an
-`X-Forwarded-Proto` header on incoming HTTP requests. If it is not set, Synapse
-will log a warning on each received request.
-
-To avoid the warning, administrators using a reverse proxy should ensure that
-the reverse proxy sets `X-Forwarded-Proto` header to `https` or `http` to
-indicate the protocol used by the client.
-
-Synapse also requires the `Host` header to be preserved.
-
-See the `reverse proxy documentation <docs/reverse_proxy.md>`_, where the
-example configurations have been updated to show how to set these headers.
-
-(Users of `Caddy <https://caddyserver.com/>`_ are unaffected, since we believe it
-sets `X-Forwarded-Proto` by default.)
-
-Upgrading to v1.27.0
-====================
-
-Changes to callback URI for OAuth2 / OpenID Connect and SAML2
--------------------------------------------------------------
-
-This version changes the URI used for callbacks from OAuth2 and SAML2 identity providers:
-
-* If your server is configured for single sign-on via an OpenID Connect or OAuth2 identity
-  provider, you will need to add ``[synapse public baseurl]/_synapse/client/oidc/callback``
-  to the list of permitted "redirect URIs" at the identity provider.
-
-  See `docs/openid.md <docs/openid.md>`_ for more information on setting up OpenID
-  Connect.
-
-* If your server is configured for single sign-on via a SAML2 identity provider, you will
-  need to add ``[synapse public baseurl]/_synapse/client/saml2/authn_response`` as a permitted
-  "ACS location" (also known as "allowed callback URLs") at the identity provider.
-
-  The "Issuer" in the "AuthnRequest" to the SAML2 identity provider is also updated to
-  ``[synapse public baseurl]/_synapse/client/saml2/metadata.xml``. If your SAML2 identity
-  provider uses this property to validate or otherwise identify Synapse, its configuration
-  will need to be updated to use the new URL. Alternatively you could create a new, separate
-  "EntityDescriptor" in your SAML2 identity provider with the new URLs and leave the URLs in
-  the existing "EntityDescriptor" as they were.
-
-Changes to HTML templates
--------------------------
-
-The HTML templates for SSO and email notifications now have `Jinja2's autoescape <https://jinja.palletsprojects.com/en/2.11.x/api/#autoescaping>`_
-enabled for files ending in ``.html``, ``.htm``, and ``.xml``. If you have customised
-these templates and see issues when viewing them you might need to update them.
-It is expected that most configurations will need no changes.
-
-If you have customised the templates *names* for these templates, it is recommended
-to verify they end in ``.html`` to ensure autoescape is enabled.
-
-The above applies to the following templates:
-
-* ``add_threepid.html``
-* ``add_threepid_failure.html``
-* ``add_threepid_success.html``
-* ``notice_expiry.html``
-* ``notice_expiry.html``
-* ``notif_mail.html`` (which, by default, includes ``room.html`` and ``notif.html``)
-* ``password_reset.html``
-* ``password_reset_confirmation.html``
-* ``password_reset_failure.html``
-* ``password_reset_success.html``
-* ``registration.html``
-* ``registration_failure.html``
-* ``registration_success.html``
-* ``sso_account_deactivated.html``
-* ``sso_auth_bad_user.html``
-* ``sso_auth_confirm.html``
-* ``sso_auth_success.html``
-* ``sso_error.html``
-* ``sso_login_idp_picker.html``
-* ``sso_redirect_confirm.html``
-
-Upgrading to v1.26.0
-====================
-
-Rolling back to v1.25.0 after a failed upgrade
-----------------------------------------------
-
-v1.26.0 includes a lot of large changes. If something problematic occurs, you
-may want to roll-back to a previous version of Synapse. Because v1.26.0 also
-includes a new database schema version, reverting that version is also required
-alongside the generic rollback instructions mentioned above. In short, to roll
-back to v1.25.0 you need to:
-
-1. Stop the server
-2. Decrease the schema version in the database:
-
-   .. code:: sql
-
-      UPDATE schema_version SET version = 58;
-
-3. Delete the ignored users & chain cover data:
-
-   .. code:: sql
-
-      DROP TABLE IF EXISTS ignored_users;
-      UPDATE rooms SET has_auth_chain_index = false;
-
-   For PostgreSQL run:
-
-   .. code:: sql
-
-      TRUNCATE event_auth_chain_links;
-      TRUNCATE event_auth_chains;
-
-   For SQLite run:
-
-   .. code:: sql
-
-      DELETE FROM event_auth_chain_links;
-      DELETE FROM event_auth_chains;
-
-4. Mark the deltas as not run (so they will re-run on upgrade).
-
-   .. code:: sql
-
-      DELETE FROM applied_schema_deltas WHERE version = 59 AND file = "59/01ignored_user.py";
-      DELETE FROM applied_schema_deltas WHERE version = 59 AND file = "59/06chain_cover_index.sql";
-
-5. Downgrade Synapse by following the instructions for your installation method
-   in the "Rolling back to older versions" section above.
-
-Upgrading to v1.25.0
-====================
-
-Last release supporting Python 3.5
-----------------------------------
-
-This is the last release of Synapse which guarantees support with Python 3.5,
-which passed its upstream End of Life date several months ago.
-
-We will attempt to maintain support through March 2021, but without guarantees.
-
-In the future, Synapse will follow upstream schedules for ending support of
-older versions of Python and PostgreSQL. Please upgrade to at least Python 3.6
-and PostgreSQL 9.6 as soon as possible.
-
-Blacklisting IP ranges
-----------------------
-
-Synapse v1.25.0 includes new settings, ``ip_range_blacklist`` and
-``ip_range_whitelist``, for controlling outgoing requests from Synapse for federation,
-identity servers, push, and for checking key validity for third-party invite events.
-The previous setting, ``federation_ip_range_blacklist``, is deprecated. The new
-``ip_range_blacklist`` defaults to private IP ranges if it is not defined.
-
-If you have never customised ``federation_ip_range_blacklist`` it is recommended
-that you remove that setting.
-
-If you have customised ``federation_ip_range_blacklist`` you should update the
-setting name to ``ip_range_blacklist``.
-
-If you have a custom push server that is reached via private IP space you may
-need to customise ``ip_range_blacklist`` or ``ip_range_whitelist``.
-
-Upgrading to v1.24.0
-====================
-
-Custom OpenID Connect mapping provider breaking change
-------------------------------------------------------
-
-This release allows the OpenID Connect mapping provider to perform normalisation
-of the localpart of the Matrix ID. This allows for the mapping provider to
-specify different algorithms, instead of the [default way](https://matrix.org/docs/spec/appendices#mapping-from-other-character-sets).
-
-If your Synapse configuration uses a custom mapping provider
-(`oidc_config.user_mapping_provider.module` is specified and not equal to
-`synapse.handlers.oidc_handler.JinjaOidcMappingProvider`) then you *must* ensure
-that `map_user_attributes` of the mapping provider performs some normalisation
-of the `localpart` returned. To match previous behaviour you can use the
-`map_username_to_mxid_localpart` function provided by Synapse. An example is
-shown below:
-
-.. code-block:: python
-
-  from synapse.types import map_username_to_mxid_localpart
-
-  class MyMappingProvider:
-      def map_user_attributes(self, userinfo, token):
-          # ... your custom logic ...
-          sso_user_id = ...
-          localpart = map_username_to_mxid_localpart(sso_user_id)
-
-          return {"localpart": localpart}
-
-Removal historical Synapse Admin API
-------------------------------------
-
-Historically, the Synapse Admin API has been accessible under:
-
-* ``/_matrix/client/api/v1/admin``
-* ``/_matrix/client/unstable/admin``
-* ``/_matrix/client/r0/admin``
-* ``/_synapse/admin/v1``
-
-The endpoints with ``/_matrix/client/*`` prefixes have been removed as of v1.24.0.
-The Admin API is now only accessible under:
-
-* ``/_synapse/admin/v1``
-
-The only exception is the `/admin/whois` endpoint, which is
-`also available via the client-server API <https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid>`_.
-
-The deprecation of the old endpoints was announced with Synapse 1.20.0 (released
-on 2020-09-22) and makes it easier for homeserver admins to lock down external
-access to the Admin API endpoints.
-
-Upgrading to v1.23.0
-====================
-
-Structured logging configuration breaking changes
--------------------------------------------------
-
-This release deprecates use of the ``structured: true`` logging configuration for
-structured logging. If your logging configuration contains ``structured: true``
-then it should be modified based on the `structured logging documentation
-<https://github.com/matrix-org/synapse/blob/master/docs/structured_logging.md>`_.
-
-The ``structured`` and ``drains`` logging options are now deprecated and should
-be replaced by standard logging configuration of ``handlers`` and ``formatters``.
-
-A future will release of Synapse will make using ``structured: true`` an error.
-
-Upgrading to v1.22.0
-====================
-
-ThirdPartyEventRules breaking changes
--------------------------------------
-
-This release introduces a backwards-incompatible change to modules making use of
-``ThirdPartyEventRules`` in Synapse. If you make use of a module defined under the
-``third_party_event_rules`` config option, please make sure it is updated to handle
-the below change:
-
-The ``http_client`` argument is no longer passed to modules as they are initialised. Instead,
-modules are expected to make use of the ``http_client`` property on the ``ModuleApi`` class.
-Modules are now passed a ``module_api`` argument during initialisation, which is an instance of
-``ModuleApi``. ``ModuleApi`` instances have a ``http_client`` property which acts the same as
-the ``http_client`` argument previously passed to ``ThirdPartyEventRules`` modules.
-
-Upgrading to v1.21.0
-====================
-
-Forwarding ``/_synapse/client`` through your reverse proxy
-----------------------------------------------------------
-
-The `reverse proxy documentation
-<https://github.com/matrix-org/synapse/blob/develop/docs/reverse_proxy.md>`_ has been updated
-to include reverse proxy directives for ``/_synapse/client/*`` endpoints. As the user password
-reset flow now uses endpoints under this prefix, **you must update your reverse proxy
-configurations for user password reset to work**.
-
-Additionally, note that the `Synapse worker documentation
-<https://github.com/matrix-org/synapse/blob/develop/docs/workers.md>`_ has been updated to
- state that the ``/_synapse/client/password_reset/email/submit_token`` endpoint can be handled
-by all workers. If you make use of Synapse's worker feature, please update your reverse proxy
-configuration to reflect this change.
-
-New HTML templates
-------------------
-
-A new HTML template,
-`password_reset_confirmation.html <https://github.com/matrix-org/synapse/blob/develop/synapse/res/templates/password_reset_confirmation.html>`_,
-has been added to the ``synapse/res/templates`` directory. If you are using a
-custom template directory, you may want to copy the template over and modify it.
-
-Note that as of v1.20.0, templates do not need to be included in custom template
-directories for Synapse to start. The default templates will be used if a custom
-template cannot be found.
-
-This page will appear to the user after clicking a password reset link that has
-been emailed to them.
-
-To complete password reset, the page must include a way to make a `POST`
-request to
-``/_synapse/client/password_reset/{medium}/submit_token``
-with the query parameters from the original link, presented as a URL-encoded form. See the file
-itself for more details.
-
-Updated Single Sign-on HTML Templates
--------------------------------------
-
-The ``saml_error.html`` template was removed from Synapse and replaced with the
-``sso_error.html`` template. If your Synapse is configured to use SAML and a
-custom ``sso_redirect_confirm_template_dir`` configuration then any customisations
-of the ``saml_error.html`` template will need to be merged into the ``sso_error.html``
-template. These templates are similar, but the parameters are slightly different:
-
-* The ``msg`` parameter should be renamed to ``error_description``.
-* There is no longer a ``code`` parameter for the response code.
-* A string ``error`` parameter is available that includes a short hint of why a
-  user is seeing the error page.
-
-Upgrading to v1.18.0
-====================
-
-Docker `-py3` suffix will be removed in future versions
--------------------------------------------------------
-
-From 10th August 2020, we will no longer publish Docker images with the `-py3` tag suffix. The images tagged with the `-py3` suffix have been identical to the non-suffixed tags since release 0.99.0, and the suffix is obsolete.
-
-On 10th August, we will remove the `latest-py3` tag. Existing per-release tags (such as `v1.18.0-py3`) will not be removed, but no new `-py3` tags will be added.
-
-Scripts relying on the `-py3` suffix will need to be updated.
-
-Redis replication is now recommended in lieu of TCP replication
----------------------------------------------------------------
-
-When setting up worker processes, we now recommend the use of a Redis server for replication. **The old direct TCP connection method is deprecated and will be removed in a future release.**
-See `docs/workers.md <docs/workers.md>`_ for more details.
-
-Upgrading to v1.14.0
-====================
-
-This version includes a database update which is run as part of the upgrade,
-and which may take a couple of minutes in the case of a large server. Synapse
-will not respond to HTTP requests while this update is taking place.
-
-Upgrading to v1.13.0
-====================
-
-Incorrect database migration in old synapse versions
-----------------------------------------------------
-
-A bug was introduced in Synapse 1.4.0 which could cause the room directory to
-be incomplete or empty if Synapse was upgraded directly from v1.2.1 or
-earlier, to versions between v1.4.0 and v1.12.x.
-
-This will *not* be a problem for Synapse installations which were:
- * created at v1.4.0 or later,
- * upgraded via v1.3.x, or
- * upgraded straight from v1.2.1 or earlier to v1.13.0 or later.
-
-If completeness of the room directory is a concern, installations which are
-affected can be repaired as follows:
-
-1. Run the following sql from a `psql` or `sqlite3` console:
-
-   .. code:: sql
-
-     INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
-        ('populate_stats_process_rooms', '{}', 'current_state_events_membership');
-
-     INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
-        ('populate_stats_process_users', '{}', 'populate_stats_process_rooms');
-
-2. Restart synapse.
-
-New Single Sign-on HTML Templates
----------------------------------
-
-New templates (``sso_auth_confirm.html``, ``sso_auth_success.html``, and
-``sso_account_deactivated.html``) were added to Synapse. If your Synapse is
-configured to use SSO and a custom  ``sso_redirect_confirm_template_dir``
-configuration then these templates will need to be copied from
-`synapse/res/templates <synapse/res/templates>`_ into that directory.
-
-Synapse SSO Plugins Method Deprecation
---------------------------------------
-
-Plugins using the ``complete_sso_login`` method of
-``synapse.module_api.ModuleApi`` should update to using the async/await
-version ``complete_sso_login_async`` which includes additional checks. The
-non-async version is considered deprecated.
-
-Rolling back to v1.12.4 after a failed upgrade
-----------------------------------------------
-
-v1.13.0 includes a lot of large changes. If something problematic occurs, you
-may want to roll-back to a previous version of Synapse. Because v1.13.0 also
-includes a new database schema version, reverting that version is also required
-alongside the generic rollback instructions mentioned above. In short, to roll
-back to v1.12.4 you need to:
-
-1. Stop the server
-2. Decrease the schema version in the database:
-
-   .. code:: sql
-
-      UPDATE schema_version SET version = 57;
-
-3. Downgrade Synapse by following the instructions for your installation method
-   in the "Rolling back to older versions" section above.
-
-
-Upgrading to v1.12.0
-====================
-
-This version includes a database update which is run as part of the upgrade,
-and which may take some time (several hours in the case of a large
-server). Synapse will not respond to HTTP requests while this update is taking
-place.
-
-This is only likely to be a problem in the case of a server which is
-participating in many rooms.
-
-0. As with all upgrades, it is recommended that you have a recent backup of
-   your database which can be used for recovery in the event of any problems.
-
-1. As an initial check to see if you will be affected, you can try running the
-   following query from the `psql` or `sqlite3` console. It is safe to run it
-   while Synapse is still running.
-
-   .. code:: sql
-
-      SELECT MAX(q.v) FROM (
-        SELECT (
-          SELECT ej.json AS v
-          FROM state_events se INNER JOIN event_json ej USING (event_id)
-          WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key=''
-          LIMIT 1
-        ) FROM rooms WHERE rooms.room_version IS NULL
-      ) q;
-
-   This query will take about the same amount of time as the upgrade process: ie,
-   if it takes 5 minutes, then it is likely that Synapse will be unresponsive for
-   5 minutes during the upgrade.
-
-   If you consider an outage of this duration to be acceptable, no further
-   action is necessary and you can simply start Synapse 1.12.0.
-
-   If you would prefer to reduce the downtime, continue with the steps below.
-
-2. The easiest workaround for this issue is to manually
-   create a new index before upgrading. On PostgreSQL, his can be done as follows:
-
-   .. code:: sql
-
-      CREATE INDEX CONCURRENTLY tmp_upgrade_1_12_0_index
-      ON state_events(room_id) WHERE type = 'm.room.create';
-
-   The above query may take some time, but is also safe to run while Synapse is
-   running.
-
-   We assume that no SQLite users have databases large enough to be
-   affected. If you *are* affected, you can run a similar query, omitting the
-   ``CONCURRENTLY`` keyword. Note however that this operation may in itself cause
-   Synapse to stop running for some time. Synapse admins are reminded that
-   `SQLite is not recommended for use outside a test
-   environment <https://github.com/matrix-org/synapse/blob/master/README.rst#using-postgresql>`_.
-
-3. Once the index has been created, the ``SELECT`` query in step 1 above should
-   complete quickly. It is therefore safe to upgrade to Synapse 1.12.0.
-
-4. Once Synapse 1.12.0 has successfully started and is responding to HTTP
-   requests, the temporary index can be removed:
-
-   .. code:: sql
-
-      DROP INDEX tmp_upgrade_1_12_0_index;
-
-Upgrading to v1.10.0
-====================
-
-Synapse will now log a warning on start up if used with a PostgreSQL database
-that has a non-recommended locale set.
-
-See `docs/postgres.md <docs/postgres.md>`_ for details.
-
-
-Upgrading to v1.8.0
-===================
-
-Specifying a ``log_file`` config option will now cause Synapse to refuse to
-start, and should be replaced by with the ``log_config`` option. Support for
-the ``log_file`` option was removed in v1.3.0 and has since had no effect.
-
-
-Upgrading to v1.7.0
-===================
-
-In an attempt to configure Synapse in a privacy preserving way, the default
-behaviours of ``allow_public_rooms_without_auth`` and
-``allow_public_rooms_over_federation`` have been inverted. This means that by
-default, only authenticated users querying the Client/Server API will be able
-to query the room directory, and relatedly that the server will not share
-room directory information with other servers over federation.
-
-If your installation does not explicitly set these settings one way or the other
-and you want either setting to be ``true`` then it will necessary to update
-your homeserver configuration file accordingly.
-
-For more details on the surrounding context see our `explainer
-<https://matrix.org/blog/2019/11/09/avoiding-unwelcome-visitors-on-private-matrix-servers>`_.
-
-
-Upgrading to v1.5.0
-===================
-
-This release includes a database migration which may take several minutes to
-complete if there are a large number (more than a million or so) of entries in
-the ``devices`` table. This is only likely to a be a problem on very large
-installations.
-
-
-Upgrading to v1.4.0
-===================
-
-New custom templates
---------------------
-
-If you have configured a custom template directory with the
-``email.template_dir`` option, be aware that there are new templates regarding
-registration and threepid management (see below) that must be included.
-
-* ``registration.html`` and ``registration.txt``
-* ``registration_success.html`` and ``registration_failure.html``
-* ``add_threepid.html`` and  ``add_threepid.txt``
-* ``add_threepid_failure.html`` and ``add_threepid_success.html``
-
-Synapse will expect these files to exist inside the configured template
-directory, and **will fail to start** if they are absent.
-To view the default templates, see `synapse/res/templates
-<https://github.com/matrix-org/synapse/tree/master/synapse/res/templates>`_.
-
-3pid verification changes
--------------------------
-
-**Note: As of this release, users will be unable to add phone numbers or email
-addresses to their accounts, without changes to the Synapse configuration. This
-includes adding an email address during registration.**
-
-It is possible for a user to associate an email address or phone number
-with their account, for a number of reasons:
-
-* for use when logging in, as an alternative to the user id.
-* in the case of email, as an alternative contact to help with account recovery.
-* in the case of email, to receive notifications of missed messages.
-
-Before an email address or phone number can be added to a user's account,
-or before such an address is used to carry out a password-reset, Synapse must
-confirm the operation with the owner of the email address or phone number.
-It does this by sending an email or text giving the user a link or token to confirm
-receipt. This process is known as '3pid verification'. ('3pid', or 'threepid',
-stands for third-party identifier, and we use it to refer to external
-identifiers such as email addresses and phone numbers.)
-
-Previous versions of Synapse delegated the task of 3pid verification to an
-identity server by default. In most cases this server is ``vector.im`` or
-``matrix.org``.
-
-In Synapse 1.4.0, for security and privacy reasons, the homeserver will no
-longer delegate this task to an identity server by default. Instead,
-the server administrator will need to explicitly decide how they would like the
-verification messages to be sent.
-
-In the medium term, the ``vector.im`` and ``matrix.org`` identity servers will
-disable support for delegated 3pid verification entirely. However, in order to
-ease the transition, they will retain the capability for a limited
-period. Delegated email verification will be disabled on Monday 2nd December
-2019 (giving roughly 2 months notice). Disabling delegated SMS verification
-will follow some time after that once SMS verification support lands in
-Synapse.
-
-Once delegated 3pid verification support has been disabled in the ``vector.im`` and
-``matrix.org`` identity servers, all Synapse versions that depend on those
-instances will be unable to verify email and phone numbers through them. There
-are no imminent plans to remove delegated 3pid verification from Sydent
-generally. (Sydent is the identity server project that backs the ``vector.im`` and
-``matrix.org`` instances).
-
-Email
-~~~~~
-Following upgrade, to continue verifying email (e.g. as part of the
-registration process), admins can either:-
-
-* Configure Synapse to use an email server.
-* Run or choose an identity server which allows delegated email verification
-  and delegate to it.
-
-Configure SMTP in Synapse
-+++++++++++++++++++++++++
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed ``email``, and be sure to have at least the ``smtp_host, smtp_port``
-and ``notif_from`` fields filled out.
-
-You may also need to set ``smtp_user``, ``smtp_pass``, and
-``require_transport_security``.
-
-See the `sample configuration file <docs/sample_config.yaml>`_ for more details
-on these settings.
-
-Delegate email to an identity server
-++++++++++++++++++++++++++++++++++++
-
-Some admins will wish to continue using email verification as part of the
-registration process, but will not immediately have an appropriate SMTP server
-at hand.
-
-To this end, we will continue to support email verification delegation via the
-``vector.im`` and ``matrix.org`` identity servers for two months. Support for
-delegated email verification will be disabled on Monday 2nd December.
-
-The ``account_threepid_delegates`` dictionary defines whether the homeserver
-should delegate an external server (typically an `identity server
-<https://matrix.org/docs/spec/identity_service/r0.2.1>`_) to handle sending
-confirmation messages via email and SMS.
-
-So to delegate email verification, in ``homeserver.yaml``, set
-``account_threepid_delegates.email`` to the base URL of an identity server. For
-example:
-
-.. code:: yaml
-
-   account_threepid_delegates:
-       email: https://example.com     # Delegate email sending to example.com
-
-Note that ``account_threepid_delegates.email`` replaces the deprecated
-``email.trust_identity_server_for_password_resets``: if
-``email.trust_identity_server_for_password_resets`` is set to ``true``, and
-``account_threepid_delegates.email`` is not set, then the first entry in
-``trusted_third_party_id_servers`` will be used as the
-``account_threepid_delegate`` for email. This is to ensure compatibility with
-existing Synapse installs that set up external server handling for these tasks
-before v1.4.0. If ``email.trust_identity_server_for_password_resets`` is
-``true`` and no trusted identity server domains are configured, Synapse will
-report an error and refuse to start.
-
-If ``email.trust_identity_server_for_password_resets`` is ``false`` or absent
-and no ``email`` delegate is configured in ``account_threepid_delegates``,
-then Synapse will send email verification messages itself, using the configured
-SMTP server (see above).
-that type.
-
-Phone numbers
-~~~~~~~~~~~~~
-
-Synapse does not support phone-number verification itself, so the only way to
-maintain the ability for users to add phone numbers to their accounts will be
-by continuing to delegate phone number verification to the ``matrix.org`` and
-``vector.im`` identity servers (or another identity server that supports SMS
-sending).
-
-The ``account_threepid_delegates`` dictionary defines whether the homeserver
-should delegate an external server (typically an `identity server
-<https://matrix.org/docs/spec/identity_service/r0.2.1>`_) to handle sending
-confirmation messages via email and SMS.
-
-So to delegate phone number verification, in ``homeserver.yaml``, set
-``account_threepid_delegates.msisdn`` to the base URL of an identity
-server. For example:
-
-.. code:: yaml
-
-   account_threepid_delegates:
-       msisdn: https://example.com     # Delegate sms sending to example.com
-
-The ``matrix.org`` and ``vector.im`` identity servers will continue to support
-delegated phone number verification via SMS until such time as it is possible
-for admins to configure their servers to perform phone number verification
-directly. More details will follow in a future release.
-
-Rolling back to v1.3.1
-----------------------
-
-If you encounter problems with v1.4.0, it should be possible to roll back to
-v1.3.1, subject to the following:
-
-* The 'room statistics' engine was heavily reworked in this release (see
-  `#5971 <https://github.com/matrix-org/synapse/pull/5971>`_), including
-  significant changes to the database schema, which are not easily
-  reverted. This will cause the room statistics engine to stop updating when
-  you downgrade.
-
-  The room statistics are essentially unused in v1.3.1 (in future versions of
-  Synapse, they will be used to populate the room directory), so there should
-  be no loss of functionality. However, the statistics engine will write errors
-  to the logs, which can be avoided by setting the following in
-  `homeserver.yaml`:
-
-  .. code:: yaml
-
-    stats:
-      enabled: false
-
-  Don't forget to re-enable it when you upgrade again, in preparation for its
-  use in the room directory!
-
-Upgrading to v1.2.0
-===================
-
-Some counter metrics have been renamed, with the old names deprecated. See
-`the metrics documentation <docs/metrics-howto.md#renaming-of-metrics--deprecation-of-old-names-in-12>`_
-for details.
-
-Upgrading to v1.1.0
-===================
-
-Synapse v1.1.0 removes support for older Python and PostgreSQL versions, as
-outlined in `our deprecation notice <https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x>`_.
-
-Minimum Python Version
-----------------------
-
-Synapse v1.1.0 has a minimum Python requirement of Python 3.5. Python 3.6 or
-Python 3.7 are recommended as they have improved internal string handling,
-significantly reducing memory usage.
-
-If you use current versions of the Matrix.org-distributed Debian packages or
-Docker images, action is not required.
-
-If you install Synapse in a Python virtual environment, please see "Upgrading to
-v0.34.0" for notes on setting up a new virtualenv under Python 3.
-
-Minimum PostgreSQL Version
---------------------------
-
-If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above.
-Please see the
-`PostgreSQL documentation <https://www.postgresql.org/docs/11/upgrading.html>`_
-for more details on upgrading your database.
-
-Upgrading to v1.0
-=================
-
-Validation of TLS certificates
-------------------------------
-
-Synapse v1.0 is the first release to enforce
-validation of TLS certificates for the federation API. It is therefore
-essential that your certificates are correctly configured. See the `FAQ
-<docs/MSC1711_certificates_FAQ.md>`_ for more information.
-
-Note, v1.0 installations will also no longer be able to federate with servers
-that have not correctly configured their certificates.
-
-In rare cases, it may be desirable to disable certificate checking: for
-example, it might be essential to be able to federate with a given legacy
-server in a closed federation. This can be done in one of two ways:-
-
-* Configure the global switch ``federation_verify_certificates`` to ``false``.
-* Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``.
-
-See the `sample configuration file <docs/sample_config.yaml>`_
-for more details on these settings.
-
-Email
------
-When a user requests a password reset, Synapse will send an email to the
-user to confirm the request.
-
-Previous versions of Synapse delegated the job of sending this email to an
-identity server. If the identity server was somehow malicious or became
-compromised, it would be theoretically possible to hijack an account through
-this means.
-
-Therefore, by default, Synapse v1.0 will send the confirmation email itself. If
-Synapse is not configured with an SMTP server, password reset via email will be
-disabled.
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
-and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
-``smtp_pass``, and ``require_transport_security``.
-
-If you are absolutely certain that you wish to continue using an identity
-server for password resets, set ``trust_identity_server_for_password_resets`` to ``true``.
-
-See the `sample configuration file <docs/sample_config.yaml>`_
-for more details on these settings.
-
-New email templates
----------------
-Some new templates have been added to the default template directory for the purpose of the
-homeserver sending its own password reset emails. If you have configured a custom
-``template_dir`` in your Synapse config, these files will need to be added.
-
-``password_reset.html`` and ``password_reset.txt`` are HTML and plain text templates
-respectively that contain the contents of what will be emailed to the user upon attempting to
-reset their password via email. ``password_reset_success.html`` and
-``password_reset_failure.html`` are HTML files that the content of which (assuming no redirect
-URL is set) will be shown to the user after they attempt to click the link in the email sent
-to them.
-
-Upgrading to v0.99.0
-====================
-
-Please be aware that, before Synapse v1.0 is released around March 2019, you
-will need to replace any self-signed certificates with those verified by a
-root CA. Information on how to do so can be found at `the ACME docs
-<docs/ACME.md>`_.
-
-For more information on configuring TLS certificates see the `FAQ <docs/MSC1711_certificates_FAQ.md>`_.
-
-Upgrading to v0.34.0
-====================
-
-1. This release is the first to fully support Python 3. Synapse will now run on
-   Python versions 3.5, or 3.6 (as well as 2.7). We recommend switching to
-   Python 3, as it has been shown to give performance improvements.
-
-   For users who have installed Synapse into a virtualenv, we recommend doing
-   this by creating a new virtualenv. For example::
-
-       virtualenv -p python3 ~/synapse/env3
-       source ~/synapse/env3/bin/activate
-       pip install matrix-synapse
-
-   You can then start synapse as normal, having activated the new virtualenv::
-
-       cd ~/synapse
-       source env3/bin/activate
-       synctl start
-
-   Users who have installed from distribution packages should see the relevant
-   package documentation. See below for notes on Debian packages.
-
-   * When upgrading to Python 3, you **must** make sure that your log files are
-     configured as UTF-8, by adding ``encoding: utf8`` to the
-     ``RotatingFileHandler`` configuration (if you have one) in your
-     ``<server>.log.config`` file. For example, if your ``log.config`` file
-     contains::
-
-       handlers:
-         file:
-           class: logging.handlers.RotatingFileHandler
-           formatter: precise
-           filename: homeserver.log
-           maxBytes: 104857600
-           backupCount: 10
-           filters: [context]
-         console:
-           class: logging.StreamHandler
-           formatter: precise
-           filters: [context]
-
-     Then you should update this to be::
-
-       handlers:
-         file:
-           class: logging.handlers.RotatingFileHandler
-           formatter: precise
-           filename: homeserver.log
-           maxBytes: 104857600
-           backupCount: 10
-           filters: [context]
-           encoding: utf8
-         console:
-           class: logging.StreamHandler
-           formatter: precise
-           filters: [context]
-
-     There is no need to revert this change if downgrading to Python 2.
-
-   We are also making available Debian packages which will run Synapse on
-   Python 3. You can switch to these packages with ``apt-get install
-   matrix-synapse-py3``, however, please read `debian/NEWS
-   <https://github.com/matrix-org/synapse/blob/release-v0.34.0/debian/NEWS>`_
-   before doing so. The existing ``matrix-synapse`` packages will continue to
-   use Python 2 for the time being.
-
-2. This release removes the ``riot.im`` from the default list of trusted
-   identity servers.
-
-   If ``riot.im`` is in your homeserver's list of
-   ``trusted_third_party_id_servers``, you should remove it. It was added in
-   case a hypothetical future identity server was put there. If you don't
-   remove it, users may be unable to deactivate their accounts.
-
-3. This release no longer installs the (unmaintained) Matrix Console web client
-   as part of the default installation. It is possible to re-enable it by
-   installing it separately and setting the ``web_client_location`` config
-   option, but please consider switching to another client.
-
-Upgrading to v0.33.7
-====================
-
-This release removes the example email notification templates from
-``res/templates`` (they are now internal to the python package). This should
-only affect you if you (a) deploy your Synapse instance from a git checkout or
-a github snapshot URL, and (b) have email notifications enabled.
-
-If you have email notifications enabled, you should ensure that
-``email.template_dir`` is either configured to point at a directory where you
-have installed customised templates, or leave it unset to use the default
-templates.
-
-Upgrading to v0.27.3
-====================
-
-This release expands the anonymous usage stats sent if the opt-in
-``report_stats`` configuration is set to ``true``. We now capture RSS memory
-and cpu use at a very coarse level. This requires administrators to install
-the optional ``psutil`` python module.
-
-We would appreciate it if you could assist by ensuring this module is available
-and ``report_stats`` is enabled. This will let us see if performance changes to
-synapse are having an impact to the general community.
-
-Upgrading to v0.15.0
-====================
-
-If you want to use the new URL previewing API (/_matrix/media/r0/preview_url)
-then you have to explicitly enable it in the config and update your dependencies
-dependencies.  See README.rst for details.
-
-
-Upgrading to v0.11.0
-====================
-
-This release includes the option to send anonymous usage stats to matrix.org,
-and requires that administrators explictly opt in or out by setting the
-``report_stats`` option to either ``true`` or ``false``.
-
-We would really appreciate it if you could help our project out by reporting
-anonymized usage statistics from your homeserver. Only very basic aggregate
-data (e.g. number of users) will be reported, but it helps us to track the
-growth of the Matrix community, and helps us to make Matrix a success, as well
-as to convince other networks that they should peer with us.
-
-
-Upgrading to v0.9.0
-===================
-
-Application services have had a breaking API change in this version.
-
-They can no longer register themselves with a home server using the AS HTTP API. This
-decision was made because a compromised application service with free reign to register
-any regex in effect grants full read/write access to the home server if a regex of ``.*``
-is used. An attack where a compromised AS re-registers itself with ``.*`` was deemed too
-big of a security risk to ignore, and so the ability to register with the HS remotely has
-been removed.
-
-It has been replaced by specifying a list of application service registrations in
-``homeserver.yaml``::
-
-  app_service_config_files: ["registration-01.yaml", "registration-02.yaml"]
-
-Where ``registration-01.yaml`` looks like::
-
-  url: <String>  # e.g. "https://my.application.service.com"
-  as_token: <String>
-  hs_token: <String>
-  sender_localpart: <String>  # This is a new field which denotes the user_id localpart when using the AS token
-  namespaces:
-    users:
-      - exclusive: <Boolean>
-        regex: <String>  # e.g. "@prefix_.*"
-    aliases:
-      - exclusive: <Boolean>
-        regex: <String>
-    rooms:
-      - exclusive: <Boolean>
-        regex: <String>
-
-Upgrading to v0.8.0
-===================
-
-Servers which use captchas will need to add their public key to::
-
-  static/client/register/register_config.js
-
-    window.matrixRegistrationConfig = {
-        recaptcha_public_key: "YOUR_PUBLIC_KEY"
-    };
-
-This is required in order to support registration fallback (typically used on
-mobile devices).
-
-
-Upgrading to v0.7.0
-===================
-
-New dependencies are:
-
-- pydenticon
-- simplejson
-- syutil
-- matrix-angular-sdk
-
-To pull in these dependencies in a virtual env, run::
-
-    python synapse/python_dependencies.py | xargs -n 1 pip install
-
-Upgrading to v0.6.0
-===================
-
-To pull in new dependencies, run::
-
-    python setup.py develop --user
-
-This update includes a change to the database schema. To upgrade you first need
-to upgrade the database by running::
-
-    python scripts/upgrade_db_to_v0.6.0.py <db> <server_name> <signing_key>
-
-Where `<db>` is the location of the database, `<server_name>` is the
-server name as specified in the synapse configuration, and `<signing_key>` is
-the location of the signing key as specified in the synapse configuration.
-
-This may take some time to complete. Failures of signatures and content hashes
-can safely be ignored.
-
-
-Upgrading to v0.5.1
-===================
-
-Depending on precisely when you installed v0.5.0 you may have ended up with
-a stale release of the reference matrix webclient installed as a python module.
-To uninstall it and ensure you are depending on the latest module, please run::
-
-    $ pip uninstall syweb
-
-Upgrading to v0.5.0
-===================
-
-The webclient has been split out into a seperate repository/pacakage in this
-release. Before you restart your homeserver you will need to pull in the
-webclient package by running::
-
-  python setup.py develop --user
-
-This release completely changes the database schema and so requires upgrading
-it before starting the new version of the homeserver.
-
-The script "database-prepare-for-0.5.0.sh" should be used to upgrade the
-database. This will save all user information, such as logins and profiles,
-but will otherwise purge the database. This includes messages, which
-rooms the home server was a member of and room alias mappings.
-
-If you would like to keep your history, please take a copy of your database
-file and ask for help in #matrix:matrix.org. The upgrade process is,
-unfortunately, non trivial and requires human intervention to resolve any
-resulting conflicts during the upgrade process.
-
-Before running the command the homeserver should be first completely
-shutdown. To run it, simply specify the location of the database, e.g.:
-
-  ./scripts/database-prepare-for-0.5.0.sh "homeserver.db"
-
-Once this has successfully completed it will be safe to restart the
-homeserver. You may notice that the homeserver takes a few seconds longer to
-restart than usual as it reinitializes the database.
-
-On startup of the new version, users can either rejoin remote rooms using room
-aliases or by being reinvited. Alternatively, if any other homeserver sends a
-message to a room that the homeserver was previously in the local HS will
-automatically rejoin the room.
-
-Upgrading to v0.4.0
-===================
-
-This release needs an updated syutil version. Run::
-
-    python setup.py develop
-
-You will also need to upgrade your configuration as the signing key format has
-changed. Run::
-
-    python -m synapse.app.homeserver --config-path <CONFIG> --generate-config
-
-
-Upgrading to v0.3.0
-===================
-
-This registration API now closely matches the login API. This introduces a bit
-more backwards and forwards between the HS and the client, but this improves
-the overall flexibility of the API. You can now GET on /register to retrieve a list
-of valid registration flows. Upon choosing one, they are submitted in the same
-way as login, e.g::
-
-  {
-    type: m.login.password,
-    user: foo,
-    password: bar
-  }
-
-The default HS supports 2 flows, with and without Identity Server email
-authentication. Enabling captcha on the HS will add in an extra step to all
-flows: ``m.login.recaptcha`` which must be completed before you can transition
-to the next stage. There is a new login type: ``m.login.email.identity`` which
-contains the ``threepidCreds`` key which were previously sent in the original
-register request. For more information on this, see the specification.
-
-Web Client
-----------
-
-The VoIP specification has changed between v0.2.0 and v0.3.0. Users should
-refresh any browser tabs to get the latest web client code. Users on
-v0.2.0 of the web client will not be able to call those on v0.3.0 and
-vice versa.
-
-
-Upgrading to v0.2.0
-===================
-
-The home server now requires setting up of SSL config before it can run. To
-automatically generate default config use::
-
-    $ python synapse/app/homeserver.py \
-        --server-name machine.my.domain.name \
-        --bind-port 8448 \
-        --config-path homeserver.config \
-        --generate-config
-
-This config can be edited if desired, for example to specify a different SSL
-certificate to use. Once done you can run the home server using::
-
-    $ python synapse/app/homeserver.py --config-path homeserver.config
-
-See the README.rst for more information.
-
-Also note that some config options have been renamed, including:
-
-- "host" to "server-name"
-- "database" to "database-path"
-- "port" to "bind-port" and "unsecure-port"
-
-
-Upgrading to v0.0.1
-===================
-
-This release completely changes the database schema and so requires upgrading
-it before starting the new version of the homeserver.
-
-The script "database-prepare-for-0.0.1.sh" should be used to upgrade the
-database. This will save all user information, such as logins and profiles,
-but will otherwise purge the database. This includes messages, which
-rooms the home server was a member of and room alias mappings.
-
-Before running the command the homeserver should be first completely
-shutdown. To run it, simply specify the location of the database, e.g.:
-
-  ./scripts/database-prepare-for-0.0.1.sh "homeserver.db"
-
-Once this has successfully completed it will be safe to restart the
-homeserver. You may notice that the homeserver takes a few seconds longer to
-restart than usual as it reinitializes the database.
-
-On startup of the new version, users can either rejoin remote rooms using room
-aliases or by being reinvited. Alternatively, if any other homeserver sends a
-message to a room that the homeserver was previously in the local HS will
-automatically rejoin the room.
+The markdown source is available in `docs/upgrade.md <docs/upgrade.md>`_.
diff --git a/book.toml b/book.toml
new file mode 100644
index 0000000000..fa83d86ffc
--- /dev/null
+++ b/book.toml
@@ -0,0 +1,39 @@
+# Documentation for possible options in this file is at
+# https://rust-lang.github.io/mdBook/format/config.html
+[book]
+title = "Synapse"
+authors = ["The Matrix.org Foundation C.I.C."]
+language = "en"
+multilingual = false
+
+# The directory that documentation files are stored in
+src = "docs"
+
+[build]
+# Prevent markdown pages from being automatically generated when they're
+# linked to in SUMMARY.md
+create-missing = false
+
+[output.html]
+# The URL visitors will be directed to when they try to edit a page
+edit-url-template = "https://github.com/matrix-org/synapse/edit/develop/{path}"
+
+# Remove the numbers that appear before each item in the sidebar, as they can
+# get quite messy as we nest deeper
+no-section-label = true
+
+# The source code URL of the repository
+git-repository-url = "https://github.com/matrix-org/synapse"
+
+# The path that the docs are hosted on
+site-url = "/synapse/"
+
+# Additional HTML, JS, CSS that's injected into each page of the book.
+# More information available in docs/website_files/README.md
+additional-css = [
+    "docs/website_files/table-of-contents.css",
+    "docs/website_files/remove-nav-buttons.css",
+    "docs/website_files/indent-section-headers.css",
+]
+additional-js = ["docs/website_files/table-of-contents.js"]
+theme = "docs/website_files/theme"
\ No newline at end of file
diff --git a/changelog.d/10713.bugfix b/changelog.d/10713.bugfix
new file mode 100644
index 0000000000..e8caf3d23a
--- /dev/null
+++ b/changelog.d/10713.bugfix
@@ -0,0 +1 @@
+Fix a regression introduced in Synapse 1.41 which broke email transmission on Systems using older versions of the Twisted library.
diff --git a/changelog.d/10723.bugfix b/changelog.d/10723.bugfix
new file mode 100644
index 0000000000..e6ffdc9512
--- /dev/null
+++ b/changelog.d/10723.bugfix
@@ -0,0 +1 @@
+Fix unauthorised exposure of room metadata to communities.
diff --git a/contrib/cmdclient/console.py b/contrib/cmdclient/console.py
index 67e032244e..856dd437db 100755
--- a/contrib/cmdclient/console.py
+++ b/contrib/cmdclient/console.py
@@ -24,6 +24,7 @@
 import time
 import urllib
 from http import TwistedHttpClient
+from typing import Optional
 
 import nacl.encoding
 import nacl.signing
@@ -718,7 +719,7 @@ def _run_and_pprint(
         method,
         path,
         data=None,
-        query_params={"access_token": None},
+        query_params: Optional[dict] = None,
         alt_text=None,
     ):
         """Runs an HTTP request and pretty prints the output.
@@ -729,6 +730,8 @@ def _run_and_pprint(
             data: Raw JSON data if any
             query_params: dict of query parameters to add to the url
         """
+        query_params = query_params or {"access_token": None}
+
         url = self._url() + path
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()
diff --git a/contrib/cmdclient/http.py b/contrib/cmdclient/http.py
index 851e80c25b..1310f078e3 100644
--- a/contrib/cmdclient/http.py
+++ b/contrib/cmdclient/http.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,6 +15,7 @@
 import json
 import urllib
 from pprint import pformat
+from typing import Optional
 
 from twisted.internet import defer, reactor
 from twisted.web.client import Agent, readBody
@@ -85,8 +85,9 @@ def get_json(self, url, args=None):
         body = yield readBody(response)
         defer.returnValue(json.loads(body))
 
-    def _create_put_request(self, url, json_data, headers_dict={}):
+    def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
         """Wrapper of _create_request to issue a PUT request"""
+        headers_dict = headers_dict or {}
 
         if "Content-Type" not in headers_dict:
             raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
@@ -95,14 +96,22 @@ def _create_put_request(self, url, json_data, headers_dict={}):
             "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
         )
 
-    def _create_get_request(self, url, headers_dict={}):
+    def _create_get_request(self, url, headers_dict: Optional[dict] = None):
         """Wrapper of _create_request to issue a GET request"""
-        return self._create_request("GET", url, headers_dict=headers_dict)
+        return self._create_request("GET", url, headers_dict=headers_dict or {})
 
     @defer.inlineCallbacks
     def do_request(
-        self, method, url, data=None, qparams=None, jsonreq=True, headers={}
+        self,
+        method,
+        url,
+        data=None,
+        qparams=None,
+        jsonreq=True,
+        headers: Optional[dict] = None,
     ):
+        headers = headers or {}
+
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
@@ -123,8 +132,12 @@ def do_request(
         defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
-    def _create_request(self, method, url, producer=None, headers_dict={}):
+    def _create_request(
+        self, method, url, producer=None, headers_dict: Optional[dict] = None
+    ):
         """Creates and sends a request to the given url"""
+        headers_dict = headers_dict or {}
+
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index d1ecd453db..26d640c448 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -56,7 +56,7 @@ services:
       - POSTGRES_USER=synapse
       - POSTGRES_PASSWORD=changeme
       # ensure the database gets created correctly
-      # https://github.com/matrix-org/synapse/blob/master/docs/postgres.md#set-up-database
+      # https://matrix-org.github.io/synapse/latest/postgres.html#set-up-database
       - POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
     volumes:
       # You may store the database tables in a local folder..
diff --git a/contrib/experiments/cursesio.py b/contrib/experiments/cursesio.py
index cff73650e6..7695cc77ca 100644
--- a/contrib/experiments/cursesio.py
+++ b/contrib/experiments/cursesio.py
@@ -46,14 +46,14 @@ def set_callback(self, callback):
         self.callback = callback
 
     def fileno(self):
-        """ We want to select on FD 0 """
+        """We want to select on FD 0"""
         return 0
 
     def connectionLost(self, reason):
         self.close()
 
     def print_line(self, text):
-        """ add a line to the internal list of lines"""
+        """add a line to the internal list of lines"""
 
         self.lines.append(text)
         self.redraw()
@@ -92,7 +92,7 @@ def printLogLine(self, text):
         )
 
     def doRead(self):
-        """ Input is ready! """
+        """Input is ready!"""
         curses.noecho()
         c = self.stdscr.getch()  # read a character
 
@@ -132,7 +132,7 @@ def logPrefix(self):
         return "CursesStdIO"
 
     def close(self):
-        """ clean up """
+        """clean up"""
 
         curses.nocbreak()
         self.stdscr.keypad(0)
diff --git a/contrib/experiments/test_messaging.py b/contrib/experiments/test_messaging.py
index 7fbc7d8fc6..31b8a68225 100644
--- a/contrib/experiments/test_messaging.py
+++ b/contrib/experiments/test_messaging.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/contrib/grafana/README.md b/contrib/grafana/README.md
index 4608793394..0d4e1b59b2 100644
--- a/contrib/grafana/README.md
+++ b/contrib/grafana/README.md
@@ -1,6 +1,6 @@
 # Using the Synapse Grafana dashboard
 
 0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
-1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
+1. Have your Prometheus scrape your Synapse. https://matrix-org.github.io/synapse/latest/metrics-howto.html
 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
-3. Set up required recording rules. https://github.com/matrix-org/synapse/tree/master/contrib/prometheus
+3. Set up required recording rules. [contrib/prometheus](../prometheus)
diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json
index 539569b5b1..ed1e8ba7f8 100644
--- a/contrib/grafana/synapse.json
+++ b/contrib/grafana/synapse.json
@@ -14,7 +14,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "6.7.4"
+      "version": "7.3.7"
     },
     {
       "type": "panel",
@@ -38,7 +38,6 @@
   "annotations": {
     "list": [
       {
-        "$$hashKey": "object:76",
         "builtIn": 1,
         "datasource": "$datasource",
         "enable": false,
@@ -55,11 +54,12 @@
   "gnetId": null,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1594646317221,
+  "iteration": 1628606819564,
   "links": [
     {
-      "asDropdown": true,
+      "asDropdown": false,
       "icon": "external link",
+      "includeVars": true,
       "keepTime": true,
       "tags": [
         "matrix"
@@ -83,73 +83,253 @@
       "title": "Overview",
       "type": "row"
     },
+    {
+      "cards": {
+        "cardPadding": -1,
+        "cardRound": 0
+      },
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolateInferno",
+        "exponent": 0.5,
+        "mode": "spectrum"
+      },
+      "dataFormat": "tsbuckets",
+      "datasource": "$datasource",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "heatmap": {},
+      "hideZeroBuckets": false,
+      "highlightCards": true,
+      "id": 189,
+      "legend": {
+        "show": false
+      },
+      "links": [],
+      "reverseYBuckets": false,
+      "targets": [
+        {
+          "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)",
+          "format": "heatmap",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{le}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Event Send Time (excluding errors, all workers)",
+      "tooltip": {
+        "show": true,
+        "showHistogram": true
+      },
+      "type": "heatmap",
+      "xAxis": {
+        "show": true
+      },
+      "xBucketNumber": null,
+      "xBucketSize": null,
+      "yAxis": {
+        "decimals": null,
+        "format": "s",
+        "logBase": 2,
+        "max": null,
+        "min": null,
+        "show": true,
+        "splitFactor": null
+      },
+      "yBucketBound": "auto",
+      "yBucketNumber": null,
+      "yBucketSize": null
+    },
     {
       "aliasColors": {},
       "bars": false,
       "dashLength": 10,
       "dashes": false,
       "datasource": "$datasource",
-      "fill": 1,
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 0,
       "fillGradient": 0,
       "gridPos": {
         "h": 9,
         "w": 12,
-        "x": 0,
+        "x": 12,
         "y": 1
       },
       "hiddenSeries": false,
-      "id": 75,
+      "id": 152,
       "legend": {
         "avg": false,
         "current": false,
         "max": false,
         "min": false,
+        "rightSide": false,
         "show": true,
         "total": false,
         "values": false
       },
       "lines": true,
-      "linewidth": 1,
+      "linewidth": 0,
       "links": [],
-      "nullPointMode": "null",
+      "nullPointMode": "connected",
       "options": {
-        "dataLinks": []
+        "alertThreshold": true
       },
       "paceLength": 10,
       "percentage": false,
+      "pluginVersion": "7.3.7",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
-      "seriesOverrides": [],
+      "seriesOverrides": [
+        {
+          "alias": "Avg",
+          "fill": 0,
+          "linewidth": 3
+        },
+        {
+          "alias": "99%",
+          "color": "#C4162A",
+          "fillBelowTo": "90%"
+        },
+        {
+          "alias": "90%",
+          "color": "#FF7383",
+          "fillBelowTo": "75%"
+        },
+        {
+          "alias": "75%",
+          "color": "#FFEE52",
+          "fillBelowTo": "50%"
+        },
+        {
+          "alias": "50%",
+          "color": "#73BF69",
+          "fillBelowTo": "25%"
+        },
+        {
+          "alias": "25%",
+          "color": "#1F60C4",
+          "fillBelowTo": "5%"
+        },
+        {
+          "alias": "5%",
+          "lines": false
+        },
+        {
+          "alias": "Average",
+          "color": "rgb(255, 255, 255)",
+          "lines": true,
+          "linewidth": 3
+        },
+        {
+          "alias": "Events",
+          "color": "#B877D9",
+          "hideTooltip": true,
+          "points": true,
+          "yaxis": 2,
+          "zindex": -3
+        }
+      ],
       "spaceLength": 10,
       "stack": false,
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+          "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
           "format": "time_series",
           "intervalFactor": 1,
-          "legendFormat": "{{job}}-{{index}} ",
+          "legendFormat": "99%",
+          "refId": "D"
+        },
+        {
+          "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "90%",
           "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "75%",
+          "refId": "C"
+        },
+        {
+          "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "50%",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+          "legendFormat": "25%",
+          "refId": "F"
+        },
+        {
+          "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+          "legendFormat": "5%",
+          "refId": "G"
+        },
+        {
+          "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))",
+          "legendFormat": "Average",
+          "refId": "H"
+        },
+        {
+          "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size]))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Events",
+          "refId": "E"
         }
       ],
       "thresholds": [
         {
-          "colorMode": "critical",
-          "fill": true,
+          "colorMode": "warning",
+          "fill": false,
           "line": true,
           "op": "gt",
           "value": 1,
           "yaxis": "left"
+        },
+        {
+          "colorMode": "critical",
+          "fill": false,
+          "line": true,
+          "op": "gt",
+          "value": 2,
+          "yaxis": "left"
         }
       ],
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "CPU usage",
+      "title": "Event Send Time Quantiles (excluding errors, all workers)",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -163,19 +343,19 @@
       "yaxes": [
         {
           "decimals": null,
-          "format": "percentunit",
-          "label": null,
+          "format": "s",
+          "label": "",
           "logBase": 1,
-          "max": "1.5",
+          "max": null,
           "min": "0",
           "show": true
         },
         {
-          "format": "short",
-          "label": null,
+          "format": "hertz",
+          "label": "",
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         }
       ],
@@ -190,37 +370,42 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$datasource",
-      "editable": true,
-      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
       "fill": 1,
       "fillGradient": 0,
-      "grid": {},
       "gridPos": {
         "h": 9,
         "w": 12,
-        "x": 12,
-        "y": 1
+        "x": 0,
+        "y": 10
       },
       "hiddenSeries": false,
-      "id": 33,
+      "id": 75,
       "legend": {
         "avg": false,
         "current": false,
         "max": false,
         "min": false,
-        "show": false,
+        "show": true,
         "total": false,
         "values": false
       },
       "lines": true,
-      "linewidth": 2,
+      "linewidth": 3,
       "links": [],
       "nullPointMode": "null",
       "options": {
-        "dataLinks": []
+        "alertThreshold": true
       },
       "paceLength": 10,
       "percentage": false,
+      "pluginVersion": "7.3.7",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -230,24 +415,32 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size])) without (job,index)",
+          "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
           "format": "time_series",
-          "intervalFactor": 2,
-          "legendFormat": "",
-          "refId": "A",
-          "step": 20,
-          "target": ""
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{job}}-{{index}} ",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 1,
+          "yaxis": "left"
         }
       ],
-      "thresholds": [],
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Events Persisted",
+      "title": "CPU usage",
       "tooltip": {
-        "shared": true,
+        "shared": false,
         "sort": 0,
-        "value_type": "cumulative"
+        "value_type": "individual"
       },
       "type": "graph",
       "xaxis": {
@@ -259,14 +452,17 @@
       },
       "yaxes": [
         {
-          "format": "hertz",
+          "decimals": null,
+          "format": "percentunit",
+          "label": null,
           "logBase": 1,
-          "max": null,
-          "min": null,
+          "max": "1.5",
+          "min": "0",
           "show": true
         },
         {
           "format": "short",
+          "label": null,
           "logBase": 1,
           "max": null,
           "min": null,
@@ -278,76 +474,24 @@
         "alignLevel": null
       }
     },
-    {
-      "cards": {
-        "cardPadding": 0,
-        "cardRound": null
-      },
-      "color": {
-        "cardColor": "#b4ff00",
-        "colorScale": "sqrt",
-        "colorScheme": "interpolateSpectral",
-        "exponent": 0.5,
-        "mode": "spectrum"
-      },
-      "dataFormat": "tsbuckets",
-      "datasource": "$datasource",
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 10
-      },
-      "heatmap": {},
-      "hideZeroBuckets": true,
-      "highlightCards": true,
-      "id": 85,
-      "legend": {
-        "show": false
-      },
-      "links": [],
-      "reverseYBuckets": false,
-      "targets": [
-        {
-          "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
-          "format": "heatmap",
-          "intervalFactor": 1,
-          "legendFormat": "{{le}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Event Send Time",
-      "tooltip": {
-        "show": true,
-        "showHistogram": false
-      },
-      "type": "heatmap",
-      "xAxis": {
-        "show": true
-      },
-      "xBucketNumber": null,
-      "xBucketSize": null,
-      "yAxis": {
-        "decimals": null,
-        "format": "s",
-        "logBase": 2,
-        "max": null,
-        "min": null,
-        "show": true,
-        "splitFactor": null
-      },
-      "yBucketBound": "auto",
-      "yBucketNumber": null,
-      "yBucketSize": null
-    },
     {
       "aliasColors": {},
       "bars": false,
       "dashLength": 10,
       "dashes": false,
       "datasource": "$datasource",
-      "fill": 0,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
       "fillGradient": 0,
+      "grid": {},
       "gridPos": {
         "h": 9,
         "w": 12,
@@ -355,7 +499,7 @@
         "y": 10
       },
       "hiddenSeries": false,
-      "id": 107,
+      "id": 198,
       "legend": {
         "avg": false,
         "current": false,
@@ -366,76 +510,52 @@
         "values": false
       },
       "lines": true,
-      "linewidth": 1,
+      "linewidth": 3,
       "links": [],
       "nullPointMode": "null",
       "options": {
-        "dataLinks": []
+        "alertThreshold": true
       },
       "paceLength": 10,
       "percentage": false,
+      "pluginVersion": "7.3.7",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
-      "repeat": null,
-      "repeatDirection": "h",
-      "seriesOverrides": [
-        {
-          "alias": "mean",
-          "linewidth": 2
-        }
-      ],
+      "seriesOverrides": [],
       "spaceLength": 10,
       "stack": false,
       "steppedLine": false,
       "targets": [
         {
-          "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method))",
+          "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
           "format": "time_series",
           "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "99%",
-          "refId": "A"
-        },
-        {
-          "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "95%",
-          "refId": "B"
-        },
-        {
-          "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "90%",
-          "refId": "C"
-        },
-        {
-          "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "50%",
-          "refId": "D"
+          "intervalFactor": 2,
+          "legendFormat": "{{job}} {{index}}",
+          "refId": "A",
+          "step": 20,
+          "target": ""
         },
         {
-          "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method)",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "mean",
-          "refId": "E"
+          "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+          "hide": true,
+          "interval": "",
+          "legendFormat": "total",
+          "refId": "B"
         }
       ],
       "thresholds": [],
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Event send time quantiles",
+      "title": "Memory",
       "tooltip": {
-        "shared": true,
+        "shared": false,
         "sort": 0,
-        "value_type": "individual"
+        "value_type": "cumulative"
       },
+      "transformations": [],
       "type": "graph",
       "xaxis": {
         "buckets": null,
@@ -446,16 +566,14 @@
       },
       "yaxes": [
         {
-          "format": "s",
-          "label": null,
+          "format": "bytes",
           "logBase": 1,
           "max": null,
-          "min": null,
+          "min": "0",
           "show": true
         },
         {
           "format": "short",
-          "label": null,
           "logBase": 1,
           "max": null,
           "min": null,
@@ -473,16 +591,23 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "$datasource",
-      "fill": 0,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
       "fillGradient": 0,
       "gridPos": {
-        "h": 9,
+        "h": 7,
         "w": 12,
-        "x": 0,
+        "x": 12,
         "y": 19
       },
       "hiddenSeries": false,
-      "id": 118,
+      "id": 37,
       "legend": {
         "avg": false,
         "current": false,
@@ -497,18 +622,20 @@
       "links": [],
       "nullPointMode": "null",
       "options": {
-        "dataLinks": []
+        "alertThreshold": true
       },
       "paceLength": 10,
       "percentage": false,
+      "pluginVersion": "7.3.7",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
-      "repeatDirection": "h",
       "seriesOverrides": [
         {
-          "alias": "mean",
-          "linewidth": 2
+          "alias": "/max$/",
+          "color": "#890F02",
+          "fill": 0,
+          "legend": false
         }
       ],
       "spaceLength": 10,
@@ -516,49 +643,33 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+          "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
           "format": "time_series",
+          "hide": false,
           "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "{{job}}-{{index}} 99%",
-          "refId": "A"
-        },
-        {
-          "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "{{job}}-{{index}} 95%",
-          "refId": "B"
-        },
-        {
-          "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "{{job}}-{{index}} 90%",
-          "refId": "C"
-        },
-        {
-          "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "{{job}}-{{index}} 50%",
-          "refId": "D"
+          "intervalFactor": 2,
+          "legendFormat": "{{job}}-{{index}}",
+          "refId": "A",
+          "step": 20
         },
         {
-          "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",
+          "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
           "format": "time_series",
-          "intervalFactor": 1,
-          "legendFormat": "{{job}}-{{index}} mean",
-          "refId": "E"
+          "hide": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{job}}-{{index}} max",
+          "refId": "B",
+          "step": 20
         }
       ],
       "thresholds": [],
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Event send time quantiles by worker",
+      "title": "Open FDs",
       "tooltip": {
-        "shared": true,
+        "shared": false,
         "sort": 0,
         "value_type": "individual"
       },
@@ -572,14 +683,16 @@
       },
       "yaxes": [
         {
-          "format": "s",
-          "label": null,
+          "decimals": null,
+          "format": "none",
+          "label": "",
           "logBase": 1,
           "max": null,
           "min": null,
           "show": true
         },
         {
+          "decimals": null,
           "format": "short",
           "label": null,
           "logBase": 1,
@@ -600,7 +713,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 28
+        "y": 26
       },
       "id": 54,
       "panels": [
@@ -612,6 +725,13 @@
           "datasource": "$datasource",
           "editable": true,
           "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "grid": {},
@@ -619,7 +739,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 2
+            "y": 25
           },
           "hiddenSeries": false,
           "id": 5,
@@ -637,14 +757,15 @@
             "values": false
           },
           "lines": true,
-          "linewidth": 1,
+          "linewidth": 3,
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -687,15 +808,25 @@
               "line": true,
               "lineColor": "rgba(216, 200, 27, 0.27)",
               "op": "gt",
-              "value": 0.5
+              "value": 0.5,
+              "yaxis": "left"
             },
             {
               "colorMode": "custom",
               "fillColor": "rgba(255, 255, 255, 1)",
               "line": true,
-              "lineColor": "rgba(234, 112, 112, 0.22)",
+              "lineColor": "rgb(87, 6, 16)",
+              "op": "gt",
+              "value": 0.8,
+              "yaxis": "left"
+            },
+            {
+              "colorMode": "critical",
+              "fill": true,
+              "line": true,
               "op": "gt",
-              "value": 0.8
+              "value": 1,
+              "yaxis": "left"
             }
           ],
           "timeFrom": null,
@@ -703,7 +834,7 @@
           "timeShift": null,
           "title": "CPU",
           "tooltip": {
-            "shared": true,
+            "shared": false,
             "sort": 0,
             "value_type": "individual"
           },
@@ -744,16 +875,25 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 2
+            "y": 25
           },
           "hiddenSeries": false,
-          "id": 37,
+          "id": 105,
+          "interval": "",
           "legend": {
             "avg": false,
             "current": false,
@@ -768,51 +908,57 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [
-            {
-              "alias": "/max$/",
-              "color": "#890F02",
-              "fill": 0,
-              "legend": false
-            }
-          ],
+          "seriesOverrides": [],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
               "format": "time_series",
-              "hide": false,
+              "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}}",
+              "legendFormat": "{{job}}-{{index}} 99%",
               "refId": "A",
               "step": 20
             },
             {
-              "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
               "format": "time_series",
-              "hide": true,
-              "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} max",
-              "refId": "B",
-              "step": 20
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} 95%",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} 90%",
+              "refId": "C"
+            },
+            {
+              "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} mean",
+              "refId": "D"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Open FDs",
+          "title": "Reactor tick quantiles",
           "tooltip": {
-            "shared": true,
+            "shared": false,
             "sort": 0,
             "value_type": "individual"
           },
@@ -826,7 +972,7 @@
           },
           "yaxes": [
             {
-              "format": "none",
+              "format": "s",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -839,7 +985,7 @@
               "logBase": 1,
               "max": null,
               "min": null,
-              "show": true
+              "show": false
             }
           ],
           "yaxis": {
@@ -855,6 +1001,13 @@
           "datasource": "$datasource",
           "editable": true,
           "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 0,
           "fillGradient": 0,
           "grid": {},
@@ -862,7 +1015,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 9
+            "y": 32
           },
           "hiddenSeries": false,
           "id": 34,
@@ -880,10 +1033,11 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -895,11 +1049,18 @@
             {
               "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 2,
               "legendFormat": "{{job}} {{index}}",
               "refId": "A",
               "step": 20,
               "target": ""
+            },
+            {
+              "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+              "interval": "",
+              "legendFormat": "total",
+              "refId": "B"
             }
           ],
           "thresholds": [],
@@ -908,10 +1069,11 @@
           "timeShift": null,
           "title": "Memory",
           "tooltip": {
-            "shared": true,
+            "shared": false,
             "sort": 0,
             "value_type": "cumulative"
           },
+          "transformations": [],
           "type": "graph",
           "xaxis": {
             "buckets": null,
@@ -947,18 +1109,23 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 9
+            "y": 32
           },
           "hiddenSeries": false,
-          "id": 105,
-          "interval": "",
+          "id": 49,
           "legend": {
             "avg": false,
             "current": false,
@@ -973,54 +1140,40 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "/^up/",
+              "legend": false,
+              "yaxis": 2
+            }
+          ],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+              "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} 99%",
+              "legendFormat": "{{job}}-{{index}}",
               "refId": "A",
               "step": 20
-            },
-            {
-              "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} 95%",
-              "refId": "B"
-            },
-            {
-              "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} 90%",
-              "refId": "C"
-            },
-            {
-              "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} mean",
-              "refId": "D"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Reactor tick quantiles",
+          "title": "Prometheus scrape time",
           "tooltip": {
             "shared": false,
             "sort": 0,
@@ -1040,15 +1193,16 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
-              "format": "short",
-              "label": null,
+              "decimals": 0,
+              "format": "none",
+              "label": "",
               "logBase": 1,
-              "max": null,
-              "min": null,
+              "max": "0",
+              "min": "-1",
               "show": false
             }
           ],
@@ -1063,13 +1217,20 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 0,
           "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 16
+            "y": 39
           },
           "hiddenSeries": false,
           "id": 53,
@@ -1087,10 +1248,11 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -1113,7 +1275,7 @@
           "timeShift": null,
           "title": "Up",
           "tooltip": {
-            "shared": true,
+            "shared": false,
             "sort": 0,
             "value_type": "individual"
           },
@@ -1154,16 +1316,23 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 16
+            "y": 39
           },
           "hiddenSeries": false,
-          "id": 49,
+          "id": 120,
           "legend": {
             "avg": false,
             "current": false,
@@ -1176,43 +1345,56 @@
           "lines": true,
           "linewidth": 1,
           "links": [],
-          "nullPointMode": "null",
+          "nullPointMode": "null as zero",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
-          "paceLength": 10,
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [
-            {
-              "alias": "/^up/",
-              "legend": false,
-              "yaxis": 2
-            }
-          ],
+          "seriesOverrides": [],
           "spaceLength": 10,
-          "stack": false,
+          "stack": true,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A"
+            },
+            {
+              "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
+              "hide": false,
+              "instant": false,
               "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}}",
-              "refId": "A",
-              "step": 20
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "critical",
+              "fill": true,
+              "line": true,
+              "op": "gt",
+              "value": 1,
+              "yaxis": "left"
             }
           ],
-          "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Prometheus scrape time",
+          "title": "Stacked CPU usage",
           "tooltip": {
-            "shared": true,
+            "shared": false,
             "sort": 0,
             "value_type": "individual"
           },
@@ -1226,21 +1408,20 @@
           },
           "yaxes": [
             {
-              "format": "s",
+              "format": "percentunit",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
-              "decimals": 0,
-              "format": "none",
-              "label": "",
+              "format": "short",
+              "label": null,
               "logBase": 1,
-              "max": "0",
-              "min": "-1",
-              "show": false
+              "max": null,
+              "min": null,
+              "show": true
             }
           ],
           "yaxis": {
@@ -1254,13 +1435,20 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 23
+            "y": 46
           },
           "hiddenSeries": false,
           "id": 136,
@@ -1278,9 +1466,10 @@
           "linewidth": 1,
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -1306,7 +1495,7 @@
           "timeShift": null,
           "title": "Outgoing HTTP request rate",
           "tooltip": {
-            "shared": true,
+            "shared": false,
             "sort": 0,
             "value_type": "individual"
           },
@@ -1340,6 +1529,90 @@
             "align": false,
             "alignLevel": null
           }
+        }
+      ],
+      "repeat": null,
+      "title": "Process info",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 27
+      },
+      "id": 56,
+      "panels": [
+        {
+          "cards": {
+            "cardPadding": -1,
+            "cardRound": 0
+          },
+          "color": {
+            "cardColor": "#b4ff00",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateInferno",
+            "exponent": 0.5,
+            "mode": "spectrum"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 21
+          },
+          "heatmap": {},
+          "hideZeroBuckets": false,
+          "highlightCards": true,
+          "id": 85,
+          "legend": {
+            "show": false
+          },
+          "links": [],
+          "reverseYBuckets": false,
+          "targets": [
+            {
+              "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
+              "format": "heatmap",
+              "intervalFactor": 1,
+              "legendFormat": "{{le}}",
+              "refId": "A"
+            }
+          ],
+          "title": "Event Send Time (Including errors, across all workers)",
+          "tooltip": {
+            "show": true,
+            "showHistogram": true
+          },
+          "type": "heatmap",
+          "xAxis": {
+            "show": true
+          },
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": null,
+            "format": "s",
+            "logBase": 2,
+            "max": null,
+            "min": null,
+            "show": true,
+            "splitFactor": null
+          },
+          "yBucketBound": "auto",
+          "yBucketNumber": null,
+          "yBucketSize": null
         },
         {
           "aliasColors": {},
@@ -1347,79 +1620,74 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "description": "",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
+          "grid": {},
           "gridPos": {
-            "h": 7,
+            "h": 9,
             "w": 12,
             "x": 12,
-            "y": 23
+            "y": 21
           },
           "hiddenSeries": false,
-          "id": 120,
+          "id": 33,
           "legend": {
             "avg": false,
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
           "lines": true,
-          "linewidth": 1,
+          "linewidth": 2,
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
+          "paceLength": 10,
           "percentage": false,
-          "pointradius": 2,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
           "spaceLength": 10,
-          "stack": true,
+          "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
-              "refId": "A"
-            },
-            {
-              "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size])) without (job,index)",
               "format": "time_series",
-              "hide": false,
-              "instant": false,
               "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} {{name}}",
-              "refId": "B"
-            }
-          ],
-          "thresholds": [
-            {
-              "colorMode": "critical",
-              "fill": true,
-              "line": true,
-              "op": "gt",
-              "value": 1,
-              "yaxis": "left"
+              "intervalFactor": 2,
+              "legendFormat": "",
+              "refId": "A",
+              "step": 20,
+              "target": ""
             }
           ],
+          "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Stacked CPU usage",
+          "title": "Events Persisted (all workers)",
           "tooltip": {
-            "shared": false,
+            "shared": true,
             "sort": 0,
-            "value_type": "individual"
+            "value_type": "cumulative"
           },
           "type": "graph",
           "xaxis": {
@@ -1431,8 +1699,7 @@
           },
           "yaxes": [
             {
-              "format": "percentunit",
-              "label": null,
+              "format": "hertz",
               "logBase": 1,
               "max": null,
               "min": null,
@@ -1440,7 +1707,6 @@
             },
             {
               "format": "short",
-              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -1451,23 +1717,7 @@
             "align": false,
             "alignLevel": null
           }
-        }
-      ],
-      "repeat": null,
-      "title": "Process info",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": "${DS_PROMETHEUS}",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 29
-      },
-      "id": 56,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
@@ -1475,13 +1725,21 @@
           "dashes": false,
           "datasource": "$datasource",
           "decimals": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
+          "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 58
+            "y": 30
           },
+          "hiddenSeries": false,
           "id": 40,
           "legend": {
             "avg": false,
@@ -1496,7 +1754,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -1561,13 +1823,21 @@
           "dashes": false,
           "datasource": "$datasource",
           "decimals": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
+          "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 58
+            "y": 30
           },
+          "hiddenSeries": false,
           "id": 46,
           "legend": {
             "avg": false,
@@ -1582,7 +1852,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -1651,13 +1925,21 @@
           "dashes": false,
           "datasource": "$datasource",
           "decimals": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
+          "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 65
+            "y": 37
           },
+          "hiddenSeries": false,
           "id": 44,
           "legend": {
             "alignAsTable": true,
@@ -1675,7 +1957,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -1741,13 +2027,21 @@
           "dashes": false,
           "datasource": "$datasource",
           "decimals": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
+          "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 65
+            "y": 37
           },
+          "hiddenSeries": false,
           "id": 45,
           "legend": {
             "alignAsTable": true,
@@ -1765,7 +2059,11 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -1823,52 +2121,35 @@
             "align": false,
             "alignLevel": null
           }
-        }
-      ],
-      "repeat": null,
-      "title": "Event persist rates",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": "${DS_PROMETHEUS}",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 30
-      },
-      "id": 57,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "decimals": null,
-          "editable": true,
-          "error": false,
-          "fill": 2,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 0,
           "fillGradient": 0,
-          "grid": {},
           "gridPos": {
-            "h": 8,
+            "h": 9,
             "w": 12,
             "x": 0,
-            "y": 31
+            "y": 44
           },
           "hiddenSeries": false,
-          "id": 4,
+          "id": 118,
           "legend": {
-            "alignAsTable": true,
             "avg": false,
             "current": false,
-            "hideEmpty": false,
-            "hideZero": true,
             "max": false,
             "min": false,
-            "rightSide": false,
             "show": true,
             "total": false,
             "values": false
@@ -1878,50 +2159,212 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
+          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [],
+          "repeatDirection": "h",
+          "seriesOverrides": [
+            {
+              "alias": "mean",
+              "linewidth": 2
+            }
+          ],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
               "format": "time_series",
               "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
-              "refId": "A",
-              "step": 20
-            }
-          ],
-          "thresholds": [
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} 99%",
+              "refId": "A"
+            },
             {
-              "colorMode": "custom",
-              "fill": true,
-              "fillColor": "rgba(216, 200, 27, 0.27)",
-              "op": "gt",
-              "value": 100
+              "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} 95%",
+              "refId": "B"
             },
             {
-              "colorMode": "custom",
-              "fill": true,
-              "fillColor": "rgba(234, 112, 112, 0.22)",
-              "op": "gt",
-              "value": 250
-            }
-          ],
+              "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} 90%",
+              "refId": "C"
+            },
+            {
+              "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} 50%",
+              "refId": "D"
+            },
+            {
+              "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} mean",
+              "refId": "E"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Event send time quantiles by worker",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "repeat": null,
+      "title": "Event persistence",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 28
+      },
+      "id": 57,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": null,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 2,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 31
+          },
+          "hiddenSeries": false,
+          "id": 4,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(216, 200, 27, 0.27)",
+              "op": "gt",
+              "value": 100,
+              "yaxis": "left"
+            },
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(234, 112, 112, 0.22)",
+              "op": "gt",
+              "value": 250,
+              "yaxis": "left"
+            }
+          ],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
           "title": "Request Count by arrival time",
           "tooltip": {
             "shared": false,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -1961,6 +2404,13 @@
           "datasource": "$datasource",
           "editable": true,
           "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "grid": {},
@@ -1986,9 +2436,10 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -2014,7 +2465,7 @@
           "title": "Top 10 Request Counts",
           "tooltip": {
             "shared": false,
-            "sort": 0,
+            "sort": 2,
             "value_type": "cumulative"
           },
           "type": "graph",
@@ -2055,6 +2506,13 @@
           "decimals": null,
           "editable": true,
           "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 2,
           "fillGradient": 0,
           "grid": {},
@@ -2084,9 +2542,10 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -2129,7 +2588,7 @@
           "title": "Total CPU Usage by Endpoint",
           "tooltip": {
             "shared": false,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -2170,7 +2629,14 @@
           "decimals": null,
           "editable": true,
           "error": false,
-          "fill": 2,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 0,
           "fillGradient": 0,
           "grid": {},
           "gridPos": {
@@ -2199,9 +2665,10 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -2214,7 +2681,7 @@
               "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
               "interval": "",
-              "intervalFactor": 2,
+              "intervalFactor": 1,
               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
               "refId": "A",
               "step": 20
@@ -2226,14 +2693,16 @@
               "fill": true,
               "fillColor": "rgba(216, 200, 27, 0.27)",
               "op": "gt",
-              "value": 100
+              "value": 100,
+              "yaxis": "left"
             },
             {
               "colorMode": "custom",
               "fill": true,
               "fillColor": "rgba(234, 112, 112, 0.22)",
               "op": "gt",
-              "value": 250
+              "value": 250,
+              "yaxis": "left"
             }
           ],
           "timeFrom": null,
@@ -2242,7 +2711,7 @@
           "title": "Average CPU Usage by Endpoint",
           "tooltip": {
             "shared": false,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -2282,6 +2751,13 @@
           "datasource": "$datasource",
           "editable": true,
           "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "grid": {},
@@ -2310,9 +2786,10 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -2325,7 +2802,7 @@
               "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
               "interval": "",
-              "intervalFactor": 2,
+              "intervalFactor": 1,
               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
               "refId": "A",
               "step": 20
@@ -2338,7 +2815,7 @@
           "title": "DB Usage by endpoint",
           "tooltip": {
             "shared": false,
-            "sort": 0,
+            "sort": 2,
             "value_type": "cumulative"
           },
           "type": "graph",
@@ -2379,6 +2856,13 @@
           "decimals": null,
           "editable": true,
           "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 2,
           "fillGradient": 0,
           "grid": {},
@@ -2408,9 +2892,10 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -2424,7 +2909,7 @@
               "format": "time_series",
               "hide": false,
               "interval": "",
-              "intervalFactor": 2,
+              "intervalFactor": 1,
               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
               "refId": "A",
               "step": 20
@@ -2437,7 +2922,7 @@
           "title": "Non-sync avg response time",
           "tooltip": {
             "shared": false,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -2475,6 +2960,13 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
@@ -2499,13 +2991,21 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "Total",
+              "color": "rgb(255, 255, 255)",
+              "fill": 0,
+              "linewidth": 3
+            }
+          ],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
@@ -2517,6 +3017,12 @@
               "intervalFactor": 1,
               "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
               "refId": "A"
+            },
+            {
+              "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+              "interval": "",
+              "legendFormat": "Total",
+              "refId": "B"
             }
           ],
           "thresholds": [],
@@ -2526,7 +3032,7 @@
           "title": "Requests in flight",
           "tooltip": {
             "shared": false,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -2572,7 +3078,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 31
+        "y": 29
       },
       "id": 97,
       "panels": [
@@ -2582,6 +3088,13 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
@@ -2605,11 +3118,9 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -2674,6 +3185,13 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
@@ -2697,11 +3215,9 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -2717,12 +3233,6 @@
               "intervalFactor": 1,
               "legendFormat": "{{job}}-{{index}} {{name}}",
               "refId": "A"
-            },
-            {
-              "expr": "",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "refId": "B"
             }
           ],
           "thresholds": [],
@@ -2731,7 +3241,7 @@
           "timeShift": null,
           "title": "DB usage by background jobs (including scheduling time)",
           "tooltip": {
-            "shared": true,
+            "shared": false,
             "sort": 0,
             "value_type": "individual"
           },
@@ -2772,6 +3282,13 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
@@ -2794,10 +3311,8 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -2864,7 +3379,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 32
+        "y": 30
       },
       "id": 81,
       "panels": [
@@ -2874,6 +3389,13 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
@@ -2898,10 +3420,11 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -2970,6 +3493,13 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
@@ -2994,10 +3524,11 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -3068,6 +3599,13 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
@@ -3092,10 +3630,11 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -3167,6 +3706,13 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
@@ -3191,10 +3737,11 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -3258,18 +3805,25 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "$datasource",
-          "description": "Number of events queued up on the master process for processing by the federation sender",
+          "datasource": "${DS_PROMETHEUS}",
+          "description": "The number of events in the in-memory queues ",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 9,
+            "h": 8,
             "w": 12,
             "x": 0,
             "y": 24
           },
           "hiddenSeries": false,
-          "id": 140,
+          "id": 142,
           "legend": {
             "avg": false,
             "current": false,
@@ -3281,14 +3835,13 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
-          "paceLength": 10,
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -3297,64 +3850,23 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
-              "format": "time_series",
+              "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
               "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "presence changed",
+              "legendFormat": "pending PDUs {{job}}-{{index}}",
               "refId": "A"
             },
             {
-              "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
-              "format": "time_series",
-              "hide": false,
+              "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
               "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "presence map",
+              "legendFormat": "pending EDUs {{job}}-{{index}}",
               "refId": "B"
-            },
-            {
-              "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "presence destinations",
-              "refId": "E"
-            },
-            {
-              "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "keyed edus",
-              "refId": "C"
-            },
-            {
-              "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "other edus",
-              "refId": "D"
-            },
-            {
-              "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
-              "format": "time_series",
-              "hide": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "stream positions",
-              "refId": "F"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Outgoing EDU queues on master",
+          "title": "In-memory federation transmission queues",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -3370,8 +3882,8 @@
           },
           "yaxes": [
             {
-              "format": "none",
-              "label": null,
+              "format": "short",
+              "label": "events",
               "logBase": 1,
               "max": null,
               "min": "0",
@@ -3379,7 +3891,7 @@
             },
             {
               "format": "short",
-              "label": null,
+              "label": "",
               "logBase": 1,
               "max": null,
               "min": null,
@@ -3396,18 +3908,25 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
-          "description": "The number of events in the in-memory queues ",
+          "datasource": "$datasource",
+          "description": "Number of events queued up on the master process for processing by the federation sender",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 8,
+            "h": 9,
             "w": 12,
             "x": 12,
             "y": 24
           },
           "hiddenSeries": false,
-          "id": 142,
+          "id": 140,
           "legend": {
             "avg": false,
             "current": false,
@@ -3419,12 +3938,15 @@
           },
           "lines": true,
           "linewidth": 1,
+          "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
+          "paceLength": 10,
           "percentage": false,
-          "pointradius": 2,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -3433,23 +3955,64 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
               "interval": "",
-              "legendFormat": "pending PDUs {{job}}-{{index}}",
+              "intervalFactor": 1,
+              "legendFormat": "presence changed",
               "refId": "A"
             },
             {
-              "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
               "interval": "",
-              "legendFormat": "pending EDUs {{job}}-{{index}}",
+              "intervalFactor": 1,
+              "legendFormat": "presence map",
               "refId": "B"
+            },
+            {
+              "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "presence destinations",
+              "refId": "E"
+            },
+            {
+              "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "keyed edus",
+              "refId": "C"
+            },
+            {
+              "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "other edus",
+              "refId": "D"
+            },
+            {
+              "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "stream positions",
+              "refId": "F"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "In-memory federation transmission queues",
+          "title": "Outgoing EDU queues on master",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -3465,18 +4028,16 @@
           },
           "yaxes": [
             {
-              "$$hashKey": "object:317",
-              "format": "short",
-              "label": "events",
+              "format": "none",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": "0",
               "show": true
             },
             {
-              "$$hashKey": "object:318",
               "format": "short",
-              "label": "",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -3487,115 +4048,273 @@
             "align": false,
             "alignLevel": null
           }
-        }
-      ],
-      "title": "Federation",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": "${DS_PROMETHEUS}",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 33
-      },
-      "id": 60,
-      "panels": [
+        },
+        {
+          "cards": {
+            "cardPadding": -1,
+            "cardRound": null
+          },
+          "color": {
+            "cardColor": "#b4ff00",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateInferno",
+            "exponent": 0.5,
+            "min": 0,
+            "mode": "spectrum"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 32
+          },
+          "heatmap": {},
+          "hideZeroBuckets": false,
+          "highlightCards": true,
+          "id": 166,
+          "legend": {
+            "show": false
+          },
+          "links": [],
+          "reverseYBuckets": false,
+          "targets": [
+            {
+              "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)",
+              "format": "heatmap",
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ le }}",
+              "refId": "A"
+            }
+          ],
+          "title": "Federation send PDU lag",
+          "tooltip": {
+            "show": true,
+            "showHistogram": true
+          },
+          "tooltipDecimals": 2,
+          "type": "heatmap",
+          "xAxis": {
+            "show": true
+          },
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": 0,
+            "format": "s",
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true,
+            "splitFactor": null
+          },
+          "yBucketBound": "auto",
+          "yBucketNumber": null,
+          "yBucketSize": null
+        },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "fill": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 0,
           "fillGradient": 0,
           "gridPos": {
-            "h": 8,
+            "h": 9,
             "w": 12,
-            "x": 0,
-            "y": 34
+            "x": 12,
+            "y": 33
           },
           "hiddenSeries": false,
-          "id": 51,
+          "id": 162,
           "legend": {
             "avg": false,
             "current": false,
             "max": false,
             "min": false,
+            "rightSide": false,
             "show": true,
             "total": false,
             "values": false
           },
           "lines": true,
-          "linewidth": 1,
+          "linewidth": 0,
           "links": [],
-          "nullPointMode": "null",
+          "nullPointMode": "connected",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "Avg",
+              "fill": 0,
+              "linewidth": 3
+            },
+            {
+              "alias": "99%",
+              "color": "#C4162A",
+              "fillBelowTo": "90%"
+            },
+            {
+              "alias": "90%",
+              "color": "#FF7383",
+              "fillBelowTo": "75%"
+            },
+            {
+              "alias": "75%",
+              "color": "#FFEE52",
+              "fillBelowTo": "50%"
+            },
+            {
+              "alias": "50%",
+              "color": "#73BF69",
+              "fillBelowTo": "25%"
+            },
+            {
+              "alias": "25%",
+              "color": "#1F60C4",
+              "fillBelowTo": "5%"
+            },
+            {
+              "alias": "5%",
+              "lines": false
+            },
+            {
+              "alias": "Average",
+              "color": "rgb(255, 255, 255)",
+              "lines": true,
+              "linewidth": 3
+            }
+          ],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_http_httppusher_http_pushes_processed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed + synapse_http_httppusher_http_pushes_processed) > 0",
+              "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
               "format": "time_series",
               "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "processed {{job}}",
-              "refId": "A",
-              "step": 20
+              "intervalFactor": 1,
+              "legendFormat": "99%",
+              "refId": "D"
             },
             {
-              "expr": "rate(synapse_http_httppusher_http_pushes_failed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed + synapse_http_httppusher_http_pushes_processed) > 0",
+              "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
               "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "failed {{job}}",
-              "refId": "B",
-              "step": 20
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeRegions": [],
-          "timeShift": null,
-          "title": "HTTP Push rate",
-          "tooltip": {
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "type": "graph",
-          "xaxis": {
-            "buckets": null,
-            "mode": "time",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "90%",
+              "refId": "A"
+            },
+            {
+              "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "75%",
+              "refId": "C"
+            },
+            {
+              "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "50%",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+              "interval": "",
+              "legendFormat": "25%",
+              "refId": "F"
+            },
+            {
+              "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+              "interval": "",
+              "legendFormat": "5%",
+              "refId": "G"
+            },
+            {
+              "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+              "interval": "",
+              "legendFormat": "Average",
+              "refId": "H"
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "warning",
+              "fill": false,
+              "line": true,
+              "op": "gt",
+              "value": 0.25,
+              "yaxis": "left"
+            },
+            {
+              "colorMode": "critical",
+              "fill": false,
+              "line": true,
+              "op": "gt",
+              "value": 1,
+              "yaxis": "left"
+            }
+          ],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Federation send PDU lag quantiles",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
             "name": null,
             "show": true,
             "values": []
           },
           "yaxes": [
             {
-              "format": "hertz",
-              "label": null,
+              "decimals": null,
+              "format": "s",
+              "label": "",
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
-              "format": "short",
-              "label": null,
+              "format": "hertz",
+              "label": "",
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             }
           ],
@@ -3604,27 +4323,107 @@
             "alignLevel": null
           }
         },
+        {
+          "cards": {
+            "cardPadding": -1,
+            "cardRound": null
+          },
+          "color": {
+            "cardColor": "#b4ff00",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateInferno",
+            "exponent": 0.5,
+            "min": 0,
+            "mode": "spectrum"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 41
+          },
+          "heatmap": {},
+          "hideZeroBuckets": false,
+          "highlightCards": true,
+          "id": 164,
+          "legend": {
+            "show": false
+          },
+          "links": [],
+          "reverseYBuckets": false,
+          "targets": [
+            {
+              "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)",
+              "format": "heatmap",
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{ le }}",
+              "refId": "A"
+            }
+          ],
+          "title": "Handle inbound PDU time",
+          "tooltip": {
+            "show": true,
+            "showHistogram": true
+          },
+          "tooltipDecimals": 2,
+          "type": "heatmap",
+          "xAxis": {
+            "show": true
+          },
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": 0,
+            "format": "s",
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true,
+            "splitFactor": null
+          },
+          "yBucketBound": "auto",
+          "yBucketNumber": null,
+          "yBucketSize": null
+        },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "description": "",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
+          "grid": {},
           "gridPos": {
-            "h": 8,
+            "h": 9,
             "w": 12,
             "x": 12,
-            "y": 34
+            "y": 42
           },
           "hiddenSeries": false,
-          "id": 134,
+          "id": 203,
           "legend": {
             "avg": false,
             "current": false,
-            "hideZero": false,
             "max": false,
             "min": false,
             "show": true,
@@ -3632,13 +4431,16 @@
             "values": false
           },
           "lines": true,
-          "linewidth": 1,
+          "linewidth": 2,
+          "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
+          "paceLength": 10,
           "percentage": false,
-          "pointradius": 2,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -3647,20 +4449,25 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})",
-              "legendFormat": "{{kind}} {{app_id}}",
-              "refId": "A"
+              "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "rss {{index}}",
+              "refId": "A",
+              "step": 4
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Active pusher instances by app",
+          "title": "Age of oldest event in staging area",
           "tooltip": {
-            "shared": false,
-            "sort": 2,
-            "value_type": "individual"
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
           },
           "type": "graph",
           "xaxis": {
@@ -3672,11 +4479,11 @@
           },
           "yaxes": [
             {
-              "format": "short",
+              "format": "ms",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": 0,
               "show": true
             },
             {
@@ -3692,39 +4499,33 @@
             "align": false,
             "alignLevel": null
           }
-        }
-      ],
-      "repeat": null,
-      "title": "Pushes",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": "${DS_PROMETHEUS}",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 34
-      },
-      "id": 58,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
+          "grid": {},
           "gridPos": {
-            "h": 7,
+            "h": 9,
             "w": 12,
             "x": 0,
-            "y": 79
+            "y": 50
           },
           "hiddenSeries": false,
-          "id": 48,
+          "id": 202,
           "legend": {
             "avg": false,
             "current": false,
@@ -3735,14 +4536,15 @@
             "values": false
           },
           "lines": true,
-          "linewidth": 1,
+          "linewidth": 2,
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -3752,23 +4554,25 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
+              "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=\"$job\",index=~\"$index\",instance=\"$instance\"}",
               "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}}",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "rss {{index}}",
               "refId": "A",
-              "step": 20
+              "step": 4
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Avg time waiting for db conn",
+          "title": "Number of events in federation staging area",
           "tooltip": {
+            "msResolution": false,
             "shared": true,
             "sort": 0,
-            "value_type": "individual"
+            "value_type": "cumulative"
           },
           "type": "graph",
           "xaxis": {
@@ -3780,12 +4584,11 @@
           },
           "yaxes": [
             {
-              "decimals": null,
-              "format": "s",
-              "label": "",
+              "format": "none",
+              "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": 0,
               "show": true
             },
             {
@@ -3794,7 +4597,7 @@
               "logBase": 1,
               "max": null,
               "min": null,
-              "show": false
+              "show": true
             }
           ],
           "yaxis": {
@@ -3807,20 +4610,24 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "$datasource",
-          "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
+          "datasource": "${DS_PROMETHEUS}",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 7,
+            "h": 8,
             "w": 12,
             "x": 12,
-            "y": 79
+            "y": 51
           },
           "hiddenSeries": false,
-          "id": 104,
+          "id": 205,
           "legend": {
-            "alignAsTable": true,
             "avg": false,
             "current": false,
             "max": false,
@@ -3831,14 +4638,13 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
-          "paceLength": 10,
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -3847,44 +4653,19 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
-              "format": "time_series",
-              "hide": false,
-              "intervalFactor": 1,
-              "legendFormat": "{{job}} {{index}} 99%",
-              "refId": "A",
-              "step": 20
-            },
-            {
-              "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}} {{index}} 95%",
-              "refId": "B"
-            },
-            {
-              "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}} {{index}} 90%",
-              "refId": "C"
-            },
-            {
-              "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
-              "format": "time_series",
+              "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))",
               "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}} {{index}} mean",
-              "refId": "D"
+              "legendFormat": "soft-failed events",
+              "refId": "A"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Db scheduling time quantiles",
+          "title": "Soft-failed event rate",
           "tooltip": {
-            "shared": false,
+            "shared": true,
             "sort": 0,
             "value_type": "individual"
           },
@@ -3898,12 +4679,11 @@
           },
           "yaxes": [
             {
-              "decimals": null,
-              "format": "s",
-              "label": "",
+              "format": "hertz",
+              "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
@@ -3919,31 +4699,48 @@
             "align": false,
             "alignLevel": null
           }
-        },
+        }
+      ],
+      "title": "Federation",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 31
+      },
+      "id": 60,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "editable": true,
-          "error": false,
-          "fill": 0,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
           "fillGradient": 0,
-          "grid": {},
           "gridPos": {
-            "h": 7,
+            "h": 8,
             "w": 12,
             "x": 0,
-            "y": 86
+            "y": 34
           },
           "hiddenSeries": false,
-          "id": 10,
+          "id": 51,
           "legend": {
             "avg": false,
             "current": false,
-            "hideEmpty": true,
-            "hideZero": true,
             "max": false,
             "min": false,
             "show": true,
@@ -3951,14 +4748,12 @@
             "values": false
           },
           "lines": true,
-          "linewidth": 2,
+          "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -3968,24 +4763,32 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "topk(10, rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "expr": "rate(synapse_http_httppusher_http_pushes_processed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed + synapse_http_httppusher_http_pushes_processed) > 0",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} {{desc}}",
+              "legendFormat": "processed {{job}}",
               "refId": "A",
               "step": 20
+            },
+            {
+              "expr": "rate(synapse_http_httppusher_http_pushes_failed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed + synapse_http_httppusher_http_pushes_processed) > 0",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "failed {{job}}",
+              "refId": "B",
+              "step": 20
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Top DB transactions by txn rate",
+          "title": "HTTP Push rate",
           "tooltip": {
-            "shared": false,
+            "shared": true,
             "sort": 0,
-            "value_type": "cumulative"
+            "value_type": "individual"
           },
           "type": "graph",
           "xaxis": {
@@ -3998,13 +4801,15 @@
           "yaxes": [
             {
               "format": "hertz",
+              "label": null,
               "logBase": 1,
               "max": null,
-              "min": 0,
+              "min": null,
               "show": true
             },
             {
               "format": "short",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -4022,24 +4827,28 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "editable": true,
-          "error": false,
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
-          "grid": {},
           "gridPos": {
-            "h": 7,
+            "h": 8,
             "w": 12,
             "x": 12,
-            "y": 86
+            "y": 34
           },
           "hiddenSeries": false,
-          "id": 11,
+          "id": 134,
           "legend": {
             "avg": false,
             "current": false,
-            "hideEmpty": true,
-            "hideZero": true,
+            "hideZero": false,
             "max": false,
             "min": false,
             "show": true,
@@ -4048,14 +4857,10 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
-          "paceLength": 10,
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.1.3",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -4064,25 +4869,20 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
-              "format": "time_series",
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} {{desc}}",
-              "refId": "A",
-              "step": 20
+              "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})",
+              "legendFormat": "{{kind}} {{app_id}}",
+              "refId": "A"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Top DB transactions by total txn time",
+          "title": "Active pusher instances by app",
           "tooltip": {
             "shared": false,
-            "sort": 0,
-            "value_type": "cumulative"
+            "sort": 2,
+            "value_type": "individual"
           },
           "type": "graph",
           "xaxis": {
@@ -4094,7 +4894,8 @@
           },
           "yaxes": [
             {
-              "format": "percentunit",
+              "format": "short",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -4102,6 +4903,7 @@
             },
             {
               "format": "short",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -4115,7 +4917,7 @@
         }
       ],
       "repeat": null,
-      "title": "Database",
+      "title": "Pushes",
       "type": "row"
     },
     {
@@ -4125,9 +4927,9 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 35
+        "y": 32
       },
-      "id": 59,
+      "id": 58,
       "panels": [
         {
           "aliasColors": {},
@@ -4135,21 +4937,24 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "editable": true,
-          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
-          "grid": {},
           "gridPos": {
-            "h": 13,
+            "h": 7,
             "w": 12,
             "x": 0,
-            "y": 80
+            "y": 33
           },
           "hiddenSeries": false,
-          "id": 12,
+          "id": 48,
           "legend": {
-            "alignAsTable": true,
             "avg": false,
             "current": false,
             "max": false,
@@ -4159,14 +4964,15 @@
             "values": false
           },
           "lines": true,
-          "linewidth": 2,
+          "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4176,11 +4982,10 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])",
+              "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
               "format": "time_series",
-              "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "legendFormat": "{{job}}-{{index}}",
               "refId": "A",
               "step": 20
             }
@@ -4189,11 +4994,11 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Total CPU Usage by Block",
+          "title": "Avg time waiting for db conn",
           "tooltip": {
-            "shared": false,
+            "shared": true,
             "sort": 0,
-            "value_type": "cumulative"
+            "value_type": "individual"
           },
           "type": "graph",
           "xaxis": {
@@ -4205,18 +5010,21 @@
           },
           "yaxes": [
             {
-              "format": "percentunit",
+              "decimals": null,
+              "format": "s",
+              "label": "",
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
               "format": "short",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
-              "show": true
+              "show": false
             }
           ],
           "yaxis": {
@@ -4230,19 +5038,24 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "editable": true,
-          "error": false,
+          "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
-          "grid": {},
           "gridPos": {
-            "h": 13,
+            "h": 7,
             "w": 12,
             "x": 12,
-            "y": 80
+            "y": 33
           },
           "hiddenSeries": false,
-          "id": 26,
+          "id": 104,
           "legend": {
             "alignAsTable": true,
             "avg": false,
@@ -4254,14 +5067,15 @@
             "values": false
           },
           "lines": true,
-          "linewidth": 2,
+          "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4271,24 +5085,46 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])) / rate(synapse_util_metrics_block_count[$bucket_size])",
+              "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
               "format": "time_series",
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} {{index}} 99%",
               "refId": "A",
               "step": 20
+            },
+            {
+              "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} {{index}} 95%",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} {{index}} 90%",
+              "refId": "C"
+            },
+            {
+              "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} {{index}} mean",
+              "refId": "D"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Average CPU Time per Block",
+          "title": "Db scheduling time quantiles",
           "tooltip": {
             "shared": false,
             "sort": 0,
-            "value_type": "cumulative"
+            "value_type": "individual"
           },
           "type": "graph",
           "xaxis": {
@@ -4300,18 +5136,21 @@
           },
           "yaxes": [
             {
-              "format": "ms",
+              "decimals": null,
+              "format": "s",
+              "label": "",
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
               "format": "short",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
-              "show": true
+              "show": false
             }
           ],
           "yaxis": {
@@ -4327,21 +5166,29 @@
           "datasource": "$datasource",
           "editable": true,
           "error": false,
-          "fill": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 0,
           "fillGradient": 0,
           "grid": {},
           "gridPos": {
-            "h": 13,
+            "h": 7,
             "w": 12,
             "x": 0,
-            "y": 93
+            "y": 40
           },
           "hiddenSeries": false,
-          "id": 13,
+          "id": 10,
           "legend": {
-            "alignAsTable": true,
             "avg": false,
             "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
             "max": false,
             "min": false,
             "show": true,
@@ -4353,10 +5200,11 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4366,11 +5214,11 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "topk(10, rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "{{job}} {{block_name}}",
+              "legendFormat": "{{job}}-{{index}} {{desc}}",
               "refId": "A",
               "step": 20
             }
@@ -4379,7 +5227,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Total DB Usage by Block",
+          "title": "Top DB transactions by txn rate",
           "tooltip": {
             "shared": false,
             "sort": 0,
@@ -4395,7 +5243,7 @@
           },
           "yaxes": [
             {
-              "format": "percentunit",
+              "format": "hertz",
               "logBase": 1,
               "max": null,
               "min": 0,
@@ -4420,24 +5268,31 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "description": "The time each database transaction takes to execute, on average, broken down by metrics block.",
           "editable": true,
           "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "grid": {},
           "gridPos": {
-            "h": 13,
+            "h": 7,
             "w": 12,
             "x": 12,
-            "y": 93
+            "y": 40
           },
           "hiddenSeries": false,
-          "id": 27,
+          "id": 11,
           "legend": {
-            "alignAsTable": true,
             "avg": false,
             "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
             "max": false,
             "min": false,
             "show": true,
@@ -4445,14 +5300,15 @@
             "values": false
           },
           "lines": true,
-          "linewidth": 2,
+          "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4462,11 +5318,12 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
+              "instant": false,
               "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{desc}}",
               "refId": "A",
               "step": 20
             }
@@ -4475,7 +5332,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Average Database Transaction time, by Block",
+          "title": "DB transactions by total txn time",
           "tooltip": {
             "shared": false,
             "sort": 0,
@@ -4491,7 +5348,7 @@
           },
           "yaxes": [
             {
-              "format": "ms",
+              "format": "percentunit",
               "logBase": 1,
               "max": null,
               "min": null,
@@ -4518,35 +5375,45 @@
           "datasource": "$datasource",
           "editable": true,
           "error": false,
-          "fill": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 0,
           "fillGradient": 0,
           "grid": {},
           "gridPos": {
-            "h": 13,
+            "h": 7,
             "w": 12,
             "x": 0,
-            "y": 106
+            "y": 47
           },
           "hiddenSeries": false,
-          "id": 28,
+          "id": 180,
           "legend": {
             "avg": false,
             "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
             "max": false,
             "min": false,
-            "show": false,
+            "show": true,
             "total": false,
             "values": false
           },
           "lines": true,
-          "linewidth": 2,
+          "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": false
           },
           "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4556,11 +5423,12 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
+              "instant": false,
               "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{desc}}",
               "refId": "A",
               "step": 20
             }
@@ -4569,7 +5437,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Average Transactions per Block",
+          "title": "Average DB txn time",
           "tooltip": {
             "shared": false,
             "sort": 0,
@@ -4585,7 +5453,7 @@
           },
           "yaxes": [
             {
-              "format": "none",
+              "format": "s",
               "logBase": 1,
               "max": null,
               "min": null,
@@ -4610,37 +5478,41 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "editable": true,
-          "error": false,
-          "fill": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 6,
           "fillGradient": 0,
-          "grid": {},
           "gridPos": {
-            "h": 13,
+            "h": 9,
             "w": 12,
             "x": 12,
-            "y": 106
+            "y": 47
           },
           "hiddenSeries": false,
-          "id": 25,
+          "id": 200,
           "legend": {
             "avg": false,
             "current": false,
             "max": false,
             "min": false,
-            "show": false,
+            "show": true,
             "total": false,
             "values": false
           },
           "lines": true,
-          "linewidth": 2,
+          "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
-          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4650,24 +5522,43 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_util_metrics_block_time_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count[$bucket_size])",
+              "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
               "format": "time_series",
-              "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} {{block_name}}",
-              "refId": "A",
-              "step": 20
+              "intervalFactor": 1,
+              "legendFormat": "99%",
+              "refId": "D"
+            },
+            {
+              "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "90%",
+              "refId": "A"
+            },
+            {
+              "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "75%",
+              "refId": "C"
+            },
+            {
+              "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "50%",
+              "refId": "B"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Average Wallclock Time per Block",
+          "title": "Time waiting for DB connection quantiles",
           "tooltip": {
-            "shared": false,
-            "sort": 0,
-            "value_type": "cumulative"
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
           },
           "type": "graph",
           "xaxis": {
@@ -4679,18 +5570,21 @@
           },
           "yaxes": [
             {
-              "format": "ms",
+              "decimals": null,
+              "format": "s",
+              "label": "",
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
               "format": "short",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
-              "show": true
+              "show": false
             }
           ],
           "yaxis": {
@@ -4700,7 +5594,7 @@
         }
       ],
       "repeat": null,
-      "title": "Per-block metrics",
+      "title": "Database",
       "type": "row"
     },
     {
@@ -4710,9 +5604,9 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 36
+        "y": 33
       },
-      "id": 61,
+      "id": 59,
       "panels": [
         {
           "aliasColors": {},
@@ -4720,26 +5614,30 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "decimals": 2,
           "editable": true,
           "error": false,
-          "fill": 0,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
           "fillGradient": 0,
           "grid": {},
           "gridPos": {
-            "h": 10,
+            "h": 13,
             "w": 12,
             "x": 0,
-            "y": 37
+            "y": 9
           },
           "hiddenSeries": false,
-          "id": 1,
+          "id": 12,
           "legend": {
             "alignAsTable": true,
             "avg": false,
             "current": false,
-            "hideEmpty": true,
-            "hideZero": false,
             "max": false,
             "min": false,
             "show": true,
@@ -4750,10 +5648,9 @@
           "linewidth": 2,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
+          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4763,10 +5660,11 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "expr": "rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
               "refId": "A",
               "step": 20
             }
@@ -4775,11 +5673,10 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Cache Hit Ratio",
+          "title": "Total CPU Usage by Block",
           "tooltip": {
-            "msResolution": true,
-            "shared": false,
-            "sort": 0,
+            "shared": true,
+            "sort": 2,
             "value_type": "cumulative"
           },
           "type": "graph",
@@ -4792,12 +5689,10 @@
           },
           "yaxes": [
             {
-              "decimals": null,
               "format": "percentunit",
-              "label": "",
               "logBase": 1,
-              "max": "1",
-              "min": 0,
+              "max": null,
+              "min": null,
               "show": true
             },
             {
@@ -4805,7 +5700,7 @@
               "logBase": 1,
               "max": null,
               "min": null,
-              "show": false
+              "show": true
             }
           ],
           "yaxis": {
@@ -4821,22 +5716,28 @@
           "datasource": "$datasource",
           "editable": true,
           "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "grid": {},
           "gridPos": {
-            "h": 10,
+            "h": 13,
             "w": 12,
             "x": 12,
-            "y": 37
+            "y": 9
           },
           "hiddenSeries": false,
-          "id": 8,
+          "id": 26,
           "legend": {
             "alignAsTable": true,
             "avg": false,
             "current": false,
-            "hideZero": false,
             "max": false,
             "min": false,
             "show": true,
@@ -4846,11 +5747,10 @@
           "lines": true,
           "linewidth": 2,
           "links": [],
-          "nullPointMode": "connected",
-          "options": {
-            "dataLinks": []
-          },
+          "nullPointMode": "null",
+          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4860,12 +5760,11 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "synapse_util_caches_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])) / rate(synapse_util_metrics_block_count[$bucket_size])",
               "format": "time_series",
-              "hide": false,
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
               "refId": "A",
               "step": 20
             }
@@ -4874,10 +5773,10 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Cache Size",
+          "title": "Average CPU Time per Block",
           "tooltip": {
-            "shared": false,
-            "sort": 0,
+            "shared": true,
+            "sort": 2,
             "value_type": "cumulative"
           },
           "type": "graph",
@@ -4890,10 +5789,10 @@
           },
           "yaxes": [
             {
-              "format": "short",
+              "format": "ms",
               "logBase": 1,
               "max": null,
-              "min": 0,
+              "min": null,
               "show": true
             },
             {
@@ -4917,22 +5816,28 @@
           "datasource": "$datasource",
           "editable": true,
           "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "grid": {},
           "gridPos": {
-            "h": 10,
+            "h": 13,
             "w": 12,
             "x": 0,
-            "y": 47
+            "y": 22
           },
           "hiddenSeries": false,
-          "id": 38,
+          "id": 13,
           "legend": {
             "alignAsTable": true,
             "avg": false,
             "current": false,
-            "hideZero": false,
             "max": false,
             "min": false,
             "show": true,
@@ -4942,11 +5847,10 @@
           "lines": true,
           "linewidth": 2,
           "links": [],
-          "nullPointMode": "connected",
-          "options": {
-            "dataLinks": []
-          },
+          "nullPointMode": "null",
+          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4956,11 +5860,11 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "legendFormat": "{{job}} {{block_name}}",
               "refId": "A",
               "step": 20
             }
@@ -4969,10 +5873,10 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Cache request rate",
+          "title": "Total DB Usage by Block",
           "tooltip": {
-            "shared": false,
-            "sort": 0,
+            "shared": true,
+            "sort": 2,
             "value_type": "cumulative"
           },
           "type": "graph",
@@ -4985,7 +5889,7 @@
           },
           "yaxes": [
             {
-              "format": "rps",
+              "format": "percentunit",
               "logBase": 1,
               "max": null,
               "min": 0,
@@ -5010,16 +5914,27 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "description": "The time each database transaction takes to execute, on average, broken down by metrics block.",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
+          "grid": {},
           "gridPos": {
-            "h": 10,
+            "h": 13,
             "w": 12,
             "x": 12,
-            "y": 47
+            "y": 22
           },
           "hiddenSeries": false,
-          "id": 39,
+          "id": 27,
           "legend": {
             "alignAsTable": true,
             "avg": false,
@@ -5031,13 +5946,12 @@
             "values": false
           },
           "lines": true,
-          "linewidth": 1,
+          "linewidth": 2,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
+          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -5047,10 +5961,11 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "topk(10, rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=\"$job\",instance=\"$instance\"}[$bucket_size]))",
+              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
               "refId": "A",
               "step": 20
             }
@@ -5059,11 +5974,11 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Top 10 cache misses",
+          "title": "Average Database Transaction time, by Block",
           "tooltip": {
-            "shared": false,
-            "sort": 0,
-            "value_type": "individual"
+            "shared": true,
+            "sort": 2,
+            "value_type": "cumulative"
           },
           "type": "graph",
           "xaxis": {
@@ -5075,8 +5990,7 @@
           },
           "yaxes": [
             {
-              "format": "rps",
-              "label": null,
+              "format": "ms",
               "logBase": 1,
               "max": null,
               "min": null,
@@ -5084,7 +5998,6 @@
             },
             {
               "format": "short",
-              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -5102,34 +6015,42 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
+          "grid": {},
           "gridPos": {
-            "h": 9,
+            "h": 13,
             "w": 12,
             "x": 0,
-            "y": 57
+            "y": 35
           },
           "hiddenSeries": false,
-          "id": 65,
+          "id": 28,
           "legend": {
-            "alignAsTable": true,
             "avg": false,
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
           "lines": true,
-          "linewidth": 1,
+          "linewidth": 2,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
+          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -5139,22 +6060,24 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_util_caches_cache:evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{name}} {{job}}-{{index}}",
-              "refId": "A"
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Cache eviction rate",
+          "title": "Average Transactions per Block",
           "tooltip": {
             "shared": false,
             "sort": 0,
-            "value_type": "individual"
+            "value_type": "cumulative"
           },
           "type": "graph",
           "xaxis": {
@@ -5166,9 +6089,7 @@
           },
           "yaxes": [
             {
-              "decimals": null,
-              "format": "hertz",
-              "label": "entries / second",
+              "format": "none",
               "logBase": 1,
               "max": null,
               "min": null,
@@ -5176,7 +6097,6 @@
             },
             {
               "format": "short",
-              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -5187,70 +6107,155 @@
             "align": false,
             "alignLevel": null
           }
-        }
-      ],
-      "repeat": null,
-      "title": "Caches",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": "${DS_PROMETHEUS}",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 37
-      },
-      "id": 62,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
+          "grid": {},
           "gridPos": {
-            "h": 9,
+            "h": 13,
             "w": 12,
-            "x": 0,
-            "y": 121
+            "x": 12,
+            "y": 35
           },
           "hiddenSeries": false,
-          "id": 91,
+          "id": 25,
           "legend": {
             "avg": false,
             "current": false,
             "max": false,
             "min": false,
-            "show": true,
+            "show": false,
             "total": false,
             "values": false
           },
           "lines": true,
-          "linewidth": 1,
+          "linewidth": 2,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
+          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
           "spaceLength": 10,
-          "stack": true,
+          "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])",
+              "expr": "rate(synapse_util_metrics_block_time_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count[$bucket_size])",
               "format": "time_series",
-              "instant": false,
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Average Wallclock Time per Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 15,
+            "w": 12,
+            "x": 0,
+            "y": 48
+          },
+          "hiddenSeries": false,
+          "id": 154,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "percentage": false,
+          "pluginVersion": "7.1.3",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "interval": "",
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
               "refId": "A"
             }
           ],
@@ -5258,10 +6263,10 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Total GC time by bucket (10m smoothing)",
+          "title": "Block count",
           "tooltip": {
             "shared": true,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -5274,12 +6279,11 @@
           },
           "yaxes": [
             {
-              "decimals": null,
-              "format": "percentunit",
+              "format": "hertz",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
@@ -5295,31 +6299,56 @@
             "align": false,
             "alignLevel": null
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "title": "Per-block metrics",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 34
+      },
+      "id": 61,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "decimals": 3,
+          "decimals": 2,
           "editable": true,
           "error": false,
-          "fill": 1,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 0,
           "fillGradient": 0,
           "grid": {},
           "gridPos": {
-            "h": 9,
+            "h": 10,
             "w": 12,
-            "x": 12,
-            "y": 121
+            "x": 0,
+            "y": 35
           },
           "hiddenSeries": false,
-          "id": 21,
+          "id": 1,
           "legend": {
             "alignAsTable": true,
             "avg": false,
             "current": false,
+            "hideEmpty": true,
+            "hideZero": false,
             "max": false,
             "min": false,
             "show": true,
@@ -5329,11 +6358,12 @@
           "lines": true,
           "linewidth": 2,
           "links": [],
-          "nullPointMode": "null as zero",
+          "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -5343,21 +6373,21 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])",
+              "expr": "rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
               "format": "time_series",
               "intervalFactor": 2,
-              "legendFormat": "{{job}} {{index}} gen {{gen}} ",
+              "legendFormat": "{{name}} {{job}}-{{index}}",
               "refId": "A",
-              "step": 20,
-              "target": ""
+              "step": 20
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Average GC Time Per Collection",
+          "title": "Cache Hit Ratio",
           "tooltip": {
+            "msResolution": true,
             "shared": false,
             "sort": 0,
             "value_type": "cumulative"
@@ -5372,10 +6402,12 @@
           },
           "yaxes": [
             {
-              "format": "s",
+              "decimals": null,
+              "format": "percentunit",
+              "label": "",
               "logBase": 1,
-              "max": null,
-              "min": null,
+              "max": "1",
+              "min": 0,
               "show": true
             },
             {
@@ -5383,7 +6415,7 @@
               "logBase": 1,
               "max": null,
               "min": null,
-              "show": true
+              "show": false
             }
           ],
           "yaxis": {
@@ -5397,21 +6429,30 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
+          "grid": {},
           "gridPos": {
-            "h": 9,
+            "h": 10,
             "w": 12,
-            "x": 0,
-            "y": 130
+            "x": 12,
+            "y": 35
           },
           "hiddenSeries": false,
-          "id": 89,
+          "id": 8,
           "legend": {
+            "alignAsTable": true,
             "avg": false,
             "current": false,
-            "hideEmpty": true,
             "hideZero": false,
             "max": false,
             "min": false,
@@ -5420,43 +6461,42 @@
             "values": false
           },
           "lines": true,
-          "linewidth": 1,
+          "linewidth": 2,
           "links": [],
-          "nullPointMode": "null",
+          "nullPointMode": "connected",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [
-            {
-              "alias": "/gen 0$/",
-              "yaxis": 2
-            }
-          ],
+          "seriesOverrides": [],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "expr": "synapse_util_caches_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
               "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
-              "refId": "A"
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Allocation counts",
+          "title": "Cache Size",
           "tooltip": {
             "shared": false,
             "sort": 0,
-            "value_type": "individual"
+            "value_type": "cumulative"
           },
           "type": "graph",
           "xaxis": {
@@ -5469,16 +6509,13 @@
           "yaxes": [
             {
               "format": "short",
-              "label": "Gen N-1 GCs since last Gen N GC",
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": 0,
               "show": true
             },
             {
-              "decimals": null,
               "format": "short",
-              "label": "Objects since last Gen 0 GC",
               "logBase": 1,
               "max": null,
               "min": null,
@@ -5496,19 +6533,31 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
+          "grid": {},
           "gridPos": {
-            "h": 9,
+            "h": 10,
             "w": 12,
-            "x": 12,
-            "y": 130
+            "x": 0,
+            "y": 45
           },
           "hiddenSeries": false,
-          "id": 93,
+          "id": 38,
           "legend": {
+            "alignAsTable": true,
             "avg": false,
             "current": false,
+            "hideZero": false,
             "max": false,
             "min": false,
             "show": true,
@@ -5516,13 +6565,14 @@
             "values": false
           },
           "lines": true,
-          "linewidth": 1,
+          "linewidth": 2,
           "links": [],
           "nullPointMode": "connected",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -5532,22 +6582,24 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
               "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
-              "refId": "A"
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Object counts per collection",
+          "title": "Cache request rate",
           "tooltip": {
-            "shared": true,
+            "shared": false,
             "sort": 0,
-            "value_type": "individual"
+            "value_type": "cumulative"
           },
           "type": "graph",
           "xaxis": {
@@ -5557,28 +6609,2420 @@
             "show": true,
             "values": []
           },
-          "yaxes": [
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": null
-          }
+          "yaxes": [
+            {
+              "format": "rps",
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 45
+          },
+          "hiddenSeries": false,
+          "id": 39,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "topk(10, rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Top 10 cache misses",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "rps",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 55
+          },
+          "hiddenSeries": false,
+          "id": 65,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_caches_cache:evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Cache eviction rate",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "hertz",
+              "label": "entries / second",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "repeat": null,
+      "title": "Caches",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 35
+      },
+      "id": 148,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 29
+          },
+          "hiddenSeries": false,
+          "id": 146,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_util_caches_response_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "interval": "",
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Response cache size",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 29
+          },
+          "hiddenSeries": false,
+          "id": 150,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_caches_response_cache:hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache:total{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
+              "interval": "",
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A"
+            },
+            {
+              "expr": "",
+              "interval": "",
+              "legendFormat": "",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Response cache hit rate",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "percentunit",
+              "label": null,
+              "logBase": 1,
+              "max": "1",
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "title": "Response caches",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 36
+      },
+      "id": 62,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 30
+          },
+          "hiddenSeries": false,
+          "id": 91,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": true,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])",
+              "format": "time_series",
+              "instant": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Total GC time by bucket (10m smoothing)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "percentunit",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": 3,
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 30
+          },
+          "hiddenSeries": false,
+          "id": 21,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}} {{index}} gen {{gen}} ",
+              "refId": "A",
+              "step": 20,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Average GC Time Per Collection",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 39
+          },
+          "hiddenSeries": false,
+          "id": 89,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "alias": "/gen 0$/",
+              "yaxis": 2
+            }
+          ],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Allocation counts",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": "Gen N-1 GCs since last Gen N GC",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": "Objects since last Gen 0 GC",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 39
+          },
+          "hiddenSeries": false,
+          "id": 93,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Object counts per collection",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 48
+          },
+          "hiddenSeries": false,
+          "id": 95,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "GC frequency",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "cards": {
+            "cardPadding": 0,
+            "cardRound": null
+          },
+          "color": {
+            "cardColor": "#b4ff00",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateSpectral",
+            "exponent": 0.5,
+            "max": null,
+            "min": 0,
+            "mode": "spectrum"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": "${DS_PROMETHEUS}",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 48
+          },
+          "heatmap": {},
+          "hideZeroBuckets": true,
+          "highlightCards": true,
+          "id": 87,
+          "legend": {
+            "show": true
+          },
+          "links": [],
+          "reverseYBuckets": false,
+          "targets": [
+            {
+              "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",
+              "format": "heatmap",
+              "intervalFactor": 1,
+              "legendFormat": "{{le}}",
+              "refId": "A"
+            }
+          ],
+          "title": "GC durations",
+          "tooltip": {
+            "show": true,
+            "showHistogram": false
+          },
+          "type": "heatmap",
+          "xAxis": {
+            "show": true
+          },
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": null,
+            "format": "s",
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true,
+            "splitFactor": null
+          },
+          "yBucketBound": "auto",
+          "yBucketNumber": null,
+          "yBucketSize": null
+        }
+      ],
+      "repeat": null,
+      "title": "GC",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 37
+      },
+      "id": 63,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 13
+          },
+          "hiddenSeries": false,
+          "id": 42,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{command}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Rate of incoming commands",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 13
+          },
+          "hiddenSeries": false,
+          "id": 144,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "interval": "",
+              "legendFormat": "{{stream_name}} {{job}}-{{index}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Queued incoming RDATA commands, by stream",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 20
+          },
+          "hiddenSeries": false,
+          "id": 43,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{command}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Rate of outgoing commands",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 20
+          },
+          "hiddenSeries": false,
+          "id": 41,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_replication_tcp_resource_stream_updates{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{stream_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Outgoing stream updates",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 27
+          },
+          "hiddenSeries": false,
+          "id": 113,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{stream_name}}",
+              "refId": "A"
+            },
+            {
+              "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Replication connections",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 27
+          },
+          "hiddenSeries": false,
+          "id": 115,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_replication_tcp_protocol_close_reason{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{reason_type}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Replication connection close reasons",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "repeat": null,
+      "title": "Replication",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 38
+      },
+      "id": 69,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 41
+          },
+          "hiddenSeries": false,
+          "id": 67,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Event processing lag",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": "events",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 41
+          },
+          "hiddenSeries": false,
+          "id": 71,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Age of last processed event",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 50
+          },
+          "hiddenSeries": false,
+          "id": 121,
+          "interval": "",
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Event processing catchup rate",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "none",
+              "label": "fallbehind(-) / catchup(+):  s/sec",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "title": "Event processing loop positions",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 39
+      },
+      "id": 126,
+      "panels": [
+        {
+          "cards": {
+            "cardPadding": 0,
+            "cardRound": null
+          },
+          "color": {
+            "cardColor": "#B877D9",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateInferno",
+            "exponent": 0.5,
+            "max": null,
+            "min": 0,
+            "mode": "opacity"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": "$datasource",
+          "description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 42
+          },
+          "heatmap": {},
+          "hideZeroBuckets": true,
+          "highlightCards": true,
+          "id": 122,
+          "legend": {
+            "show": true
+          },
+          "links": [],
+          "reverseYBuckets": false,
+          "targets": [
+            {
+              "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
+              "format": "heatmap",
+              "intervalFactor": 1,
+              "legendFormat": "{{le}}",
+              "refId": "A"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Number of rooms, by number of forward extremities in room",
+          "tooltip": {
+            "show": true,
+            "showHistogram": true
+          },
+          "type": "heatmap",
+          "xAxis": {
+            "show": true
+          },
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": 0,
+            "format": "short",
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true,
+            "splitFactor": null
+          },
+          "yBucketBound": "auto",
+          "yBucketNumber": null,
+          "yBucketSize": null
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 0,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 42
+          },
+          "hiddenSeries": false,
+          "id": 124,
+          "interval": "",
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pluginVersion": "7.1.3",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0",
+              "format": "heatmap",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{le}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Room counts, by number of extremities",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "none",
+              "label": "Number of rooms",
+              "logBase": 10,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "cards": {
+            "cardPadding": 0,
+            "cardRound": null
+          },
+          "color": {
+            "cardColor": "#5794F2",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateInferno",
+            "exponent": 0.5,
+            "min": 0,
+            "mode": "opacity"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": "$datasource",
+          "description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 50
+          },
+          "heatmap": {},
+          "hideZeroBuckets": true,
+          "highlightCards": true,
+          "id": 127,
+          "legend": {
+            "show": true
+          },
+          "links": [],
+          "reverseYBuckets": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
+              "format": "heatmap",
+              "intervalFactor": 1,
+              "legendFormat": "{{le}}",
+              "refId": "A"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Events persisted, by number of forward extremities in room (heatmap)",
+          "tooltip": {
+            "show": true,
+            "showHistogram": true
+          },
+          "type": "heatmap",
+          "xAxis": {
+            "show": true
+          },
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": 0,
+            "format": "short",
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true,
+            "splitFactor": null
+          },
+          "yBucketBound": "auto",
+          "yBucketNumber": null,
+          "yBucketSize": null
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 50
+          },
+          "hiddenSeries": false,
+          "id": 128,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pluginVersion": "7.1.3",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "50%",
+              "refId": "A"
+            },
+            {
+              "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "75%",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "90%",
+              "refId": "C"
+            },
+            {
+              "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "99%",
+              "refId": "D"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Events persisted, by number of forward extremities in room (quantiles)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": "Number of extremities in room",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "cards": {
+            "cardPadding": 0,
+            "cardRound": null
+          },
+          "color": {
+            "cardColor": "#FF9830",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateInferno",
+            "exponent": 0.5,
+            "min": 0,
+            "mode": "opacity"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": "$datasource",
+          "description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 58
+          },
+          "heatmap": {},
+          "hideZeroBuckets": true,
+          "highlightCards": true,
+          "id": 129,
+          "legend": {
+            "show": true
+          },
+          "links": [],
+          "reverseYBuckets": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
+              "format": "heatmap",
+              "intervalFactor": 1,
+              "legendFormat": "{{le}}",
+              "refId": "A"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Events persisted, by number of stale forward extremities in room (heatmap)",
+          "tooltip": {
+            "show": true,
+            "showHistogram": true
+          },
+          "type": "heatmap",
+          "xAxis": {
+            "show": true
+          },
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": 0,
+            "format": "short",
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true,
+            "splitFactor": null
+          },
+          "yBucketBound": "auto",
+          "yBucketNumber": null,
+          "yBucketSize": null
         },
         {
           "aliasColors": {},
@@ -5586,16 +9030,24 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "description": "For  given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 9,
+            "h": 8,
             "w": 12,
-            "x": 0,
-            "y": 139
+            "x": 12,
+            "y": 58
           },
           "hiddenSeries": false,
-          "id": 95,
+          "id": 130,
           "legend": {
             "avg": false,
             "current": false,
@@ -5609,11 +9061,9 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.1.3",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -5622,18 +9072,39 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
               "format": "time_series",
               "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "legendFormat": "50%",
               "refId": "A"
+            },
+            {
+              "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "75%",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "90%",
+              "refId": "C"
+            },
+            {
+              "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "99%",
+              "refId": "D"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "GC frequency",
+          "title": "Events persisted, by number of stale forward extremities in room (quantiles)",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -5649,11 +9120,11 @@
           },
           "yaxes": [
             {
-              "format": "hertz",
-              "label": null,
+              "format": "short",
+              "label": "Number of stale forward extremities in room",
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -5676,26 +9147,32 @@
             "cardRound": null
           },
           "color": {
-            "cardColor": "#b4ff00",
+            "cardColor": "#73BF69",
             "colorScale": "sqrt",
-            "colorScheme": "interpolateSpectral",
+            "colorScheme": "interpolateInferno",
             "exponent": 0.5,
-            "max": null,
             "min": 0,
-            "mode": "spectrum"
+            "mode": "opacity"
           },
           "dataFormat": "tsbuckets",
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
+          "description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "gridPos": {
-            "h": 9,
+            "h": 8,
             "w": 12,
-            "x": 12,
-            "y": 139
+            "x": 0,
+            "y": 66
           },
           "heatmap": {},
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 87,
+          "id": 131,
           "legend": {
             "show": true
           },
@@ -5703,17 +9180,20 @@
           "reverseYBuckets": false,
           "targets": [
             {
-              "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",
+              "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "heatmap",
+              "interval": "",
               "intervalFactor": 1,
               "legendFormat": "{{le}}",
               "refId": "A"
             }
           ],
-          "title": "GC durations",
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Number of state resolution performed, by number of state groups involved (heatmap)",
           "tooltip": {
             "show": true,
-            "showHistogram": false
+            "showHistogram": true
           },
           "type": "heatmap",
           "xAxis": {
@@ -5722,8 +9202,8 @@
           "xBucketNumber": null,
           "xBucketSize": null,
           "yAxis": {
-            "decimals": null,
-            "format": "s",
+            "decimals": 0,
+            "format": "short",
             "logBase": 1,
             "max": null,
             "min": null,
@@ -5733,39 +9213,32 @@
           "yBucketBound": "auto",
           "yBucketNumber": null,
           "yBucketSize": null
-        }
-      ],
-      "repeat": null,
-      "title": "GC",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": "${DS_PROMETHEUS}",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 38
-      },
-      "id": 63,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 7,
+            "h": 8,
             "w": 12,
-            "x": 0,
+            "x": 12,
             "y": 66
           },
           "hiddenSeries": false,
-          "id": 2,
+          "id": 132,
+          "interval": "",
           "legend": {
             "avg": false,
             "current": false,
@@ -5779,12 +9252,9 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
-          "paceLength": 10,
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.1.3",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -5793,53 +9263,150 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_replication_tcp_resource_user_sync{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
               "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "user started/stopped syncing",
-              "refId": "A",
-              "step": 20
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "50%",
+              "refId": "A"
             },
             {
-              "expr": "rate(synapse_replication_tcp_resource_federation_ack{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
               "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "federation ack",
-              "refId": "B",
-              "step": 20
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "75%",
+              "refId": "B"
             },
             {
-              "expr": "rate(synapse_replication_tcp_resource_remove_pusher{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
               "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "remove pusher",
-              "refId": "C",
-              "step": 20
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "90%",
+              "refId": "C"
             },
             {
-              "expr": "rate(synapse_replication_tcp_resource_invalidate_cache{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
               "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "invalidate cache",
-              "refId": "D",
-              "step": 20
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "99%",
+              "refId": "D"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Number of state resolutions performed, by number of state groups involved (quantiles)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": "Number of state groups",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
             },
             {
-              "expr": "rate(synapse_replication_tcp_resource_user_ip_cache{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
-              "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "user ip cache",
-              "refId": "E",
-              "step": 20
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 74
+          },
+          "hiddenSeries": false,
+          "id": 179,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "percentage": false,
+          "pluginVersion": "7.1.3",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "interval": "",
+              "legendFormat": "State res ",
+              "refId": "A"
+            },
+            {
+              "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "interval": "",
+              "legendFormat": "Potential to prune",
+              "refId": "B"
+            },
+            {
+              "expr": "sum(rate(synapse_storage_events_times_pruned_extremities{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "interval": "",
+              "legendFormat": "Pruned",
+              "refId": "C"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Rate of events on replication master",
+          "title": "Stale extremity dropping",
           "tooltip": {
-            "shared": false,
+            "shared": true,
             "sort": 0,
             "value_type": "individual"
           },
@@ -5873,23 +9440,45 @@
             "align": false,
             "alignLevel": null
           }
-        },
+        }
+      ],
+      "title": "Extremities",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 40
+      },
+      "id": 158,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 7,
+            "h": 8,
             "w": 12,
-            "x": 12,
-            "y": 66
+            "x": 0,
+            "y": 41
           },
           "hiddenSeries": false,
-          "id": 41,
+          "id": 156,
           "legend": {
             "avg": false,
             "current": false,
@@ -5904,35 +9493,49 @@
           "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
-          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "Max",
+              "color": "#bf1b00",
+              "fill": 0,
+              "linewidth": 2
+            }
+          ],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_replication_tcp_resource_stream_updates{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "expr": "synapse_admin_mau:current{instance=\"$instance\", job=~\"$job\"}",
               "format": "time_series",
               "interval": "",
-              "intervalFactor": 2,
-              "legendFormat": "{{stream_name}}",
-              "refId": "A",
-              "step": 20
+              "intervalFactor": 1,
+              "legendFormat": "Current",
+              "refId": "A"
+            },
+            {
+              "expr": "synapse_admin_mau:max{instance=\"$instance\", job=~\"$job\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "Max",
+              "refId": "B"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Outgoing stream updates",
+          "title": "MAU Limits",
           "tooltip": {
-            "shared": false,
+            "shared": true,
             "sort": 0,
             "value_type": "individual"
           },
@@ -5946,11 +9549,11 @@
           },
           "yaxes": [
             {
-              "format": "hertz",
+              "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -5973,16 +9576,22 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 7,
+            "h": 8,
             "w": 12,
-            "x": 0,
-            "y": 73
+            "x": 12,
+            "y": 41
           },
           "hiddenSeries": false,
-          "id": 42,
+          "id": 160,
           "legend": {
             "avg": false,
             "current": false,
@@ -5994,14 +9603,13 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
-          "paceLength": 10,
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -6010,21 +9618,19 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
-              "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} {{command}}",
-              "refId": "A",
-              "step": 20
+              "expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}",
+              "interval": "",
+              "legendFormat": "{{ app_service }}",
+              "refId": "A"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Rate of incoming commands",
+          "title": "MAU by Appservice",
           "tooltip": {
-            "shared": false,
+            "shared": true,
             "sort": 0,
             "value_type": "individual"
           },
@@ -6038,7 +9644,7 @@
           },
           "yaxes": [
             {
-              "format": "hertz",
+              "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -6058,23 +9664,45 @@
             "align": false,
             "alignLevel": null
           }
-        },
+        }
+      ],
+      "title": "MAU",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 41
+      },
+      "id": 177,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
-            "y": 73
+            "x": 0,
+            "y": 1
           },
           "hiddenSeries": false,
-          "id": 43,
+          "id": 173,
           "legend": {
             "avg": false,
             "current": false,
@@ -6088,11 +9716,8 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
-          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -6102,22 +9727,24 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+              "expr": "rate(synapse_notifier_users_woken_by_stream{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
               "format": "time_series",
+              "hide": false,
               "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}} {{command}}",
+              "legendFormat": "{{stream}} {{index}}",
+              "metric": "synapse_notifier",
               "refId": "A",
-              "step": 20
+              "step": 2
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Rate of outgoing commands",
+          "title": "Notifier Streams Woken",
           "tooltip": {
-            "shared": false,
-            "sort": 0,
+            "shared": true,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -6157,16 +9784,23 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 80
+            "x": 12,
+            "y": 1
           },
           "hiddenSeries": false,
-          "id": 113,
+          "id": 175,
           "legend": {
             "avg": false,
             "current": false,
@@ -6180,11 +9814,8 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
-          "paceLength": 10,
           "percentage": false,
+          "pluginVersion": "7.1.3",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -6194,28 +9825,23 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} {{stream_name}}",
-              "refId": "A"
-            },
-            {
-              "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "expr": "rate(synapse_handler_presence_get_updates{job=~\"$job\",instance=\"$instance\"}[$bucket_size])",
               "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}}",
-              "refId": "B"
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{type}} {{index}}",
+              "refId": "A",
+              "step": 2
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Replication connections",
+          "title": "Presence Stream Fetch Type Rates",
           "tooltip": {
             "shared": true,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -6228,7 +9854,7 @@
           },
           "yaxes": [
             {
-              "format": "short",
+              "format": "hertz",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -6248,23 +9874,44 @@
             "align": false,
             "alignLevel": null
           }
-        },
+        }
+      ],
+      "title": "Notifier",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 42
+      },
+      "id": 170,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 7,
+            "h": 8,
             "w": 12,
-            "x": 12,
-            "y": 80
+            "x": 0,
+            "y": 43
           },
           "hiddenSeries": false,
-          "id": 115,
+          "id": 168,
           "legend": {
             "avg": false,
             "current": false,
@@ -6276,14 +9923,13 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
-          "paceLength": 10,
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -6292,10 +9938,9 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_replication_tcp_protocol_close_reason{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} {{reason_type}}",
+              "expr": "rate(synapse_appservice_api_sent_events{instance=\"$instance\"}[$bucket_size])",
+              "interval": "",
+              "legendFormat": "{{service}}",
               "refId": "A"
             }
           ],
@@ -6303,7 +9948,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Replication connection close reasons",
+          "title": "Sent Events rate",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -6339,39 +9984,29 @@
             "align": false,
             "alignLevel": null
           }
-        }
-      ],
-      "repeat": null,
-      "title": "Replication",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": "${DS_PROMETHEUS}",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 39
-      },
-      "id": 69,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 9,
+            "h": 8,
             "w": 12,
-            "x": 0,
-            "y": 40
+            "x": 12,
+            "y": 43
           },
           "hiddenSeries": false,
-          "id": 67,
+          "id": 171,
           "legend": {
             "avg": false,
             "current": false,
@@ -6383,14 +10018,13 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
-          "nullPointMode": "connected",
+          "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
-          "paceLength": 10,
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -6399,11 +10033,9 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - ignoring(instance,index, job, name) group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
-              "format": "time_series",
+              "expr": "rate(synapse_appservice_api_sent_transactions{instance=\"$instance\"}[$bucket_size])",
               "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "legendFormat": "{{service}}",
               "refId": "A"
             }
           ],
@@ -6411,7 +10043,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Event processing lag",
+          "title": "Transactions rate",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -6427,8 +10059,8 @@
           },
           "yaxes": [
             {
-              "format": "short",
-              "label": "events",
+              "format": "hertz",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -6447,23 +10079,44 @@
             "align": false,
             "alignLevel": null
           }
-        },
+        }
+      ],
+      "title": "Appservices",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 43
+      },
+      "id": 188,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 9,
+            "h": 8,
             "w": 12,
-            "x": 12,
-            "y": 40
+            "x": 0,
+            "y": 44
           },
           "hiddenSeries": false,
-          "id": 71,
+          "id": 182,
           "legend": {
             "avg": false,
             "current": false,
@@ -6475,14 +10128,13 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
-          "nullPointMode": "connected",
+          "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
-          "paceLength": 10,
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -6491,23 +10143,44 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
-              "format": "time_series",
-              "hide": false,
+              "expr": "rate(synapse_handler_presence_notified_presence{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
               "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "legendFormat": "Notified",
+              "refId": "A"
+            },
+            {
+              "expr": "rate(synapse_handler_presence_federation_presence_out{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "interval": "",
+              "legendFormat": "Remote ping",
               "refId": "B"
+            },
+            {
+              "expr": "rate(synapse_handler_presence_presence_updates{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "interval": "",
+              "legendFormat": "Total updates",
+              "refId": "C"
+            },
+            {
+              "expr": "rate(synapse_handler_presence_federation_presence{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "interval": "",
+              "legendFormat": "Remote updates",
+              "refId": "D"
+            },
+            {
+              "expr": "rate(synapse_handler_presence_bump_active_time{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "interval": "",
+              "legendFormat": "Bump active time",
+              "refId": "E"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Age of last processed event",
+          "title": "Presence",
           "tooltip": {
             "shared": true,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -6520,7 +10193,7 @@
           },
           "yaxes": [
             {
-              "format": "ms",
+              "format": "hertz",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -6547,17 +10220,22 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 9,
+            "h": 8,
             "w": 12,
-            "x": 0,
-            "y": 49
+            "x": 12,
+            "y": 44
           },
           "hiddenSeries": false,
-          "id": 121,
-          "interval": "",
+          "id": 184,
           "legend": {
             "avg": false,
             "current": false,
@@ -6569,14 +10247,13 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
-          "nullPointMode": "connected",
+          "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
-          "paceLength": 10,
           "percentage": false,
-          "pointradius": 5,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
           "points": false,
           "renderer": "flot",
           "seriesOverrides": [],
@@ -6585,23 +10262,20 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1",
-              "format": "time_series",
-              "hide": false,
+              "expr": "rate(synapse_handler_presence_state_transition{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
               "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} {{name}}",
-              "refId": "B"
+              "legendFormat": "{{from}} -> {{to}}",
+              "refId": "A"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Event processing catchup rate",
+          "title": "Presence state transitions",
           "tooltip": {
             "shared": true,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -6614,9 +10288,8 @@
           },
           "yaxes": [
             {
-              "decimals": null,
-              "format": "none",
-              "label": "fallbehind(-) / catchup(+):  s/sec",
+              "format": "hertz",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -6635,88 +10308,6 @@
             "align": false,
             "alignLevel": null
           }
-        }
-      ],
-      "title": "Event processing loop positions",
-      "type": "row"
-    },
-    {
-      "collapsed": true,
-      "datasource": "${DS_PROMETHEUS}",
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 40
-      },
-      "id": 126,
-      "panels": [
-        {
-          "cards": {
-            "cardPadding": 0,
-            "cardRound": null
-          },
-          "color": {
-            "cardColor": "#B877D9",
-            "colorScale": "sqrt",
-            "colorScheme": "interpolateInferno",
-            "exponent": 0.5,
-            "max": null,
-            "min": 0,
-            "mode": "opacity"
-          },
-          "dataFormat": "tsbuckets",
-          "datasource": "$datasource",
-          "description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.",
-          "gridPos": {
-            "h": 8,
-            "w": 12,
-            "x": 0,
-            "y": 86
-          },
-          "heatmap": {},
-          "hideZeroBuckets": true,
-          "highlightCards": true,
-          "id": 122,
-          "legend": {
-            "show": true
-          },
-          "links": [],
-          "reverseYBuckets": false,
-          "targets": [
-            {
-              "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
-              "format": "heatmap",
-              "intervalFactor": 1,
-              "legendFormat": "{{le}}",
-              "refId": "A"
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Number of rooms, by number of forward extremities in room",
-          "tooltip": {
-            "show": true,
-            "showHistogram": true
-          },
-          "type": "heatmap",
-          "xAxis": {
-            "show": true
-          },
-          "xBucketNumber": null,
-          "xBucketSize": null,
-          "yAxis": {
-            "decimals": 0,
-            "format": "short",
-            "logBase": 1,
-            "max": null,
-            "min": null,
-            "show": true,
-            "splitFactor": null
-          },
-          "yBucketBound": "auto",
-          "yBucketNumber": null,
-          "yBucketSize": null
         },
         {
           "aliasColors": {},
@@ -6724,18 +10315,22 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.",
-          "fill": 0,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
-            "x": 12,
-            "y": 86
+            "x": 0,
+            "y": 52
           },
           "hiddenSeries": false,
-          "id": 124,
-          "interval": "",
+          "id": 186,
           "legend": {
             "avg": false,
             "current": false,
@@ -6747,12 +10342,12 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -6762,11 +10357,9 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0",
-              "format": "time_series",
+              "expr": "rate(synapse_handler_presence_notify_reason{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
               "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{le}}",
+              "legendFormat": "{{reason}}",
               "refId": "A"
             }
           ],
@@ -6774,10 +10367,10 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Room counts, by number of extremities",
+          "title": "Presence  notify reason",
           "tooltip": {
-            "shared": false,
-            "sort": 1,
+            "shared": true,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -6790,9 +10383,8 @@
           },
           "yaxes": [
             {
-              "decimals": null,
-              "format": "none",
-              "label": "Number of rooms",
+              "format": "hertz",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -6804,97 +10396,51 @@
               "logBase": 1,
               "max": null,
               "min": null,
-              "show": false
+              "show": true
             }
           ],
           "yaxis": {
             "align": false,
             "alignLevel": null
           }
-        },
-        {
-          "cards": {
-            "cardPadding": 0,
-            "cardRound": null
-          },
-          "color": {
-            "cardColor": "#5794F2",
-            "colorScale": "sqrt",
-            "colorScheme": "interpolateInferno",
-            "exponent": 0.5,
-            "min": 0,
-            "mode": "opacity"
-          },
-          "dataFormat": "tsbuckets",
-          "datasource": "$datasource",
-          "description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.",
-          "gridPos": {
-            "h": 8,
-            "w": 12,
-            "x": 0,
-            "y": 94
-          },
-          "heatmap": {},
-          "hideZeroBuckets": true,
-          "highlightCards": true,
-          "id": 127,
-          "legend": {
-            "show": true
-          },
-          "links": [],
-          "reverseYBuckets": false,
-          "targets": [
-            {
-              "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
-              "format": "heatmap",
-              "intervalFactor": 1,
-              "legendFormat": "{{le}}",
-              "refId": "A"
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Events persisted, by number of forward extremities in room (heatmap)",
-          "tooltip": {
-            "show": true,
-            "showHistogram": true
-          },
-          "type": "heatmap",
-          "xAxis": {
-            "show": true
-          },
-          "xBucketNumber": null,
-          "xBucketSize": null,
-          "yAxis": {
-            "decimals": 0,
-            "format": "short",
-            "logBase": 1,
-            "max": null,
-            "min": null,
-            "show": true,
-            "splitFactor": null
-          },
-          "yBucketBound": "auto",
-          "yBucketNumber": null,
-          "yBucketSize": null
-        },
+        }
+      ],
+      "title": "Presence",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 44
+      },
+      "id": 197,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
-            "x": 12,
-            "y": 94
+            "x": 0,
+            "y": 1
           },
           "hiddenSeries": false,
-          "id": 128,
+          "id": 191,
           "legend": {
             "avg": false,
             "current": false,
@@ -6906,12 +10452,12 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -6921,42 +10467,20 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "50%",
-              "refId": "A"
-            },
-            {
-              "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "75%",
-              "refId": "B"
-            },
-            {
-              "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "90%",
-              "refId": "C"
-            },
-            {
-              "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "99%",
-              "refId": "D"
+              "expr": "rate(synapse_external_cache_set{job=\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
+              "interval": "",
+              "legendFormat": "{{ cache_name }} {{ index }}",
+              "refId": "A"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Events persisted, by number of forward extremities in room (quantiles)",
+          "title": "External Cache Set Rate",
           "tooltip": {
             "shared": true,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -6969,11 +10493,11 @@
           },
           "yaxes": [
             {
-              "format": "short",
-              "label": "Number of extremities in room",
+              "format": "hertz",
+              "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
@@ -6990,89 +10514,29 @@
             "alignLevel": null
           }
         },
-        {
-          "cards": {
-            "cardPadding": 0,
-            "cardRound": null
-          },
-          "color": {
-            "cardColor": "#FF9830",
-            "colorScale": "sqrt",
-            "colorScheme": "interpolateInferno",
-            "exponent": 0.5,
-            "min": 0,
-            "mode": "opacity"
-          },
-          "dataFormat": "tsbuckets",
-          "datasource": "$datasource",
-          "description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
-          "gridPos": {
-            "h": 8,
-            "w": 12,
-            "x": 0,
-            "y": 102
-          },
-          "heatmap": {},
-          "hideZeroBuckets": true,
-          "highlightCards": true,
-          "id": 129,
-          "legend": {
-            "show": true
-          },
-          "links": [],
-          "reverseYBuckets": false,
-          "targets": [
-            {
-              "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
-              "format": "heatmap",
-              "intervalFactor": 1,
-              "legendFormat": "{{le}}",
-              "refId": "A"
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Events persisted, by number of stale forward extremities in room (heatmap)",
-          "tooltip": {
-            "show": true,
-            "showHistogram": true
-          },
-          "type": "heatmap",
-          "xAxis": {
-            "show": true
-          },
-          "xBucketNumber": null,
-          "xBucketSize": null,
-          "yAxis": {
-            "decimals": 0,
-            "format": "short",
-            "logBase": 1,
-            "max": null,
-            "min": null,
-            "show": true,
-            "splitFactor": null
-          },
-          "yBucketBound": "auto",
-          "yBucketNumber": null,
-          "yBucketSize": null
-        },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "description": "For  given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 102
+            "y": 1
           },
           "hiddenSeries": false,
-          "id": 130,
+          "id": 193,
           "legend": {
             "avg": false,
             "current": false,
@@ -7084,12 +10548,12 @@
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
-            "dataLinks": []
+            "alertThreshold": true
           },
           "percentage": false,
+          "pluginVersion": "7.3.7",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -7099,42 +10563,20 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "50%",
+              "expr": "rate(synapse_external_cache_get{job=\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
+              "interval": "",
+              "legendFormat": "{{ cache_name }} {{ index }}",
               "refId": "A"
-            },
-            {
-              "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "75%",
-              "refId": "B"
-            },
-            {
-              "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "90%",
-              "refId": "C"
-            },
-            {
-              "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "99%",
-              "refId": "D"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Events persisted, by number of stale forward extremities in room (quantiles)",
+          "title": "External Cache Get Rate",
           "tooltip": {
             "shared": true,
-            "sort": 0,
+            "sort": 2,
             "value_type": "individual"
           },
           "type": "graph",
@@ -7147,11 +10589,11 @@
           },
           "yaxes": [
             {
-              "format": "short",
-              "label": "Number of stale forward extremities in room",
+              "format": "hertz",
+              "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
@@ -7170,52 +10612,57 @@
         },
         {
           "cards": {
-            "cardPadding": 0,
+            "cardPadding": -1,
             "cardRound": null
           },
           "color": {
-            "cardColor": "#73BF69",
+            "cardColor": "#b4ff00",
             "colorScale": "sqrt",
             "colorScheme": "interpolateInferno",
             "exponent": 0.5,
             "min": 0,
-            "mode": "opacity"
+            "mode": "spectrum"
           },
           "dataFormat": "tsbuckets",
           "datasource": "$datasource",
-          "description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
           "gridPos": {
-            "h": 8,
+            "h": 9,
             "w": 12,
             "x": 0,
-            "y": 110
+            "y": 9
           },
           "heatmap": {},
-          "hideZeroBuckets": true,
+          "hideZeroBuckets": false,
           "highlightCards": true,
-          "id": 131,
+          "id": 195,
           "legend": {
-            "show": true
+            "show": false
           },
           "links": [],
           "reverseYBuckets": false,
           "targets": [
             {
-              "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le)",
               "format": "heatmap",
+              "instant": false,
               "interval": "",
               "intervalFactor": 1,
               "legendFormat": "{{le}}",
               "refId": "A"
             }
           ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Number of state resolution performed, by number of state groups involved (heatmap)",
+          "title": "External Cache Response Time",
           "tooltip": {
             "show": true,
             "showHistogram": true
           },
+          "tooltipDecimals": 2,
           "type": "heatmap",
           "xAxis": {
             "show": true
@@ -7224,7 +10671,7 @@
           "xBucketSize": null,
           "yAxis": {
             "decimals": 0,
-            "format": "short",
+            "format": "s",
             "logBase": 1,
             "max": null,
             "min": null,
@@ -7234,131 +10681,14 @@
           "yBucketBound": "auto",
           "yBucketNumber": null,
           "yBucketSize": null
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "dashLength": 10,
-          "dashes": false,
-          "datasource": "$datasource",
-          "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.",
-          "fill": 1,
-          "fillGradient": 0,
-          "gridPos": {
-            "h": 8,
-            "w": 12,
-            "x": 12,
-            "y": 110
-          },
-          "hiddenSeries": false,
-          "id": 132,
-          "interval": "",
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 1,
-          "links": [],
-          "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 2,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "spaceLength": 10,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "50%",
-              "refId": "A"
-            },
-            {
-              "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "75%",
-              "refId": "B"
-            },
-            {
-              "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "90%",
-              "refId": "C"
-            },
-            {
-              "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
-              "format": "time_series",
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "99%",
-              "refId": "D"
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": null,
-          "timeRegions": [],
-          "timeShift": null,
-          "title": "Number of state resolutions performed, by number of state groups involved (quantiles)",
-          "tooltip": {
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "type": "graph",
-          "xaxis": {
-            "buckets": null,
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "format": "short",
-              "label": "Number of state groups",
-              "logBase": 1,
-              "max": null,
-              "min": "0",
-              "show": true
-            },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": null
-          }
         }
       ],
-      "title": "Extremities",
+      "title": "External Cache",
       "type": "row"
     }
   ],
-  "refresh": "5m",
-  "schemaVersion": 22,
+  "refresh": false,
+  "schemaVersion": 26,
   "style": "dark",
   "tags": [
     "matrix"
@@ -7368,9 +10698,10 @@
       {
         "current": {
           "selected": false,
-          "text": "Prometheus",
-          "value": "Prometheus"
+          "text": "default",
+          "value": "default"
         },
+        "error": null,
         "hide": 0,
         "includeAll": false,
         "label": null,
@@ -7378,6 +10709,7 @@
         "name": "datasource",
         "options": [],
         "query": "prometheus",
+        "queryValue": "",
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
@@ -7387,13 +10719,14 @@
         "allFormat": "glob",
         "auto": true,
         "auto_count": 100,
-        "auto_min": "30s",
+        "auto_min": "60s",
         "current": {
           "selected": false,
           "text": "auto",
           "value": "$__auto_interval_bucket_size"
         },
         "datasource": null,
+        "error": null,
         "hide": 0,
         "includeAll": false,
         "label": "Bucket Size",
@@ -7438,6 +10771,7 @@
           }
         ],
         "query": "30s,1m,2m,5m,10m,15m",
+        "queryValue": "",
         "refresh": 2,
         "skipUrlSync": false,
         "type": "interval"
@@ -7447,9 +10781,9 @@
         "current": {},
         "datasource": "$datasource",
         "definition": "",
+        "error": null,
         "hide": 0,
         "includeAll": false,
-        "index": -1,
         "label": null,
         "multi": false,
         "name": "instance",
@@ -7458,7 +10792,7 @@
         "refresh": 2,
         "regex": "",
         "skipUrlSync": false,
-        "sort": 0,
+        "sort": 1,
         "tagValuesQuery": "",
         "tags": [],
         "tagsQuery": "",
@@ -7471,10 +10805,10 @@
         "current": {},
         "datasource": "$datasource",
         "definition": "",
+        "error": null,
         "hide": 0,
         "hideLabel": false,
         "includeAll": true,
-        "index": -1,
         "label": "Job",
         "multi": true,
         "multiFormat": "regex values",
@@ -7498,10 +10832,10 @@
         "current": {},
         "datasource": "$datasource",
         "definition": "",
+        "error": null,
         "hide": 0,
         "hideLabel": false,
         "includeAll": true,
-        "index": -1,
         "label": "",
         "multi": true,
         "multiFormat": "regex values",
@@ -7522,7 +10856,7 @@
     ]
   },
   "time": {
-    "from": "now-1h",
+    "from": "now-3h",
     "to": "now"
   },
   "timepicker": {
@@ -7554,8 +10888,5 @@
   "timezone": "",
   "title": "Synapse",
   "uid": "000000012",
-  "variables": {
-    "list": []
-  },
-  "version": 32
+  "version": 99
 }
\ No newline at end of file
diff --git a/contrib/prometheus/README.md b/contrib/prometheus/README.md
index b3f23bcc80..4dbf648df8 100644
--- a/contrib/prometheus/README.md
+++ b/contrib/prometheus/README.md
@@ -34,7 +34,7 @@ Add a new job to the main prometheus.yml file:
 ```
 
 An example of a Prometheus configuration with workers can be found in
-[metrics-howto.md](https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md).
+[metrics-howto.md](https://matrix-org.github.io/synapse/latest/metrics-howto.html).
 
 To use `synapse.rules` add
 
diff --git a/contrib/purge_api/README.md b/contrib/purge_api/README.md
index 06b4cdb9f7..2f2e5c58cd 100644
--- a/contrib/purge_api/README.md
+++ b/contrib/purge_api/README.md
@@ -3,8 +3,9 @@ Purge history API examples
 
 # `purge_history.sh`
 
-A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
-purge all messages in a list of rooms up to a certain event. You can select a 
+A bash file, that uses the
+[purge history API](https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html)
+to purge all messages in a list of rooms up to a certain event. You can select a 
 timeframe or a number of messages that you want to keep in the room.
 
 Just configure the variables DOMAIN, ADMIN, ROOMS_ARRAY and TIME at the top of
@@ -12,5 +13,6 @@ the script.
 
 # `purge_remote_media.sh`
 
-A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
-purge all old cached remote media.
+A bash file, that uses the
+[purge history API](https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html)
+to purge all old cached remote media.
diff --git a/contrib/purge_api/purge_history.sh b/contrib/purge_api/purge_history.sh
index c45136ff53..9d5324ea1c 100644
--- a/contrib/purge_api/purge_history.sh
+++ b/contrib/purge_api/purge_history.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # this script will use the api:
-#    https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst
+#    https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html
 # 
 # It will purge all messages in a list of rooms up to a cetrain event
 
diff --git a/contrib/systemd-with-workers/README.md b/contrib/systemd-with-workers/README.md
index 8d21d532bd..9b19b042e9 100644
--- a/contrib/systemd-with-workers/README.md
+++ b/contrib/systemd-with-workers/README.md
@@ -1,2 +1,3 @@
 The documentation for using systemd to manage synapse workers is now part of
-the main synapse distribution. See [docs/systemd-with-workers](../../docs/systemd-with-workers).
+the main synapse distribution. See
+[docs/systemd-with-workers](https://matrix-org.github.io/synapse/latest/systemd-with-workers/index.html).
diff --git a/contrib/systemd/README.md b/contrib/systemd/README.md
new file mode 100644
index 0000000000..2844cbc8e0
--- /dev/null
+++ b/contrib/systemd/README.md
@@ -0,0 +1,18 @@
+# Setup Synapse with Systemd
+This is a setup for managing synapse with a user contributed systemd unit 
+file. It provides a `matrix-synapse` systemd unit file that should be tailored 
+to accommodate your installation in accordance with the installation 
+instructions provided in
+[installation instructions](https://matrix-org.github.io/synapse/latest/setup/installation.html).
+
+## Setup
+1. Under the service section, ensure the `User` variable matches which user
+you installed synapse under and wish to run it as. 
+2. Under the service section, ensure the `WorkingDirectory` variable matches
+where you have installed synapse.
+3. Under the service section, ensure the `ExecStart` variable matches the
+appropriate locations of your installation.
+4. Copy the `matrix-synapse.service` to `/etc/systemd/system/`
+5. Start Synapse: `sudo systemctl start matrix-synapse`
+6. Verify Synapse is running: `sudo systemctl status matrix-synapse`
+7. *optional* Enable Synapse to start at system boot: `sudo systemctl enable matrix-synapse`
diff --git a/contrib/systemd/override-hardened.conf b/contrib/systemd/override-hardened.conf
new file mode 100644
index 0000000000..b2fa3ae7c5
--- /dev/null
+++ b/contrib/systemd/override-hardened.conf
@@ -0,0 +1,71 @@
+[Service]
+# The following directives give the synapse service R/W access to:
+# - /run/matrix-synapse
+# - /var/lib/matrix-synapse
+# - /var/log/matrix-synapse
+
+RuntimeDirectory=matrix-synapse
+StateDirectory=matrix-synapse
+LogsDirectory=matrix-synapse
+
+######################
+## Security Sandbox ##
+######################
+
+# Make sure that the service has its own unshared tmpfs at /tmp and that it
+# cannot see or change any real devices
+PrivateTmp=true
+PrivateDevices=true
+
+# We give no capabilities to a service by default
+CapabilityBoundingSet=
+AmbientCapabilities=
+
+# Protect the following from modification:
+# - The entire filesystem
+# - sysctl settings and loaded kernel modules
+# - No modifications allowed to Control Groups
+# - Hostname
+# - System Clock
+ProtectSystem=strict
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+ProtectClock=true
+ProtectHostname=true
+
+# Prevent access to the following:
+# - /home directory
+# - Kernel logs
+ProtectHome=tmpfs
+ProtectKernelLogs=true
+
+# Make sure that the process can only see PIDs and process details of itself,
+# and the second option disables seeing details of things like system load and
+# I/O etc
+ProtectProc=invisible
+ProcSubset=pid
+
+# While not needed, we set these options explicitly
+# - This process has been given access to the host network
+# - It can also communicate with any IP Address
+PrivateNetwork=false
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+IPAddressAllow=any
+
+# Restrict system calls to a sane bunch
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallFilter=~@privileged @resources @obsolete
+
+# Misc restrictions
+# - Since the process is a python process it needs to be able to write and
+#   execute memory regions, so we set MemoryDenyWriteExecute to false
+RestrictSUIDSGID=true
+RemoveIPC=true
+NoNewPrivileges=true
+RestrictRealtime=true
+RestrictNamespaces=true
+LockPersonality=true
+PrivateUsers=true
+MemoryDenyWriteExecute=false
diff --git a/debian/build_virtualenv b/debian/build_virtualenv
index cad7d16883..801ecb9086 100755
--- a/debian/build_virtualenv
+++ b/debian/build_virtualenv
@@ -33,13 +33,11 @@ esac
 # Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
 # than the 2/3 compatible `virtualenv`.
 
-# Pin pip to 20.3.4 to fix breakage in 21.0 on py3.5 (xenial)
-
 dh_virtualenv \
     --install-suffix "matrix-synapse" \
     --builtin-venv \
     --python "$SNAKE" \
-    --upgrade-pip-to="20.3.4" \
+    --upgrade-pip \
     --preinstall="lxml" \
     --preinstall="mock" \
     --extra-pip-arg="--no-cache-dir" \
@@ -50,15 +48,24 @@ PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
 TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
 
-# we copy the tests to a temporary directory so that we can put them on the
-# PYTHONPATH without putting the uninstalled synapse on the pythonpath.
-tmpdir=`mktemp -d`
-trap "rm -r $tmpdir" EXIT
+case "$DEB_BUILD_OPTIONS" in
+    *nocheck*)
+        # Skip running tests if "nocheck" present in $DEB_BUILD_OPTIONS
+        ;;
+
+    *)
+        # Copy tests to a temporary directory so that we can put them on the
+        # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
+        tmpdir=`mktemp -d`
+        trap "rm -r $tmpdir" EXIT
 
-cp -r tests "$tmpdir"
+        cp -r tests "$tmpdir"
 
-PYTHONPATH="$tmpdir" \
-    "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
+        PYTHONPATH="$tmpdir" \
+            "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
+
+        ;;
+esac
 
 # build the config file
 "${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_config" \
@@ -93,3 +100,18 @@ PYTHONPATH="$tmpdir" \
 # add a dependency on the right version of python to substvars.
 PYPKG=`basename $SNAKE`
 echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars
+
+
+# add a couple of triggers.  This is needed so that dh-virtualenv can rebuild
+# the venv when the system python changes (see
+# https://dh-virtualenv.readthedocs.io/en/latest/tutorial.html#step-2-set-up-packaging-for-your-project)
+#
+# we do it here rather than the more conventional way of just adding it to
+# debian/matrix-synapse-py3.triggers, because we need to add a trigger on the
+# right version of python.
+cat >>"debian/.debhelper/generated/matrix-synapse-py3/triggers" <<EOF
+# triggers for dh-virtualenv
+interest-noawait $SNAKE
+interest dh-virtualenv-interpreter-update
+
+EOF
diff --git a/debian/changelog b/debian/changelog
index 09602ff54b..5f7a795b6e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,167 @@
+matrix-synapse-py3 (1.41.1) stable; urgency=high
+
+  * New synapse release 1.41.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 31 Aug 2021 12:59:10 +0100
+
+matrix-synapse-py3 (1.41.0) stable; urgency=medium
+
+  * New synapse release 1.41.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 24 Aug 2021 15:31:45 +0100
+
+matrix-synapse-py3 (1.41.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.41.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 18 Aug 2021 15:52:00 +0100
+
+matrix-synapse-py3 (1.40.0) stable; urgency=medium
+
+  * New synapse release 1.40.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 10 Aug 2021 13:50:48 +0100
+
+matrix-synapse-py3 (1.40.0~rc3) stable; urgency=medium
+
+  * New synapse release 1.40.0~rc3.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 09 Aug 2021 13:41:08 +0100
+
+matrix-synapse-py3 (1.40.0~rc2) stable; urgency=medium
+
+  * New synapse release 1.40.0~rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 04 Aug 2021 17:08:55 +0100
+
+matrix-synapse-py3 (1.40.0~rc1) stable; urgency=medium
+
+  [ Richard van der Hoff ]
+  * Drop backwards-compatibility code that was required to support Ubuntu Xenial.
+  * Update package triggers so that the virtualenv is correctly rebuilt
+    when the system python is rebuilt, on recent Python versions.
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.40.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 03 Aug 2021 11:31:49 +0100
+
+matrix-synapse-py3 (1.39.0) stable; urgency=medium
+
+  * New synapse release 1.39.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 29 Jul 2021 09:59:00 +0100
+
+matrix-synapse-py3 (1.39.0~rc3) stable; urgency=medium
+
+  * New synapse release 1.39.0~rc3.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 28 Jul 2021 13:30:58 +0100
+
+matrix-synapse-py3 (1.38.1) stable; urgency=medium
+
+  * New synapse release 1.38.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 22 Jul 2021 15:37:06 +0100
+
+matrix-synapse-py3 (1.39.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.39.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Jul 2021 14:28:34 +0100
+
+matrix-synapse-py3 (1.38.0) stable; urgency=medium
+
+  * New synapse release 1.38.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jul 2021 13:20:56 +0100
+
+matrix-synapse-py3 (1.38.0rc3) prerelease; urgency=medium
+
+  [ Erik Johnston ]
+  * Add synapse_review_recent_signups script
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.38.0rc3.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jul 2021 11:53:56 +0100
+
+matrix-synapse-py3 (1.37.1) stable; urgency=medium
+
+  * New synapse release 1.37.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 30 Jun 2021 12:24:06 +0100
+
+matrix-synapse-py3 (1.37.0) stable; urgency=medium
+
+  * New synapse release 1.37.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Jun 2021 10:15:25 +0100
+
+matrix-synapse-py3 (1.36.0) stable; urgency=medium
+
+  * New synapse release 1.36.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Jun 2021 15:41:53 +0100
+
+matrix-synapse-py3 (1.35.1) stable; urgency=medium
+
+  * New synapse release 1.35.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 03 Jun 2021 08:11:29 -0400
+
+matrix-synapse-py3 (1.35.0) stable; urgency=medium
+
+  * New synapse release 1.35.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 01 Jun 2021 13:23:35 +0100
+
+matrix-synapse-py3 (1.34.0) stable; urgency=medium
+
+  * New synapse release 1.34.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 17 May 2021 11:34:18 +0100
+
+matrix-synapse-py3 (1.33.2) stable; urgency=medium
+
+  * New synapse release 1.33.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 11 May 2021 11:17:59 +0100
+
+matrix-synapse-py3 (1.33.1) stable; urgency=medium
+
+  * New synapse release 1.33.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 06 May 2021 14:06:33 +0100
+
+matrix-synapse-py3 (1.33.0) stable; urgency=medium
+
+  * New synapse release 1.33.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 05 May 2021 14:15:27 +0100
+
+matrix-synapse-py3 (1.32.2) stable; urgency=medium
+
+  * New synapse release 1.32.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 22 Apr 2021 12:43:52 +0100
+
+matrix-synapse-py3 (1.32.1) stable; urgency=medium
+
+  * New synapse release 1.32.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 21 Apr 2021 14:00:55 +0100
+
+matrix-synapse-py3 (1.32.0) stable; urgency=medium
+
+  [ Dan Callahan ]
+  * Skip tests when DEB_BUILD_OPTIONS contains "nocheck".
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.32.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Apr 2021 14:28:39 +0100
+
 matrix-synapse-py3 (1.31.0) stable; urgency=medium
 
   * New synapse release 1.31.0.
diff --git a/debian/compat b/debian/compat
index ec635144f6..f599e28b8a 100644
--- a/debian/compat
+++ b/debian/compat
@@ -1 +1 @@
-9
+10
diff --git a/debian/control b/debian/control
index 8167a901a4..763fabd6f6 100644
--- a/debian/control
+++ b/debian/control
@@ -3,11 +3,8 @@ Section: contrib/python
 Priority: extra
 Maintainer: Synapse Packaging team <packages@matrix.org>
 # keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv.
-# TODO: Remove the dependency on dh-systemd after dropping support for Ubuntu xenial
-#       On all other supported releases, it's merely a transitional package which
-#       does nothing but depends on debhelper (> 9.20160709)
 Build-Depends:
- debhelper (>= 9.20160709) | dh-systemd,
+ debhelper (>= 10),
  dh-virtualenv (>= 1.1),
  libsystemd-dev,
  libpq-dev,
diff --git a/debian/hash_password.1 b/debian/hash_password.1
index 383f452991..d64b91e7c8 100644
--- a/debian/hash_password.1
+++ b/debian/hash_password.1
@@ -1,90 +1,58 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
-.
-.TH "HASH_PASSWORD" "1" "February 2017" "" ""
-.
+.\" generated with Ronn-NG/v0.8.0
+.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
+.TH "HASH_PASSWORD" "1" "July 2021" "" ""
 .SH "NAME"
 \fBhash_password\fR \- Calculate the hash of a new password, so that passwords can be reset
-.
 .SH "SYNOPSIS"
 \fBhash_password\fR [\fB\-p\fR|\fB\-\-password\fR [password]] [\fB\-c\fR|\fB\-\-config\fR \fIfile\fR]
-.
 .SH "DESCRIPTION"
 \fBhash_password\fR calculates the hash of a supplied password using bcrypt\.
-.
 .P
 \fBhash_password\fR takes a password as an parameter either on the command line or the \fBSTDIN\fR if not supplied\.
-.
 .P
 It accepts an YAML file which can be used to specify parameters like the number of rounds for bcrypt and password_config section having the pepper value used for the hashing\. By default \fBbcrypt_rounds\fR is set to \fB10\fR\.
-.
 .P
 The hashed password is written on the \fBSTDOUT\fR\.
-.
 .SH "FILES"
 A sample YAML file accepted by \fBhash_password\fR is described below:
-.
 .P
 bcrypt_rounds: 17 password_config: pepper: "random hashing pepper"
-.
 .SH "OPTIONS"
-.
 .TP
 \fB\-p\fR, \fB\-\-password\fR
 Read the password form the command line if [password] is supplied\. If not, prompt the user and read the password form the \fBSTDIN\fR\. It is not recommended to type the password on the command line directly\. Use the STDIN instead\.
-.
 .TP
 \fB\-c\fR, \fB\-\-config\fR
 Read the supplied YAML \fIfile\fR containing the options \fBbcrypt_rounds\fR and the \fBpassword_config\fR section containing the \fBpepper\fR value\.
-.
 .SH "EXAMPLES"
 Hash from the command line:
-.
 .IP "" 4
-.
 .nf
-
 $ hash_password \-p "p@ssw0rd"
 $2b$12$VJNqWQYfsWTEwcELfoSi4Oa8eA17movHqqi8\.X8fWFpum7SxZ9MFe
-.
 .fi
-.
 .IP "" 0
-.
 .P
 Hash from the STDIN:
-.
 .IP "" 4
-.
 .nf
-
 $ hash_password
 Password:
 Confirm password:
 $2b$12$AszlvfmJl2esnyhmn8m/kuR2tdXgROWtWxnX\.rcuAbM8ErLoUhybG
-.
 .fi
-.
 .IP "" 0
-.
 .P
 Using a config file:
-.
 .IP "" 4
-.
 .nf
-
 $ hash_password \-c config\.yml
 Password:
 Confirm password:
 $2b$12$CwI\.wBNr\.w3kmiUlV3T5s\.GT2wH7uebDCovDrCOh18dFedlANK99O
-.
 .fi
-.
 .IP "" 0
-.
 .SH "COPYRIGHT"
-This man page was written by Rahul De <\fIrahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
-.
+This man page was written by Rahul De <\fI\%mailto:rahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
 .SH "SEE ALSO"
-synctl(1), synapse_port_db(1), register_new_matrix_user(1)
+synctl(1), synapse_port_db(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
diff --git a/debian/hash_password.ronn b/debian/hash_password.ronn
index 0b2afa7374..eeb354602d 100644
--- a/debian/hash_password.ronn
+++ b/debian/hash_password.ronn
@@ -66,4 +66,4 @@ for Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synctl(1), synapse_port_db(1), register_new_matrix_user(1)
+synctl(1), synapse_port_db(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
diff --git a/debian/manpages b/debian/manpages
index 2c30583530..4b13f52853 100644
--- a/debian/manpages
+++ b/debian/manpages
@@ -1,4 +1,5 @@
 debian/hash_password.1
 debian/register_new_matrix_user.1
 debian/synapse_port_db.1
+debian/synapse_review_recent_signups.1
 debian/synctl.1
diff --git a/debian/matrix-synapse-py3.links b/debian/matrix-synapse-py3.links
index bf19efa562..53e2965418 100644
--- a/debian/matrix-synapse-py3.links
+++ b/debian/matrix-synapse-py3.links
@@ -1,4 +1,5 @@
 opt/venvs/matrix-synapse/bin/hash_password usr/bin/hash_password
 opt/venvs/matrix-synapse/bin/register_new_matrix_user usr/bin/register_new_matrix_user
 opt/venvs/matrix-synapse/bin/synapse_port_db usr/bin/synapse_port_db
+opt/venvs/matrix-synapse/bin/synapse_review_recent_signups usr/bin/synapse_review_recent_signups
 opt/venvs/matrix-synapse/bin/synctl usr/bin/synctl
diff --git a/debian/matrix-synapse-py3.triggers b/debian/matrix-synapse-py3.triggers
deleted file mode 100644
index f8c1fdb021..0000000000
--- a/debian/matrix-synapse-py3.triggers
+++ /dev/null
@@ -1,9 +0,0 @@
-# Register interest in Python interpreter changes and
-# don't make the Python package dependent on the virtualenv package
-# processing (noawait)
-interest-noawait /usr/bin/python3.5
-interest-noawait /usr/bin/python3.6
-interest-noawait /usr/bin/python3.7
-
-# Also provide a symbolic trigger for all dh-virtualenv packages
-interest dh-virtualenv-interpreter-update
diff --git a/debian/register_new_matrix_user.1 b/debian/register_new_matrix_user.1
index 99156a7354..57bfc4e024 100644
--- a/debian/register_new_matrix_user.1
+++ b/debian/register_new_matrix_user.1
@@ -1,72 +1,47 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
-.
-.TH "REGISTER_NEW_MATRIX_USER" "1" "February 2017" "" ""
-.
+.\" generated with Ronn-NG/v0.8.0
+.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
+.TH "REGISTER_NEW_MATRIX_USER" "1" "July 2021" "" ""
 .SH "NAME"
 \fBregister_new_matrix_user\fR \- Used to register new users with a given home server when registration has been disabled
-.
 .SH "SYNOPSIS"
-\fBregister_new_matrix_user\fR options\.\.\.
-.
+\fBregister_new_matrix_user\fR options\|\.\|\.\|\.
 .SH "DESCRIPTION"
 \fBregister_new_matrix_user\fR registers new users with a given home server when registration has been disabled\. For this to work, the home server must be configured with the \'registration_shared_secret\' option set\.
-.
 .P
 This accepts the user credentials like the username, password, is user an admin or not and registers the user onto the homeserver database\. Also, a YAML file containing the shared secret can be provided\. If not, the shared secret can be provided via the command line\.
-.
 .P
 By default it assumes the home server URL to be \fBhttps://localhost:8448\fR\. This can be changed via the \fBserver_url\fR command line option\.
-.
 .SH "FILES"
 A sample YAML file accepted by \fBregister_new_matrix_user\fR is described below:
-.
 .IP "" 4
-.
 .nf
-
 registration_shared_secret: "s3cr3t"
-.
 .fi
-.
 .IP "" 0
-.
 .SH "OPTIONS"
-.
 .TP
 \fB\-u\fR, \fB\-\-user\fR
 Local part of the new user\. Will prompt if omitted\.
-.
 .TP
 \fB\-p\fR, \fB\-\-password\fR
 New password for user\. Will prompt if omitted\. Supplying the password on the command line is not recommended\. Use the STDIN instead\.
-.
 .TP
 \fB\-a\fR, \fB\-\-admin\fR
 Register new user as an admin\. Will prompt if omitted\.
-.
 .TP
 \fB\-c\fR, \fB\-\-config\fR
 Path to server config file containing the shared secret\.
-.
 .TP
 \fB\-k\fR, \fB\-\-shared\-secret\fR
 Shared secret as defined in server config file\. This is an optional parameter as it can be also supplied via the YAML file\.
-.
 .TP
 \fBserver_url\fR
 URL of the home server\. Defaults to \'https://localhost:8448\'\.
-.
 .SH "EXAMPLES"
-.
 .nf
-
 $ register_new_matrix_user \-u user1 \-p p@ssword \-a \-c config\.yaml
-.
 .fi
-.
 .SH "COPYRIGHT"
-This man page was written by Rahul De <\fIrahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
-.
+This man page was written by Rahul De <\fI\%mailto:rahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
 .SH "SEE ALSO"
-synctl(1), synapse_port_db(1), hash_password(1)
+synctl(1), synapse_port_db(1), hash_password(1), synapse_review_recent_signups(1)
diff --git a/debian/register_new_matrix_user.ronn b/debian/register_new_matrix_user.ronn
index 4c22e74dde..0410b1f4cd 100644
--- a/debian/register_new_matrix_user.ronn
+++ b/debian/register_new_matrix_user.ronn
@@ -58,4 +58,4 @@ for Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synctl(1), synapse_port_db(1), hash_password(1)
+synctl(1), synapse_port_db(1), hash_password(1), synapse_review_recent_signups(1)
diff --git a/debian/rules b/debian/rules
index c744060a57..b9d490adc9 100755
--- a/debian/rules
+++ b/debian/rules
@@ -51,7 +51,5 @@ override_dh_shlibdeps:
 override_dh_virtualenv:
 	./debian/build_virtualenv
 
-# We are restricted to compat level 9 (because xenial), so have to
-# enable the systemd bits manually.
 %:
-	dh $@ --with python-virtualenv --with systemd
+	dh $@ --with python-virtualenv
diff --git a/debian/synapse_port_db.1 b/debian/synapse_port_db.1
index 4e6bc04827..0e7e20001c 100644
--- a/debian/synapse_port_db.1
+++ b/debian/synapse_port_db.1
@@ -1,83 +1,56 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
-.
-.TH "SYNAPSE_PORT_DB" "1" "February 2017" "" ""
-.
+.\" generated with Ronn-NG/v0.8.0
+.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
+.TH "SYNAPSE_PORT_DB" "1" "July 2021" "" ""
 .SH "NAME"
 \fBsynapse_port_db\fR \- A script to port an existing synapse SQLite database to a new PostgreSQL database\.
-.
 .SH "SYNOPSIS"
 \fBsynapse_port_db\fR [\-v] \-\-sqlite\-database=\fIdbfile\fR \-\-postgres\-config=\fIyamlconfig\fR [\-\-curses] [\-\-batch\-size=\fIbatch\-size\fR]
-.
 .SH "DESCRIPTION"
 \fBsynapse_port_db\fR ports an existing synapse SQLite database to a new PostgreSQL database\.
-.
 .P
 SQLite database is specified with \fB\-\-sqlite\-database\fR option and PostgreSQL configuration required to connect to PostgreSQL database is provided using \fB\-\-postgres\-config\fR configuration\. The configuration is specified in YAML format\.
-.
 .SH "OPTIONS"
-.
 .TP
 \fB\-v\fR
 Print log messages in \fBdebug\fR level instead of \fBinfo\fR level\.
-.
 .TP
 \fB\-\-sqlite\-database\fR
 The snapshot of the SQLite database file\. This must not be currently used by a running synapse server\.
-.
 .TP
 \fB\-\-postgres\-config\fR
 The database config file for the PostgreSQL database\.
-.
 .TP
 \fB\-\-curses\fR
 Display a curses based progress UI\.
-.
 .SH "CONFIG FILE"
 The postgres configuration file must be a valid YAML file with the following options\.
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBdatabase\fR: Database configuration section\. This section header can be ignored and the options below may be specified as top level keys\.
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBname\fR: Connector to use when connecting to the database\. This value must be \fBpsycopg2\fR\.
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBargs\fR: DB API 2\.0 compatible arguments to send to the \fBpsycopg2\fR module\.
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBdbname\fR \- the database name
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBuser\fR \- user name used to authenticate
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBpassword\fR \- password used to authenticate
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBhost\fR \- database host address (defaults to UNIX socket if not provided)
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBport\fR \- connection port number (defaults to 5432 if not provided)
-.
 .IP "" 0
 
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBsynchronous_commit\fR: Optional\. Default is True\. If the value is \fBFalse\fR, enable asynchronous commit and don\'t wait for the server to call fsync before ending the transaction\. See: https://www\.postgresql\.org/docs/current/static/wal\-async\-commit\.html
-.
 .IP "" 0
 
-.
 .IP "" 0
-.
 .P
 Following example illustrates the configuration file format\.
-.
 .IP "" 4
-.
 .nf
-
 database:
   name: psycopg2
   args:
@@ -86,13 +59,9 @@ database:
     password: ORohmi9Eet=ohphi
     host: localhost
   synchronous_commit: false
-.
 .fi
-.
 .IP "" 0
-.
 .SH "COPYRIGHT"
-This man page was written by Sunil Mohan Adapa <\fIsunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
-.
+This man page was written by Sunil Mohan Adapa <\fI\%mailto:sunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
 .SH "SEE ALSO"
-synctl(1), hash_password(1), register_new_matrix_user(1)
+synctl(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
diff --git a/debian/synapse_port_db.ronn b/debian/synapse_port_db.ronn
index fcb32ebd0d..e167af2ba4 100644
--- a/debian/synapse_port_db.ronn
+++ b/debian/synapse_port_db.ronn
@@ -47,7 +47,7 @@ following options.
     * `args`:
       DB API 2.0 compatible arguments to send to the `psycopg2` module.
 
-      * `dbname` - the database name 
+      * `dbname` - the database name
 
       * `user` - user name used to authenticate
 
@@ -58,7 +58,7 @@ following options.
 
       * `port` - connection port number (defaults to 5432 if not
         provided)
-      
+
 
     * `synchronous_commit`:
       Optional.  Default is True.  If the value is `False`, enable
@@ -76,7 +76,7 @@ Following example illustrates the configuration file format.
         password: ORohmi9Eet=ohphi
         host: localhost
       synchronous_commit: false
-  
+
 ## COPYRIGHT
 
 This man page was written by Sunil Mohan Adapa <<sunil@medhas.org>> for
@@ -84,4 +84,4 @@ Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synctl(1), hash_password(1), register_new_matrix_user(1)
+synctl(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
diff --git a/debian/synapse_review_recent_signups.1 b/debian/synapse_review_recent_signups.1
new file mode 100644
index 0000000000..2976c085f9
--- /dev/null
+++ b/debian/synapse_review_recent_signups.1
@@ -0,0 +1,26 @@
+.\" generated with Ronn-NG/v0.8.0
+.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
+.TH "SYNAPSE_REVIEW_RECENT_SIGNUPS" "1" "July 2021" "" ""
+.SH "NAME"
+\fBsynapse_review_recent_signups\fR \- Print users that have recently registered on Synapse
+.SH "SYNOPSIS"
+\fBsynapse_review_recent_signups\fR \fB\-c\fR|\fB\-\-config\fR \fIfile\fR [\fB\-s\fR|\fB\-\-since\fR \fIperiod\fR] [\fB\-e\fR|\fB\-\-exclude\-emails\fR] [\fB\-u\fR|\fB\-\-only\-users\fR]
+.SH "DESCRIPTION"
+\fBsynapse_review_recent_signups\fR prints out recently registered users on a Synapse server, as well as some basic information about the user\.
+.P
+\fBsynapse_review_recent_signups\fR must be supplied with the config of the Synapse server, so that it can fetch the database config and connect to the database\.
+.SH "OPTIONS"
+.TP
+\fB\-c\fR, \fB\-\-config\fR
+The config file(s) used by the Synapse server\.
+.TP
+\fB\-s\fR, \fB\-\-since\fR
+How far back to search for newly registered users\. Defaults to 7d, i\.e\. up to seven days in the past\. Valid units are \'s\', \'m\', \'h\', \'d\', \'w\', or \'y\'\.
+.TP
+\fB\-e\fR, \fB\-\-exclude\-emails\fR
+Do not print out users that have validated emails associated with their account\.
+.TP
+\fB\-u\fR, \fB\-\-only\-users\fR
+Only print out the user IDs of recently registered users, without any additional information
+.SH "SEE ALSO"
+synctl(1), synapse_port_db(1), register_new_matrix_user(1), hash_password(1)
diff --git a/debian/synapse_review_recent_signups.ronn b/debian/synapse_review_recent_signups.ronn
new file mode 100644
index 0000000000..77f2b040b9
--- /dev/null
+++ b/debian/synapse_review_recent_signups.ronn
@@ -0,0 +1,37 @@
+synapse_review_recent_signups(1) -- Print users that have recently registered on Synapse
+========================================================================================
+
+## SYNOPSIS
+
+`synapse_review_recent_signups` `-c`|`--config` <file> [`-s`|`--since` <period>] [`-e`|`--exclude-emails`] [`-u`|`--only-users`]
+
+## DESCRIPTION
+
+**synapse_review_recent_signups** prints out recently registered users on a
+Synapse server, as well as some basic information about the user.
+
+`synapse_review_recent_signups` must be supplied with the config of the Synapse
+server, so that it can fetch the database config and connect to the database.
+
+
+## OPTIONS
+
+  * `-c`, `--config`:
+    The config file(s) used by the Synapse server.
+
+  * `-s`, `--since`:
+    How far back to search for newly registered users. Defaults to 7d, i.e. up
+    to seven days in the past. Valid units are 's', 'm', 'h', 'd', 'w', or 'y'.
+
+  * `-e`, `--exclude-emails`:
+    Do not print out users that have validated emails associated with their
+    account.
+
+  * `-u`, `--only-users`:
+    Only print out the user IDs of recently registered users, without any
+    additional information
+
+
+## SEE ALSO
+
+synctl(1), synapse_port_db(1), register_new_matrix_user(1), hash_password(1)
diff --git a/debian/synctl.1 b/debian/synctl.1
index af58c8d224..2fdd770f09 100644
--- a/debian/synctl.1
+++ b/debian/synctl.1
@@ -1,63 +1,41 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
-.
-.TH "SYNCTL" "1" "February 2017" "" ""
-.
+.\" generated with Ronn-NG/v0.8.0
+.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
+.TH "SYNCTL" "1" "July 2021" "" ""
 .SH "NAME"
 \fBsynctl\fR \- Synapse server control interface
-.
 .SH "SYNOPSIS"
 Start, stop or restart synapse server\.
-.
 .P
 \fBsynctl\fR {start|stop|restart} [configfile] [\-w|\-\-worker=\fIWORKERCONFIG\fR] [\-a|\-\-all\-processes=\fIWORKERCONFIGDIR\fR]
-.
 .SH "DESCRIPTION"
 \fBsynctl\fR can be used to start, stop or restart Synapse server\. The control operation can be done on all processes or a single worker process\.
-.
 .SH "OPTIONS"
-.
 .TP
 \fBaction\fR
 The value of action should be one of \fBstart\fR, \fBstop\fR or \fBrestart\fR\.
-.
 .TP
 \fBconfigfile\fR
 Optional path of the configuration file to use\. Default value is \fBhomeserver\.yaml\fR\. The configuration file must exist for the operation to succeed\.
-.
 .TP
 \fB\-w\fR, \fB\-\-worker\fR:
-.
-.IP
-Perform start, stop or restart operations on a single worker\. Incompatible with \fB\-a\fR|\fB\-\-all\-processes\fR\. Value passed must be a valid worker\'s configuration file\.
-.
+
 .TP
 \fB\-a\fR, \fB\-\-all\-processes\fR:
-.
-.IP
-Perform start, stop or restart operations on all the workers in the given directory and the main synapse process\. Incompatible with \fB\-w\fR|\fB\-\-worker\fR\. Value passed must be a directory containing valid work configuration files\. All files ending with \fB\.yaml\fR extension shall be considered as configuration files and all other files in the directory are ignored\.
-.
+
 .SH "CONFIGURATION FILE"
 Configuration file may be generated as follows:
-.
 .IP "" 4
-.
 .nf
-
 $ python \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
-.
 .fi
-.
 .IP "" 0
-.
 .SH "ENVIRONMENT"
-.
 .TP
 \fBSYNAPSE_CACHE_FACTOR\fR
-Synapse\'s architecture is quite RAM hungry currently \- a lot of recent room data and metadata is deliberately cached in RAM in order to speed up common requests\. This will be improved in future, but for now the easiest way to either reduce the RAM usage (at the risk of slowing things down) is to set the SYNAPSE_CACHE_FACTOR environment variable\. Roughly speaking, a SYNAPSE_CACHE_FACTOR of 1\.0 will max out at around 3\-4GB of resident memory \- this is what we currently run the matrix\.org on\. The default setting is currently 0\.1, which is probably around a ~700MB footprint\. You can dial it down further to 0\.02 if desired, which targets roughly ~512MB\. Conversely you can dial it up if you need performance for lots of users and have a box with a lot of RAM\.
-.
+Synapse\'s architecture is quite RAM hungry currently \- we deliberately cache a lot of recent room data and metadata in RAM in order to speed up common requests\. We\'ll improve this in the future, but for now the easiest way to either reduce the RAM usage (at the risk of slowing things down) is to set the almost\-undocumented \fBSYNAPSE_CACHE_FACTOR\fR environment variable\. The default is 0\.5, which can be decreased to reduce RAM usage in memory constrained enviroments, or increased if performance starts to degrade\.
+.IP
+However, degraded performance due to a low cache factor, common on machines with slow disks, often leads to explosions in memory use due backlogged requests\. In this case, reducing the cache factor will make things worse\. Instead, try increasing it drastically\. 2\.0 is a good starting value\.
 .SH "COPYRIGHT"
-This man page was written by Sunil Mohan Adapa <\fIsunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
-.
+This man page was written by Sunil Mohan Adapa <\fI\%mailto:sunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
 .SH "SEE ALSO"
-synapse_port_db(1), hash_password(1), register_new_matrix_user(1)
+synapse_port_db(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
diff --git a/debian/synctl.ronn b/debian/synctl.ronn
index 10cbda988f..eca6a16815 100644
--- a/debian/synctl.ronn
+++ b/debian/synctl.ronn
@@ -68,4 +68,4 @@ Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synapse_port_db(1), hash_password(1), register_new_matrix_user(1)
+synapse_port_db(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
diff --git a/demo/start.sh b/demo/start.sh
index 621a5698b8..bc4854091b 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -96,18 +96,48 @@ for port in 8080 8081 8082; do
     # Check script parameters
     if [ $# -eq 1 ]; then
         if [ $1 = "--no-rate-limit" ]; then
-            # messages rate limit
-            echo 'rc_messages_per_second: 1000' >> $DIR/etc/$port.config
-            echo 'rc_message_burst_count: 1000' >> $DIR/etc/$port.config
-
-            # registration rate limit
-            printf 'rc_registration:\n  per_second: 1000\n  burst_count: 1000\n' >> $DIR/etc/$port.config
-
-            # login rate limit
-            echo 'rc_login:' >> $DIR/etc/$port.config
-            printf '  address:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
-            printf '  account:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
-            printf '  failed_attempts:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
+
+            # Disable any rate limiting
+            ratelimiting=$(cat <<-RC
+			rc_message:
+			  per_second: 1000
+			  burst_count: 1000
+			rc_registration:
+			  per_second: 1000
+			  burst_count: 1000
+			rc_login:
+			  address:
+			    per_second: 1000
+			    burst_count: 1000
+			  account:
+			    per_second: 1000
+			    burst_count: 1000
+			  failed_attempts:
+			    per_second: 1000
+			    burst_count: 1000
+			rc_admin_redaction:
+			  per_second: 1000
+			  burst_count: 1000
+			rc_joins:
+			  local:
+			    per_second: 1000
+			    burst_count: 1000
+			  remote:
+			    per_second: 1000
+			    burst_count: 1000
+			rc_3pid_validation:
+			  per_second: 1000
+			  burst_count: 1000
+			rc_invites:
+			  per_room:
+			    per_second: 1000
+			    burst_count: 1000
+			  per_user:
+			    per_second: 1000
+			    burst_count: 1000
+			RC
+			)
+            echo "${ratelimiting}" >> $DIR/etc/$port.config
         fi
     fi
 
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 5b7bf02776..2bdc607e66 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -18,11 +18,6 @@ ARG PYTHON_VERSION=3.8
 ###
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
-LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
-LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
-LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
-LABEL org.opencontainers.image.licenses='Apache-2.0'
-
 # install the OS build deps
 RUN apt-get update && apt-get install -y \
     build-essential \
@@ -66,6 +61,11 @@ RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 FROM docker.io/python:${PYTHON_VERSION}-slim
 
+LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
+LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
+LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
 RUN apt-get update && apt-get install -y \
     curl \
     gosu \
@@ -88,5 +88,5 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp
 
 ENTRYPOINT ["/start.py"]
 
-HEALTHCHECK --interval=1m --timeout=5s \
+HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
     CMD curl -fSs http://localhost:8008/health || exit 1
diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv
index 0d74630370..017be8555e 100644
--- a/docker/Dockerfile-dhvirtualenv
+++ b/docker/Dockerfile-dhvirtualenv
@@ -15,6 +15,15 @@ ARG distro=""
 ###
 ### Stage 0: build a dh-virtualenv
 ###
+
+# This is only really needed on bionic and focal, since other distributions we
+# care about have a recent version of dh-virtualenv by default. Unfortunately,
+# it looks like focal is going to be with us for a while.
+#
+# (focal doesn't have a dh-virtualenv package at all. There is a PPA at
+# https://launchpad.net/~jyrki-pulliainen/+archive/ubuntu/dh-virtualenv, but
+# it's not obviously easier to use that than to build our own.)
+
 FROM ${distro} as builder
 
 RUN apt-get update -qq -o Acquire::Languages=none
@@ -27,7 +36,7 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
         wget
 
 # fetch and unpack the package
-# TODO: Upgrade to 1.2.2 once xenial is dropped
+# TODO: Upgrade to 1.2.2 once bionic is dropped (1.2.2 requires debhelper 12; bionic has only 11)
 RUN mkdir /dh-virtualenv
 RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/ac6e1b1.tar.gz
 RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz
@@ -59,8 +68,6 @@ ENV LANG C.UTF-8
 #
 # NB: keep this list in sync with the list of build-deps in debian/control
 # TODO: it would be nice to do that automatically.
-# TODO: Remove the dh-systemd stanza after dropping support for Ubuntu xenial
-#       it's a transitional package on all other, more recent releases
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
@@ -76,10 +83,7 @@ RUN apt-get update -qq -o Acquire::Languages=none \
         python3-venv \
         sqlite3 \
         libpq-dev \
-        xmlsec1 \
-    && ( env DEBIAN_FRONTEND=noninteractive apt-get install \
-         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
-         dh-systemd || true )
+        xmlsec1
 
 COPY --from=builder /dh-virtualenv_1.2~dev-1_all.deb /
 
diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers
new file mode 100644
index 0000000000..969cf97286
--- /dev/null
+++ b/docker/Dockerfile-workers
@@ -0,0 +1,23 @@
+# Inherit from the official Synapse docker image
+FROM matrixdotorg/synapse
+
+# Install deps
+RUN apt-get update
+RUN apt-get install -y supervisor redis nginx
+
+# Remove the default nginx sites
+RUN rm /etc/nginx/sites-enabled/default
+
+# Copy Synapse worker, nginx and supervisord configuration template files
+COPY ./docker/conf-workers/* /conf/
+
+# Expose nginx listener port
+EXPOSE 8080/tcp
+
+# Volume for user-editable config files, logs etc.
+VOLUME ["/data"]
+
+# A script to read environment variables and create the necessary
+# files to run the desired worker configuration. Will start supervisord.
+COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
+ENTRYPOINT ["/configure_workers_and_start.py"]
diff --git a/docker/README-testing.md b/docker/README-testing.md
new file mode 100644
index 0000000000..6a5baf9e28
--- /dev/null
+++ b/docker/README-testing.md
@@ -0,0 +1,140 @@
+# Running tests against a dockerised Synapse
+
+It's possible to run integration tests against Synapse
+using [Complement](https://github.com/matrix-org/complement). Complement is a Matrix Spec
+compliance test suite for homeservers, and supports any homeserver docker image configured
+to listen on ports 8008/8448. This document contains instructions for building Synapse
+docker images that can be run inside Complement for testing purposes.
+
+Note that running Synapse's unit tests from within the docker image is not supported.
+
+## Testing with SQLite and single-process Synapse
+
+> Note that `scripts-dev/complement.sh` is a script that will automatically build 
+> and run an SQLite-based, single-process of Synapse against Complement.
+
+The instructions below will set up Complement testing for a single-process, 
+SQLite-based Synapse deployment.
+
+Start by building the base Synapse docker image. If you wish to run tests with the latest
+release of Synapse, instead of your current checkout, you can skip this step. From the
+root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+```
+
+This will build an image with the tag `matrixdotorg/synapse`.
+
+Next, build the Synapse image for Complement. You will need a local checkout 
+of Complement. Change to the root of your Complement checkout and run:
+
+```sh
+docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
+```
+
+This will build an image with the tag `complement-synapse`, which can be handed to 
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
+how to run the tests, as well as the various available command line flags.
+
+## Testing with PostgreSQL and single or multi-process Synapse
+
+The above docker image only supports running Synapse with SQLite and in a 
+single-process topology. The following instructions are used to build a Synapse image for 
+Complement that supports either single or multi-process topology with a PostgreSQL 
+database backend.
+
+As with the single-process image, build the base Synapse docker image. If you wish to run
+tests with the latest release of Synapse, instead of your current checkout, you can skip
+this step. From the root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+```
+
+This will build an image with the tag `matrixdotorg/synapse`.
+
+Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
+Again, from the root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
+```
+
+This will build an image with the tag` matrixdotorg/synapse-workers`.
+
+It's worth noting at this point that this image is fully functional, and 
+can be used for testing against locally. See instructions for using the container 
+under
+[Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
+below.
+
+Finally, build the Synapse image for Complement, which is based on
+`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
+the root of your Complement checkout and run:
+
+```sh
+docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
+```
+
+This will build an image with the tag `complement-synapse`, which can be handed to
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for
+how to run the tests, as well as the various available command line flags.
+
+## Running the Dockerfile-worker image standalone
+
+For manual testing of a multi-process Synapse instance in Docker,
+[Dockerfile-workers](Dockerfile-workers) is a Dockerfile that will produce an image
+bundling all necessary components together for a workerised homeserver instance.
+
+This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
+a redis for worker communication and a supervisord instance to start up and monitor all
+processes. You will need to provide your own postgres container to connect to, and TLS 
+is not handled by the container.
+
+Once you've built the image using the above instructions, you can run it. Be sure 
+you've set up a volume according to the [usual Synapse docker instructions](README.md).
+Then run something along the lines of:
+
+```
+docker run -d --name synapse \
+    --mount type=volume,src=synapse-data,dst=/data \
+    -p 8008:8008 \
+    -e SYNAPSE_SERVER_NAME=my.matrix.host \
+    -e SYNAPSE_REPORT_STATS=no \
+    -e POSTGRES_HOST=postgres \
+    -e POSTGRES_USER=postgres \
+    -e POSTGRES_PASSWORD=somesecret \
+    -e SYNAPSE_WORKER_TYPES=synchrotron,media_repository,user_dir \
+    -e SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1 \
+    matrixdotorg/synapse-workers
+```
+
+...substituting `POSTGRES*` variables for those that match a postgres host you have 
+available (usually a running postgres docker container).
+
+The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
+use when running the container. All possible worker names are defined by the keys of the
+`WORKERS_CONFIG` variable in [this script](configure_workers_and_start.py), which the
+Dockerfile makes use of to generate appropriate worker, nginx and supervisord config
+files.
+
+Sharding is supported for a subset of workers, in line with the
+[worker documentation](../docs/workers.md). To run multiple instances of a given worker
+type, simply specify the type multiple times in `SYNAPSE_WORKER_TYPES`
+(e.g `SYNAPSE_WORKER_TYPES=event_creator,event_creator...`).
+
+Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no workers
+(leaving only the main process). The container is configured to use redis-based worker
+mode.
+
+Logs for workers and the main process are logged to stdout and can be viewed with 
+standard `docker logs` tooling. Worker logs contain their worker name 
+after the timestamp.
+
+Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
+`<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
+00, according to the container's clock. Logging for the main process must still be 
+configured by modifying the homeserver's log config in your Synapse data volume.
diff --git a/docker/README.md b/docker/README.md
index 3a7dc585e7..edf917bb11 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -2,13 +2,16 @@
 
 This Docker image will run Synapse as a single process. By default it uses a
 sqlite database; for production use you should connect it to a separate
-postgres database.
+postgres database. The image also does *not* provide a TURN server.
 
-The image also does *not* provide a TURN server.
+This image should work on all platforms that are supported by Docker upstream.
+Note that Docker's WS1-backend Linux Containers on Windows
+platform is [experimental](https://github.com/docker/for-win/issues/6470) and
+is not supported by this image.
 
 ## Volumes
 
-By default, the image expects a single volume, located at ``/data``, that will hold:
+By default, the image expects a single volume, located at `/data`, that will hold:
 
 * configuration files;
 * uploaded media and thumbnails;
@@ -16,11 +19,11 @@ By default, the image expects a single volume, located at ``/data``, that will h
 * the appservices configuration.
 
 You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, ``/data/media`` could be stored on a large but low
+disposal. For instance, `/data/media` could be stored on a large but low
 performance hdd storage while other files could be stored on high performance
 endpoints.
 
-In order to setup an application service, simply create an ``appservices``
+In order to setup an application service, simply create an `appservices`
 directory in the data volume and write the application service Yaml
 configuration file there. Multiple application services are supported.
 
@@ -42,7 +45,7 @@ docker run -it --rm \
 ```
 
 For information on picking a suitable server name, see
-https://github.com/matrix-org/synapse/blob/master/INSTALL.md.
+https://matrix-org.github.io/synapse/latest/setup/installation.html.
 
 The above command will generate a `homeserver.yaml` in (typically)
 `/var/lib/docker/volumes/synapse-data/_data`. You should check this file, and
@@ -53,6 +56,8 @@ The following environment variables are supported in `generate` mode:
 * `SYNAPSE_SERVER_NAME` (mandatory): the server public hostname.
 * `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable
   anonymous statistics reporting.
+* `SYNAPSE_HTTP_PORT`: the port Synapse should listen on for http traffic.
+      Defaults to `8008`.
 * `SYNAPSE_CONFIG_DIR`: where additional config files (such as the log config
   and event signing key) will be stored. Defaults to `/data`.
 * `SYNAPSE_CONFIG_PATH`: path to the file to be generated. Defaults to
@@ -73,6 +78,8 @@ docker run -d --name synapse \
     matrixdotorg/synapse:latest
 ```
 
+(assuming 8008 is the port Synapse is configured to listen on for http traffic.)
+
 You can then check that it has started correctly with:
 
 ```
@@ -132,7 +139,7 @@ For documentation on using a reverse proxy, see
 https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md.
 
 For more information on enabling TLS support in synapse itself, see
-https://github.com/matrix-org/synapse/blob/master/INSTALL.md#tls-certificates. Of
+https://matrix-org.github.io/synapse/latest/setup/installation.html#tls-certificates. Of
 course, you will need to expose the TLS port from the container with a `-p`
 argument to `docker run`.
 
@@ -184,6 +191,16 @@ whilst running the above `docker run` commands.
 ```
    --no-healthcheck
 ```
+
+## Disabling the healthcheck in docker-compose file
+
+If you wish to disable the healthcheck via docker-compose, append the following to your service configuration.
+
+```
+  healthcheck:
+    disable: true
+```
+
 ## Setting custom healthcheck on docker run
 
 If you wish to point the healthcheck at a different port with docker command, add the following
@@ -195,17 +212,18 @@ If you wish to point the healthcheck at a different port with docker command, ad
 ## Setting the healthcheck in docker-compose file
 
 You can add the following to set a custom healthcheck in a docker compose file.
-You will need version >2.1 for this to work. 
+You will need docker-compose version >2.1 for this to work. 
 
 ```
 healthcheck:
   test: ["CMD", "curl", "-fSs", "http://localhost:8008/health"]
-  interval: 1m
-  timeout: 10s
+  interval: 15s
+  timeout: 5s
   retries: 3
+  start_period: 5s
 ```
 
 ## Using jemalloc
 
 Jemalloc is embedded in the image and will be used instead of the default allocator.
-You can read about jemalloc by reading the Synapse [README](../README.md)
\ No newline at end of file
+You can read about jemalloc by reading the Synapse [README](../README.rst).
diff --git a/docker/build_debian.sh b/docker/build_debian.sh
index f426d2b77b..801ff45471 100644
--- a/docker/build_debian.sh
+++ b/docker/build_debian.sh
@@ -11,6 +11,19 @@ DIST=`cut -d ':' -f2 <<< $distro`
 cp -aT /synapse/source /synapse/build
 cd /synapse/build
 
+# if this is a prerelease, set the Section accordingly.
+#
+# When the package is later added to the package repo, reprepro will use the
+# Section to determine which "component" it should go into (see
+# https://manpages.debian.org/stretch/reprepro/reprepro.1.en.html#GUESSING)
+
+DEB_VERSION=`dpkg-parsechangelog -SVersion`
+case $DEB_VERSION in
+    *~rc*|*~a*|*~b*|*~c*)
+        sed -ie '/^Section:/c\Section: prerelease' debian/control
+        ;;
+esac
+
 # add an entry to the changelog for this distribution
 dch -M -l "+$DIST" "build for $DIST"
 dch -M -r "" --force-distribution --distribution "$DIST"
diff --git a/docker/conf-workers/nginx.conf.j2 b/docker/conf-workers/nginx.conf.j2
new file mode 100644
index 0000000000..1081979e06
--- /dev/null
+++ b/docker/conf-workers/nginx.conf.j2
@@ -0,0 +1,27 @@
+# This file contains the base config for the reverse proxy, as part of ../Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+
+{{ upstream_directives }}
+
+server {
+    # Listen on an unoccupied port number
+    listen 8008;
+    listen [::]:8008;
+
+    server_name localhost;
+
+    # Nginx by default only allows file uploads up to 1M in size
+    # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+    client_max_body_size 100M;
+
+{{ worker_locations }}
+
+    # Send all other traffic to the main process
+    location ~* ^(\\/_matrix|\\/_synapse) {
+        proxy_pass http://localhost:8080;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host $host;
+    }
+}
diff --git a/docker/conf-workers/shared.yaml.j2 b/docker/conf-workers/shared.yaml.j2
new file mode 100644
index 0000000000..f94b8c6aca
--- /dev/null
+++ b/docker/conf-workers/shared.yaml.j2
@@ -0,0 +1,9 @@
+# This file contains the base for the shared homeserver config file between Synapse workers,
+# as part of ./Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+
+redis:
+    enabled: true
+
+{{ shared_worker_config }}
\ No newline at end of file
diff --git a/docker/conf-workers/supervisord.conf.j2 b/docker/conf-workers/supervisord.conf.j2
new file mode 100644
index 0000000000..0de2c6143b
--- /dev/null
+++ b/docker/conf-workers/supervisord.conf.j2
@@ -0,0 +1,41 @@
+# This file contains the base config for supervisord, as part of ../Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+[supervisord]
+nodaemon=true
+user=root
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+priority=500
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+username=www-data
+autorestart=true
+
+[program:redis]
+command=/usr/bin/redis-server /etc/redis/redis.conf --daemonize no
+priority=1
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+username=redis
+autorestart=true
+
+[program:synapse_main]
+command=/usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml
+priority=10
+# Log startup failures to supervisord's stdout/err
+# Regular synapse logs will still go in the configured data directory
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=unexpected
+exitcodes=0
+
+# Additional process blocks
+{{ worker_config }}
\ No newline at end of file
diff --git a/docker/conf-workers/worker.yaml.j2 b/docker/conf-workers/worker.yaml.j2
new file mode 100644
index 0000000000..42131afc95
--- /dev/null
+++ b/docker/conf-workers/worker.yaml.j2
@@ -0,0 +1,26 @@
+# This is a configuration template for a single worker instance, and is
+# used by Dockerfile-workers.
+# Values will be change depending on whichever workers are selected when
+# running that image.
+
+worker_app: "{{ app }}"
+worker_name: "{{ name }}"
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_listeners:
+  - type: http
+    port: {{ port }}
+{% if listener_resources %}
+    resources:
+      - names:
+{%- for resource in listener_resources %}
+        - {{ resource }}
+{%- endfor %}
+{% endif %}
+
+worker_log_config: {{ worker_log_config_filepath }}
+
+{{ worker_extra_conf }}
diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml
index 0dea62a87d..3cba594d02 100644
--- a/docker/conf/homeserver.yaml
+++ b/docker/conf/homeserver.yaml
@@ -7,12 +7,6 @@
 tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt"
 tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key"
 
-{% if SYNAPSE_ACME %}
-acme:
-    enabled: true
-    port: 8009
-{% endif %}
-
 {% endif %}
 
 ## Server ##
@@ -40,7 +34,9 @@ listeners:
         compress: false
   {% endif %}
 
-  - port: 8008
+  # Allow configuring in case we want to reverse proxy 8008
+  # using another process in the same container
+  - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
     tls: false
     bind_addresses: ['::']
     type: http
@@ -173,18 +169,10 @@ report_stats: False
 
 ## API Configuration ##
 
-room_invite_state_types:
-    - "m.room.join_rules"
-    - "m.room.canonical_alias"
-    - "m.room.avatar"
-    - "m.room.name"
-
 {% if SYNAPSE_APPSERVICES %}
 app_service_config_files:
 {% for appservice in SYNAPSE_APPSERVICES %}    - "{{ appservice }}"
 {% endfor %}
-{% else %}
-app_service_config_files: []
 {% endif %}
 
 macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}"
diff --git a/docker/conf/log.config b/docker/conf/log.config
index 491bbcc87a..7a216a36a0 100644
--- a/docker/conf/log.config
+++ b/docker/conf/log.config
@@ -2,9 +2,49 @@ version: 1
 
 formatters:
   precise:
-   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% if worker_name %}
+    format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% else %}
+    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% endif %}
 
 handlers:
+{% if LOG_FILE_PATH %}
+  file:
+    class: logging.handlers.TimedRotatingFileHandler
+    formatter: precise
+    filename: {{ LOG_FILE_PATH }}
+    when: "midnight"
+    backupCount: 6  # Does not include the current log file.
+    encoding: utf8
+
+  # Default to buffering writes to log file for efficiency.
+  # WARNING/ERROR logs will still be flushed immediately, but there will be a
+  # delay (of up to `period` seconds, or until the buffer is full with
+  # `capacity` messages) before INFO/DEBUG logs get written.
+  buffer:
+    class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler
+    target: file
+
+    # The capacity is the maximum number of log lines that are buffered
+    # before being written to disk. Increasing this will lead to better
+    # performance, at the expensive of it taking longer for log lines to
+    # be written to disk.
+    # This parameter is required.
+    capacity: 10
+
+    # Logs with a level at or above the flush level will cause the buffer to
+    # be flushed immediately.
+    # Default value: 40 (ERROR)
+    # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG)
+    flushLevel: 30  # Flush immediately for WARNING logs and higher
+
+    # The period of time, in seconds, between forced flushes.
+    # Messages will not be delayed for longer than this time.
+    # Default value: 5 seconds
+    period: 5
+{% endif %}
+
   console:
     class: logging.StreamHandler
     formatter: precise
@@ -17,6 +57,11 @@ loggers:
 
 root:
     level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
+
+{% if LOG_FILE_PATH %}
+    handlers: [console, buffer]
+{% else %}
     handlers: [console]
+{% endif %}
 
 disable_existing_loggers: false
diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py
new file mode 100755
index 0000000000..1d22a4d571
--- /dev/null
+++ b/docker/configure_workers_and_start.py
@@ -0,0 +1,558 @@
+#!/usr/bin/env python
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script reads environment variables and generates a shared Synapse worker,
+# nginx and supervisord configs depending on the workers requested.
+#
+# The environment variables it reads are:
+#   * SYNAPSE_SERVER_NAME: The desired server_name of the homeserver.
+#   * SYNAPSE_REPORT_STATS: Whether to report stats.
+#   * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG
+#         below. Leave empty for no workers, or set to '*' for all possible workers.
+#
+# NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
+# in the project's README), this script may be run multiple times, and functionality should
+# continue to work if so.
+
+import os
+import subprocess
+import sys
+
+import jinja2
+import yaml
+
+MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
+
+
+WORKERS_CONFIG = {
+    "pusher": {
+        "app": "synapse.app.pusher",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {"start_pushers": False},
+        "worker_extra_conf": "",
+    },
+    "user_dir": {
+        "app": "synapse.app.user_dir",
+        "listener_resources": ["client"],
+        "endpoint_patterns": [
+            "^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$"
+        ],
+        "shared_extra_conf": {"update_user_directory": False},
+        "worker_extra_conf": "",
+    },
+    "media_repository": {
+        "app": "synapse.app.media_repository",
+        "listener_resources": ["media"],
+        "endpoint_patterns": [
+            "^/_matrix/media/",
+            "^/_synapse/admin/v1/purge_media_cache$",
+            "^/_synapse/admin/v1/room/.*/media.*$",
+            "^/_synapse/admin/v1/user/.*/media.*$",
+            "^/_synapse/admin/v1/media/.*$",
+            "^/_synapse/admin/v1/quarantine_media/.*$",
+        ],
+        "shared_extra_conf": {"enable_media_repo": False},
+        "worker_extra_conf": "enable_media_repo: true",
+    },
+    "appservice": {
+        "app": "synapse.app.appservice",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {"notify_appservices": False},
+        "worker_extra_conf": "",
+    },
+    "federation_sender": {
+        "app": "synapse.app.federation_sender",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {"send_federation": False},
+        "worker_extra_conf": "",
+    },
+    "synchrotron": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["client"],
+        "endpoint_patterns": [
+            "^/_matrix/client/(v2_alpha|r0)/sync$",
+            "^/_matrix/client/(api/v1|v2_alpha|r0)/events$",
+            "^/_matrix/client/(api/v1|r0)/initialSync$",
+            "^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$",
+        ],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "federation_reader": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["federation"],
+        "endpoint_patterns": [
+            "^/_matrix/federation/(v1|v2)/event/",
+            "^/_matrix/federation/(v1|v2)/state/",
+            "^/_matrix/federation/(v1|v2)/state_ids/",
+            "^/_matrix/federation/(v1|v2)/backfill/",
+            "^/_matrix/federation/(v1|v2)/get_missing_events/",
+            "^/_matrix/federation/(v1|v2)/publicRooms",
+            "^/_matrix/federation/(v1|v2)/query/",
+            "^/_matrix/federation/(v1|v2)/make_join/",
+            "^/_matrix/federation/(v1|v2)/make_leave/",
+            "^/_matrix/federation/(v1|v2)/send_join/",
+            "^/_matrix/federation/(v1|v2)/send_leave/",
+            "^/_matrix/federation/(v1|v2)/invite/",
+            "^/_matrix/federation/(v1|v2)/query_auth/",
+            "^/_matrix/federation/(v1|v2)/event_auth/",
+            "^/_matrix/federation/(v1|v2)/exchange_third_party_invite/",
+            "^/_matrix/federation/(v1|v2)/user/devices/",
+            "^/_matrix/federation/(v1|v2)/get_groups_publicised$",
+            "^/_matrix/key/v2/query",
+        ],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "federation_inbound": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["federation"],
+        "endpoint_patterns": ["/_matrix/federation/(v1|v2)/send/"],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "event_persister": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["replication"],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "background_worker": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        # This worker cannot be sharded. Therefore there should only ever be one background
+        # worker, and it should be named background_worker1
+        "shared_extra_conf": {"run_background_tasks_on": "background_worker1"},
+        "worker_extra_conf": "",
+    },
+    "event_creator": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["client"],
+        "endpoint_patterns": [
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact",
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send",
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
+            "^/_matrix/client/(api/v1|r0|unstable)/join/",
+            "^/_matrix/client/(api/v1|r0|unstable)/profile/",
+        ],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "frontend_proxy": {
+        "app": "synapse.app.frontend_proxy",
+        "listener_resources": ["client", "replication"],
+        "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|unstable)/keys/upload"],
+        "shared_extra_conf": {},
+        "worker_extra_conf": (
+            "worker_main_http_uri: http://127.0.0.1:%d"
+            % (MAIN_PROCESS_HTTP_LISTENER_PORT,),
+        ),
+    },
+}
+
+# Templates for sections that may be inserted multiple times in config files
+SUPERVISORD_PROCESS_CONFIG_BLOCK = """
+[program:synapse_{name}]
+command=/usr/local/bin/python -m {app} \
+    --config-path="{config_path}" \
+    --config-path=/conf/workers/shared.yaml \
+    --config-path=/conf/workers/{name}.yaml
+autorestart=unexpected
+priority=500
+exitcodes=0
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+"""
+
+NGINX_LOCATION_CONFIG_BLOCK = """
+    location ~* {endpoint} {{
+        proxy_pass {upstream};
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host $host;
+    }}
+"""
+
+NGINX_UPSTREAM_CONFIG_BLOCK = """
+upstream {upstream_worker_type} {{
+{body}
+}}
+"""
+
+
+# Utility functions
+def log(txt: str):
+    """Log something to the stdout.
+
+    Args:
+        txt: The text to log.
+    """
+    print(txt)
+
+
+def error(txt: str):
+    """Log something and exit with an error code.
+
+    Args:
+        txt: The text to log in error.
+    """
+    log(txt)
+    sys.exit(2)
+
+
+def convert(src: str, dst: str, **template_vars):
+    """Generate a file from a template
+
+    Args:
+        src: Path to the input file.
+        dst: Path to write to.
+        template_vars: The arguments to replace placeholder variables in the template with.
+    """
+    # Read the template file
+    with open(src) as infile:
+        template = infile.read()
+
+    # Generate a string from the template. We disable autoescape to prevent template
+    # variables from being escaped.
+    rendered = jinja2.Template(template, autoescape=False).render(**template_vars)
+
+    # Write the generated contents to a file
+    #
+    # We use append mode in case the files have already been written to by something else
+    # (for instance, as part of the instructions in a dockerfile).
+    with open(dst, "a") as outfile:
+        # In case the existing file doesn't end with a newline
+        outfile.write("\n")
+
+        outfile.write(rendered)
+
+
+def add_sharding_to_shared_config(
+    shared_config: dict,
+    worker_type: str,
+    worker_name: str,
+    worker_port: int,
+) -> None:
+    """Given a dictionary representing a config file shared across all workers,
+    append sharded worker information to it for the current worker_type instance.
+
+    Args:
+        shared_config: The config dict that all worker instances share (after being converted to YAML)
+        worker_type: The type of worker (one of those defined in WORKERS_CONFIG).
+        worker_name: The name of the worker instance.
+        worker_port: The HTTP replication port that the worker instance is listening on.
+    """
+    # The instance_map config field marks the workers that write to various replication streams
+    instance_map = shared_config.setdefault("instance_map", {})
+
+    # Worker-type specific sharding config
+    if worker_type == "pusher":
+        shared_config.setdefault("pusher_instances", []).append(worker_name)
+
+    elif worker_type == "federation_sender":
+        shared_config.setdefault("federation_sender_instances", []).append(worker_name)
+
+    elif worker_type == "event_persister":
+        # Event persisters write to the events stream, so we need to update
+        # the list of event stream writers
+        shared_config.setdefault("stream_writers", {}).setdefault("events", []).append(
+            worker_name
+        )
+
+        # Map of stream writer instance names to host/ports combos
+        instance_map[worker_name] = {
+            "host": "localhost",
+            "port": worker_port,
+        }
+
+    elif worker_type == "media_repository":
+        # The first configured media worker will run the media background jobs
+        shared_config.setdefault("media_instance_running_background_jobs", worker_name)
+
+
+def generate_base_homeserver_config():
+    """Starts Synapse and generates a basic homeserver config, which will later be
+    modified for worker support.
+
+    Raises: CalledProcessError if calling start.py returned a non-zero exit code.
+    """
+    # start.py already does this for us, so just call that.
+    # note that this script is copied in in the official, monolith dockerfile
+    os.environ["SYNAPSE_HTTP_PORT"] = str(MAIN_PROCESS_HTTP_LISTENER_PORT)
+    subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
+
+
+def generate_worker_files(environ, config_path: str, data_dir: str):
+    """Read the desired list of workers from environment variables and generate
+    shared homeserver, nginx and supervisord configs.
+
+    Args:
+        environ: _Environ[str]
+        config_path: Where to output the generated Synapse main worker config file.
+        data_dir: The location of the synapse data directory. Where log and
+            user-facing config files live.
+    """
+    # Note that yaml cares about indentation, so care should be taken to insert lines
+    # into files at the correct indentation below.
+
+    # shared_config is the contents of a Synapse config file that will be shared amongst
+    # the main Synapse process as well as all workers.
+    # It is intended mainly for disabling functionality when certain workers are spun up,
+    # and adding a replication listener.
+
+    # First read the original config file and extract the listeners block. Then we'll add
+    # another listener for replication. Later we'll write out the result.
+    listeners = [
+        {
+            "port": 9093,
+            "bind_address": "127.0.0.1",
+            "type": "http",
+            "resources": [{"names": ["replication"]}],
+        }
+    ]
+    with open(config_path) as file_stream:
+        original_config = yaml.safe_load(file_stream)
+        original_listeners = original_config.get("listeners")
+        if original_listeners:
+            listeners += original_listeners
+
+    # The shared homeserver config. The contents of which will be inserted into the
+    # base shared worker jinja2 template.
+    #
+    # This config file will be passed to all workers, included Synapse's main process.
+    shared_config = {"listeners": listeners}
+
+    # The supervisord config. The contents of which will be inserted into the
+    # base supervisord jinja2 template.
+    #
+    # Supervisord will be in charge of running everything, from redis to nginx to Synapse
+    # and all of its worker processes. Load the config template, which defines a few
+    # services that are necessary to run.
+    supervisord_config = ""
+
+    # Upstreams for load-balancing purposes. This dict takes the form of a worker type to the
+    # ports of each worker. For example:
+    # {
+    #   worker_type: {1234, 1235, ...}}
+    # }
+    # and will be used to construct 'upstream' nginx directives.
+    nginx_upstreams = {}
+
+    # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
+    # placed after the proxy_pass directive. The main benefit to representing this data as a
+    # dict over a str is that we can easily deduplicate endpoints across multiple instances
+    # of the same worker.
+    #
+    # An nginx site config that will be amended to depending on the workers that are
+    # spun up. To be placed in /etc/nginx/conf.d.
+    nginx_locations = {}
+
+    # Read the desired worker configuration from the environment
+    worker_types = environ.get("SYNAPSE_WORKER_TYPES")
+    if worker_types is None:
+        # No workers, just the main process
+        worker_types = []
+    else:
+        # Split type names by comma
+        worker_types = worker_types.split(",")
+
+    # Create the worker configuration directory if it doesn't already exist
+    os.makedirs("/conf/workers", exist_ok=True)
+
+    # Start worker ports from this arbitrary port
+    worker_port = 18009
+
+    # A counter of worker_type -> int. Used for determining the name for a given
+    # worker type when generating its config file, as each worker's name is just
+    # worker_type + instance #
+    worker_type_counter = {}
+
+    # For each worker type specified by the user, create config values
+    for worker_type in worker_types:
+        worker_type = worker_type.strip()
+
+        worker_config = WORKERS_CONFIG.get(worker_type)
+        if worker_config:
+            worker_config = worker_config.copy()
+        else:
+            log(worker_type + " is an unknown worker type! It will be ignored")
+            continue
+
+        new_worker_count = worker_type_counter.setdefault(worker_type, 0) + 1
+        worker_type_counter[worker_type] = new_worker_count
+
+        # Name workers by their type concatenated with an incrementing number
+        # e.g. federation_reader1
+        worker_name = worker_type + str(new_worker_count)
+        worker_config.update(
+            {"name": worker_name, "port": worker_port, "config_path": config_path}
+        )
+
+        # Update the shared config with any worker-type specific options
+        shared_config.update(worker_config["shared_extra_conf"])
+
+        # Check if more than one instance of this worker type has been specified
+        worker_type_total_count = worker_types.count(worker_type)
+        if worker_type_total_count > 1:
+            # Update the shared config with sharding-related options if necessary
+            add_sharding_to_shared_config(
+                shared_config, worker_type, worker_name, worker_port
+            )
+
+        # Enable the worker in supervisord
+        supervisord_config += SUPERVISORD_PROCESS_CONFIG_BLOCK.format_map(worker_config)
+
+        # Add nginx location blocks for this worker's endpoints (if any are defined)
+        for pattern in worker_config["endpoint_patterns"]:
+            # Determine whether we need to load-balance this worker
+            if worker_type_total_count > 1:
+                # Create or add to a load-balanced upstream for this worker
+                nginx_upstreams.setdefault(worker_type, set()).add(worker_port)
+
+                # Upstreams are named after the worker_type
+                upstream = "http://" + worker_type
+            else:
+                upstream = "http://localhost:%d" % (worker_port,)
+
+            # Note that this endpoint should proxy to this upstream
+            nginx_locations[pattern] = upstream
+
+        # Write out the worker's logging config file
+
+        # Check whether we should write worker logs to disk, in addition to the console
+        extra_log_template_args = {}
+        if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
+            extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
+                dir=data_dir, name=worker_name
+            )
+
+        # Render and write the file
+        log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
+        convert(
+            "/conf/log.config",
+            log_config_filepath,
+            worker_name=worker_name,
+            **extra_log_template_args,
+        )
+
+        # Then a worker config file
+        convert(
+            "/conf/worker.yaml.j2",
+            "/conf/workers/{name}.yaml".format(name=worker_name),
+            **worker_config,
+            worker_log_config_filepath=log_config_filepath,
+        )
+
+        worker_port += 1
+
+    # Build the nginx location config blocks
+    nginx_location_config = ""
+    for endpoint, upstream in nginx_locations.items():
+        nginx_location_config += NGINX_LOCATION_CONFIG_BLOCK.format(
+            endpoint=endpoint,
+            upstream=upstream,
+        )
+
+    # Determine the load-balancing upstreams to configure
+    nginx_upstream_config = ""
+    for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
+        body = ""
+        for port in upstream_worker_ports:
+            body += "    server localhost:%d;\n" % (port,)
+
+        # Add to the list of configured upstreams
+        nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
+            upstream_worker_type=upstream_worker_type,
+            body=body,
+        )
+
+    # Finally, we'll write out the config files.
+
+    # Shared homeserver config
+    convert(
+        "/conf/shared.yaml.j2",
+        "/conf/workers/shared.yaml",
+        shared_worker_config=yaml.dump(shared_config),
+    )
+
+    # Nginx config
+    convert(
+        "/conf/nginx.conf.j2",
+        "/etc/nginx/conf.d/matrix-synapse.conf",
+        worker_locations=nginx_location_config,
+        upstream_directives=nginx_upstream_config,
+    )
+
+    # Supervisord config
+    convert(
+        "/conf/supervisord.conf.j2",
+        "/etc/supervisor/conf.d/supervisord.conf",
+        main_config_path=config_path,
+        worker_config=supervisord_config,
+    )
+
+    # Ensure the logging directory exists
+    log_dir = data_dir + "/logs"
+    if not os.path.exists(log_dir):
+        os.mkdir(log_dir)
+
+
+def start_supervisord():
+    """Starts up supervisord which then starts and monitors all other necessary processes
+
+    Raises: CalledProcessError if calling start.py return a non-zero exit code.
+    """
+    subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
+
+
+def main(args, environ):
+    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
+    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
+    data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
+
+    # override SYNAPSE_NO_TLS, we don't support TLS in worker mode,
+    # this needs to be handled by a frontend proxy
+    environ["SYNAPSE_NO_TLS"] = "yes"
+
+    # Generate the base homeserver config if one does not yet exist
+    if not os.path.exists(config_path):
+        log("Generating base homeserver config")
+        generate_base_homeserver_config()
+
+    # This script may be run multiple times (mostly by Complement, see note at top of file).
+    # Don't re-configure workers in this instance.
+    mark_filepath = "/conf/workers_have_been_configured"
+    if not os.path.exists(mark_filepath):
+        # Always regenerate all other config files
+        generate_worker_files(environ, config_path, data_dir)
+
+        # Mark workers as being configured
+        with open(mark_filepath, "w") as f:
+            f.write("")
+
+    # Start supervisord, which will start Synapse, all of the configured worker
+    # processes, redis, nginx etc. according to the config we created above.
+    start_supervisord()
+
+
+if __name__ == "__main__":
+    main(sys.argv, os.environ)
diff --git a/docs/.sample_config_header.yaml b/docs/.sample_config_header.yaml
index 8c9b31acdb..09e86ca0ca 100644
--- a/docs/.sample_config_header.yaml
+++ b/docs/.sample_config_header.yaml
@@ -8,7 +8,8 @@
 #
 # It is *not* intended to be copied and used as the basis for a real
 # homeserver.yaml. Instead, if you are starting from scratch, please generate
-# a fresh config using Synapse by following the instructions in INSTALL.md.
+# a fresh config using Synapse by following the instructions in
+# https://matrix-org.github.io/synapse/latest/setup/installation.html.
 
 # Configuration options that take a time period can be set using a number
 # followed by a letter. Letters have the following meanings:
diff --git a/docs/ACME.md b/docs/ACME.md
deleted file mode 100644
index a7a498f575..0000000000
--- a/docs/ACME.md
+++ /dev/null
@@ -1,161 +0,0 @@
-# ACME
-
-From version 1.0 (June 2019) onwards, Synapse requires valid TLS
-certificates for communication between servers (by default on port
-`8448`) in addition to those that are client-facing (port `443`). To
-help homeserver admins fulfil this new requirement, Synapse v0.99.0
-introduced support for automatically provisioning certificates through 
-[Let's Encrypt](https://letsencrypt.org/) using the ACME protocol.
-
-## Deprecation of ACME v1
-
-In [March 2019](https://community.letsencrypt.org/t/end-of-life-plan-for-acmev1/88430),
-Let's Encrypt announced that they were deprecating version 1 of the ACME
-protocol, with the plan to disable the use of it for new accounts in
-November 2019, for new domains in June 2020, and for existing accounts and
-domains in June 2021.
-
-Synapse doesn't currently support version 2 of the ACME protocol, which
-means that:
-
-* for existing installs, Synapse's built-in ACME support will continue
-  to work until June 2021.
-* for new installs, this feature will not work at all.
-
-Either way, it is recommended to move from Synapse's ACME support
-feature to an external automated tool such as [certbot](https://github.com/certbot/certbot)
-(or browse [this list](https://letsencrypt.org/fr/docs/client-options/)
-for an alternative ACME client).
-
-It's also recommended to use a reverse proxy for the server-facing
-communications (more documentation about this can be found
-[here](/docs/reverse_proxy.md)) as well as the client-facing ones and
-have it serve the certificates.
-
-In case you can't do that and need Synapse to serve them itself, make
-sure to set the `tls_certificate_path` configuration setting to the path
-of the certificate (make sure to use the certificate containing the full
-certification chain, e.g. `fullchain.pem` if using certbot) and
-`tls_private_key_path` to the path of the matching private key. Note
-that in this case you will need to restart Synapse after each
-certificate renewal so that Synapse stops using the old certificate.
-
-If you still want to use Synapse's built-in ACME support, the rest of
-this document explains how to set it up. 
-
-## Initial setup 
-
-In the case that your `server_name` config variable is the same as
-the hostname that the client connects to, then the same certificate can be
-used between client and federation ports without issue.
-
-If your configuration file does not already have an `acme` section, you can
-generate an example config by running the `generate_config` executable. For
-example:
-
-```
-~/synapse/env3/bin/generate_config
-```
-
-You will need to provide Let's Encrypt (or another ACME provider) access to
-your Synapse ACME challenge responder on port 80, at the domain of your
-homeserver. This requires you to either change the port of the ACME listener
-provided by Synapse to a high port and reverse proxy to it, or use a tool
-like `authbind` to allow Synapse to listen on port 80 without root access.
-(Do not run Synapse with root permissions!) Detailed instructions are
-available under "ACME setup" below.
-
-If you already have certificates, you will need to back up or delete them
-(files `example.com.tls.crt` and `example.com.tls.key` in Synapse's root
-directory), Synapse's ACME implementation will not overwrite them.
-
-## ACME setup
-
-The main steps for enabling ACME support in short summary are:
-
-1. Allow Synapse to listen for incoming ACME challenges.
-1. Enable ACME support in `homeserver.yaml`.
-1. Move your old certificates (files `example.com.tls.crt` and `example.com.tls.key` out of the way if they currently exist at the paths specified in `homeserver.yaml`.
-1. Restart Synapse.
-
-Detailed instructions for each step are provided below.
-
-### Listening on port 80
-
-In order for Synapse to complete the ACME challenge to provision a
-certificate, it needs access to port 80. Typically listening on port 80 is
-only granted to applications running as root. There are thus two solutions to
-this problem.
-
-#### Using a reverse proxy
-
-A reverse proxy such as Apache or nginx allows a single process (the web
-server) to listen on port 80 and proxy traffic to the appropriate program
-running on your server. It is the recommended method for setting up ACME as
-it allows you to use your existing webserver while also allowing Synapse to
-provision certificates as needed.
-
-For nginx users, add the following line to your existing `server` block:
-
-```
-location /.well-known/acme-challenge {
-    proxy_pass http://localhost:8009;
-}
-```
-
-For Apache, add the following to your existing webserver config:
-
-```
-ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge
-```
-
-Make sure to restart/reload your webserver after making changes.
-
-Now make the relevant changes in `homeserver.yaml` to enable ACME support:
-
-```
-acme:
-    enabled: true
-    port: 8009
-```
-
-#### Authbind
-
-`authbind` allows a program which does not run as root to bind to
-low-numbered ports in a controlled way. The setup is simpler, but requires a
-webserver not to already be running on port 80. **This includes every time
-Synapse renews a certificate**, which may be cumbersome if you usually run a
-web server on port 80. Nevertheless, if you're sure port 80 is not being used
-for any other purpose then all that is necessary is the following:
-
-Install `authbind`. For example, on Debian/Ubuntu:
-
-```
-sudo apt-get install authbind
-```
-
-Allow `authbind` to bind port 80:
-
-```
-sudo touch /etc/authbind/byport/80
-sudo chmod 777 /etc/authbind/byport/80
-```
-
-When Synapse is started, use the following syntax:
-
-```
-authbind --deep <synapse start command>
-```
-
-Make the relevant changes in `homeserver.yaml` to enable ACME support:
-
-```
-acme:
-    enabled: true
-```
-
-### (Re)starting synapse
-
-Ensure that the certificate paths specified in `homeserver.yaml` (`tls_certificate_path` and `tls_private_key_path`) do not currently point to any files. Synapse will not provision certificates if files exist, as it does not want to overwrite existing certificates.
-
-Finally, start/restart Synapse.
diff --git a/docs/CAPTCHA_SETUP.md b/docs/CAPTCHA_SETUP.md
index 331e5d059a..fabdd7b726 100644
--- a/docs/CAPTCHA_SETUP.md
+++ b/docs/CAPTCHA_SETUP.md
@@ -1,31 +1,37 @@
 # Overview
-Captcha can be enabled for this home server. This file explains how to do that.
-The captcha mechanism used is Google's ReCaptcha. This requires API keys from Google.
-
-## Getting keys
-
-Requires a site/secret key pair from:
-
-<https://developers.google.com/recaptcha/>
-
-Must be a reCAPTCHA v2 key using the "I'm not a robot" Checkbox option
-
-## Setting ReCaptcha Keys
-
-The keys are a config option on the home server config. If they are not
-visible, you can generate them via `--generate-config`. Set the following value:
-
+A captcha can be enabled on your homeserver to help prevent bots from registering
+accounts. Synapse currently uses Google's reCAPTCHA service which requires API keys
+from Google.
+
+## Getting API keys
+
+1. Create a new site at <https://www.google.com/recaptcha/admin/create>
+1. Set the label to anything you want
+1. Set the type to reCAPTCHA v2 using the "I'm not a robot" Checkbox option.
+This is the only type of captcha that works with Synapse.
+1. Add the public hostname for your server, as set in `public_baseurl`
+in `homeserver.yaml`, to the list of authorized domains. If you have not set
+`public_baseurl`, use `server_name`.
+1. Agree to the terms of service and submit.
+1. Copy your site key and secret key and add them to your `homeserver.yaml`
+configuration file
+    ```
     recaptcha_public_key: YOUR_SITE_KEY
     recaptcha_private_key: YOUR_SECRET_KEY
-
-In addition, you MUST enable captchas via:
-
+    ```
+1. Enable the CAPTCHA for new registrations
+    ```
     enable_registration_captcha: true
+    ```
+1. Go to the settings page for the CAPTCHA you just created
+1. Uncheck the "Verify the origin of reCAPTCHA solutions" checkbox so that the
+captcha can be displayed in any client. If you do not disable this option then you
+must specify the domains of every client that is allowed to display the CAPTCHA.
 
 ## Configuring IP used for auth
 
-The ReCaptcha API requires that the IP address of the user who solved the
-captcha is sent. If the client is connecting through a proxy or load balancer,
+The reCAPTCHA API requires that the IP address of the user who solved the
+CAPTCHA is sent. If the client is connecting through a proxy or load balancer,
 it may be required to use the `X-Forwarded-For` (XFF) header instead of the origin
 IP address. This can be configured using the `x_forwarded` directive in the
-listeners section of the homeserver.yaml configuration file.
+listeners section of the `homeserver.yaml` configuration file.
diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md
index 80bd1294c7..7d71c190ab 100644
--- a/docs/MSC1711_certificates_FAQ.md
+++ b/docs/MSC1711_certificates_FAQ.md
@@ -14,7 +14,7 @@ upgraded, however it may be of use to those with old installs returning to the
 project.
 
 If you are setting up a server from scratch you almost certainly should look at
-the [installation guide](../INSTALL.md) instead.
+the [installation guide](setup/installation.md) instead.
 
 ## Introduction
 The goal of Synapse 0.99.0 is to act as a stepping stone to Synapse 1.0.0. It
@@ -101,15 +101,6 @@ In this case, your `server_name` points to the host where your Synapse is
 running. There is no need to create a `.well-known` URI or an SRV record, but
 you will need to give Synapse a valid, signed, certificate.
 
-The easiest way to do that is with Synapse's built-in ACME (Let's Encrypt)
-support. Full details are in [ACME.md](./ACME.md) but, in a nutshell:
-
- 1. Allow Synapse to listen on port 80 with `authbind`, or forward it from a
-    reverse proxy.
- 2. Enable acme support in `homeserver.yaml`.
- 3. Move your old certificates out of the way.
- 4. Restart Synapse.
-
 ### If you do have an SRV record currently
 
 If you are using an SRV record, your matrix domain (`server_name`) may not
@@ -130,15 +121,9 @@ In this situation, you have three choices for how to proceed:
 #### Option 1: give Synapse a certificate for your matrix domain
 
 Synapse 1.0 will expect your server to present a TLS certificate for your
-`server_name` (`example.com` in the above example). You can achieve this by
-doing one of the following:
-
- * Acquire a certificate for the `server_name` yourself (for example, using
-   `certbot`), and give it and the key to Synapse via `tls_certificate_path`
-   and `tls_private_key_path`, or:
-
- * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the
-   `server_name` domain to your Synapse instance.
+`server_name` (`example.com` in the above example). You can achieve this by acquiring a
+certificate for the `server_name` yourself (for example, using `certbot`), and giving it
+and the key to Synapse via `tls_certificate_path` and `tls_private_key_path`.
 
 #### Option 2: run Synapse behind a reverse proxy
 
@@ -147,7 +132,7 @@ your domain, you can simply route all traffic through the reverse proxy by
 updating the SRV record appropriately (or removing it, if the proxy listens on
 8448).
 
-See [reverse_proxy.md](reverse_proxy.md) for information on setting up a
+See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
 reverse proxy.
 
 #### Option 3: add a .well-known file to delegate your matrix traffic
@@ -161,10 +146,9 @@ You can do this with a `.well-known` file as follows:
     with Synapse 0.34 and earlier.
 
   2. Give Synapse a certificate corresponding to the target domain
-    (`customer.example.net` in the above example). You can either use Synapse's
-    built-in [ACME support](./ACME.md) for this (via the `domain` parameter in
-    the `acme` section), or acquire a certificate yourself and give it to
-    Synapse via `tls_certificate_path` and `tls_private_key_path`.
+    (`customer.example.net` in the above example). You can do this by acquire a 
+    certificate for the target domain and giving it to Synapse via `tls_certificate_path`
+    and `tls_private_key_path`.
 
  3. Restart Synapse to ensure the new certificate is loaded.
 
@@ -319,7 +303,7 @@ We no longer actively recommend against using a reverse proxy. Many admins will
 find it easier to direct federation traffic to a reverse proxy and manage their
 own TLS certificates, and this is a supported configuration.
 
-See [reverse_proxy.md](reverse_proxy.md) for information on setting up a
+See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
 reverse proxy.
 
 ### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?
diff --git a/docs/README.md b/docs/README.md
index 3c6ea48c66..e113f55d2a 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,7 +1,72 @@
 # Synapse Documentation
 
-This directory contains documentation specific to the `synapse` homeserver.
+**The documentation is currently hosted [here](https://matrix-org.github.io/synapse).**
+Please update any links to point to the new website instead.
 
-All matrix-generic documentation now lives in its own project, located at [matrix-org/matrix-doc](https://github.com/matrix-org/matrix-doc)
+## About
 
-(Note:  some items here may be moved to [matrix-org/matrix-doc](https://github.com/matrix-org/matrix-doc) at some point in the future.)
+This directory currently holds a series of markdown files documenting how to install, use
+and develop Synapse, the reference Matrix homeserver. The documentation is readable directly
+from this repository, but it is recommended to instead browse through the
+[website](https://matrix-org.github.io/synapse) for easier discoverability.
+
+## Adding to the documentation
+
+Most of the documentation currently exists as top-level files, as when organising them into
+a structured website, these files were kept in place so that existing links would not break.
+The rest of the documentation is stored in folders, such as `setup`, `usage`, and `development`
+etc. **All new documentation files should be placed in structured folders.** For example:
+
+To create a new user-facing documentation page about a new Single Sign-On protocol named
+"MyCoolProtocol", one should create a new file with a relevant name, such as "my_cool_protocol.md".
+This file might fit into the documentation structure at:
+
+- Usage
+  - Configuration
+    - User Authentication
+      - Single Sign-On
+        - **My Cool Protocol**
+
+Given that, one would place the new file under
+`usage/configuration/user_authentication/single_sign_on/my_cool_protocol.md`.
+
+Note that the structure of the documentation (and thus the left sidebar on the website) is determined
+by the list in [SUMMARY.md](SUMMARY.md). The final thing to do when adding a new page is to add a new
+line linking to the new documentation file:
+
+```markdown
+- [My Cool Protocol](usage/configuration/user_authentication/single_sign_on/my_cool_protocol.md)
+```
+
+## Building the documentation
+
+The documentation is built with [mdbook](https://rust-lang.github.io/mdBook/), and the outline of the
+documentation is determined by the structure of [SUMMARY.md](SUMMARY.md).
+
+First, [get mdbook](https://github.com/rust-lang/mdBook#installation). Then, **from the root of the repository**,
+build the documentation with:
+
+```sh
+mdbook build
+```
+
+The rendered contents will be outputted to a new `book/` directory at the root of the repository. You can
+browse the book by opening `book/index.html` in a web browser.
+
+You can also have mdbook host the docs on a local webserver with hot-reload functionality via:
+
+```sh
+mdbook serve
+```
+
+The URL at which the docs can be viewed at will be logged.
+
+## Configuration and theming
+
+The look and behaviour of the website is configured by the [book.toml](../book.toml) file
+at the root of the repository. See
+[mdbook's documentation on configuration](https://rust-lang.github.io/mdBook/format/config.html)
+for available options.
+
+The site can be themed and additionally extended with extra UI and features. See
+[website_files/README.md](website_files/README.md) for details.
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
new file mode 100644
index 0000000000..56e0141c2b
--- /dev/null
+++ b/docs/SUMMARY.md
@@ -0,0 +1,92 @@
+# Summary
+
+# Introduction
+- [Welcome and Overview](welcome_and_overview.md)
+
+# Setup
+  - [Installation](setup/installation.md)
+  - [Using Postgres](postgres.md)
+  - [Configuring a Reverse Proxy](reverse_proxy.md)
+  - [Configuring a Forward/Outbound Proxy](setup/forward_proxy.md)
+  - [Configuring a Turn Server](turn-howto.md)
+  - [Delegation](delegate.md)
+
+# Upgrading
+  - [Upgrading between Synapse Versions](upgrade.md)
+  - [Upgrading from pre-Synapse 1.0](MSC1711_certificates_FAQ.md)
+
+# Usage
+  - [Federation](federate.md)
+  - [Configuration](usage/configuration/README.md)
+    - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md)
+    - [Logging Sample Config File](usage/configuration/logging_sample_config.md)
+    - [Structured Logging](structured_logging.md)
+    - [Templates](templates.md)
+    - [User Authentication](usage/configuration/user_authentication/README.md)
+      - [Single-Sign On]()
+        - [OpenID Connect](openid.md)
+        - [SAML]()
+        - [CAS]()
+        - [SSO Mapping Providers](sso_mapping_providers.md)
+      - [Password Auth Providers](password_auth_providers.md)
+      - [JSON Web Tokens](jwt.md)
+    - [Registration Captcha](CAPTCHA_SETUP.md)
+    - [Application Services](application_services.md)
+    - [Server Notices](server_notices.md)
+    - [Consent Tracking](consent_tracking.md)
+    - [URL Previews](url_previews.md)
+    - [User Directory](user_directory.md)
+    - [Message Retention Policies](message_retention_policies.md)
+    - [Pluggable Modules](modules.md)
+      - [Third Party Rules]()
+      - [Spam Checker](spam_checker.md)
+      - [Presence Router](presence_router_module.md)
+      - [Media Storage Providers]()
+    - [Workers](workers.md)
+      - [Using `synctl` with Workers](synctl_workers.md)
+      - [Systemd](systemd-with-workers/README.md)
+  - [Administration](usage/administration/README.md)
+    - [Admin API](usage/administration/admin_api/README.md)
+      - [Account Validity](admin_api/account_validity.md)
+      - [Delete Group](admin_api/delete_group.md)
+      - [Event Reports](admin_api/event_reports.md)
+      - [Media](admin_api/media_admin_api.md)
+      - [Purge History](admin_api/purge_history_api.md)
+      - [Purge Rooms](admin_api/purge_room.md)
+      - [Register Users](admin_api/register_api.md)
+      - [Manipulate Room Membership](admin_api/room_membership.md)
+      - [Rooms](admin_api/rooms.md)
+      - [Server Notices](admin_api/server_notices.md)
+      - [Shutdown Room](admin_api/shutdown_room.md)
+      - [Statistics](admin_api/statistics.md)
+      - [Users](admin_api/user_admin_api.md)
+      - [Server Version](admin_api/version_api.md)
+    - [Manhole](manhole.md)
+    - [Monitoring](metrics-howto.md)
+    - [Request log format](usage/administration/request_log.md)
+    - [Scripts]()
+
+# Development
+  - [Contributing Guide](development/contributing_guide.md)
+  - [Code Style](code_style.md)
+  - [Git Usage](development/git.md)
+  - [Testing]()
+  - [OpenTracing](opentracing.md)
+  - [Database Schemas](development/database_schema.md)
+  - [Synapse Architecture]()
+    - [Log Contexts](log_contexts.md)
+    - [Replication](replication.md)
+    - [TCP Replication](tcp_replication.md)
+  - [Internal Documentation](development/internal_documentation/README.md)
+    - [Single Sign-On]()
+      - [SAML](development/saml.md)
+      - [CAS](development/cas.md)
+    - [Room DAG concepts](development/room-dag-concepts.md)
+    - [State Resolution]()
+      - [The Auth Chain Difference Algorithm](auth_chain_difference_algorithm.md)
+    - [Media Repository](media_repository.md)
+    - [Room and User Statistics](room_and_user_statistics.md)
+  - [Scripts]()
+
+# Other
+  - [Dependency Deprecation Policy](deprecation_policy.md)
diff --git a/docs/admin_api/README.rst b/docs/admin_api/README.rst
index 9587bee0ce..8d6e76580a 100644
--- a/docs/admin_api/README.rst
+++ b/docs/admin_api/README.rst
@@ -1,28 +1,14 @@
 Admin APIs
 ==========
 
-This directory includes documentation for the various synapse specific admin
-APIs available.
-
-Authenticating as a server admin
---------------------------------
-
-Many of the API calls in the admin api will require an `access_token` for a
-server admin. (Note that a server admin is distinct from a room admin.)
-
-A user can be marked as a server admin by updating the database directly, e.g.:
-
-.. code-block:: sql
+**Note**: The latest documentation can be viewed `here <https://matrix-org.github.io/synapse>`_.
+See `docs/README.md <../README.md>`_ for more information.
 
-    UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
+**Please update links to point to the website instead.** Existing files in this directory
+are preserved to maintain historical links, but may be moved in the future.
 
-A new server admin user can also be created using the
-``register_new_matrix_user`` script.
-
-Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.
-
-Once you have your `access_token`, to include it in a request, the best option is to add the token to a request header:
-
-``curl --header "Authorization: Bearer <access_token>" <the_rest_of_your_API_request>``
+This directory includes documentation for the various synapse specific admin
+APIs available. Updates to the existing Admin API documentation should still
+be made to these files, but any new documentation files should instead be placed under
+`docs/usage/administration/admin_api <../usage/administration/admin_api>`_.
 
-Fore more details, please refer to the complete `matrix spec documentation <https://matrix.org/docs/spec/client_server/r0.5.0#using-access-tokens>`_.
diff --git a/docs/admin_api/account_validity.md b/docs/admin_api/account_validity.md
new file mode 100644
index 0000000000..b74b5d0c1a
--- /dev/null
+++ b/docs/admin_api/account_validity.md
@@ -0,0 +1,42 @@
+# Account validity API
+
+This API allows a server administrator to manage the validity of an account. To
+use it, you must enable the account validity feature (under
+`account_validity`) in Synapse's configuration.
+
+## Renew account
+
+This API extends the validity of an account by as much time as configured in the
+`period` parameter from the `account_validity` configuration.
+
+The API is:
+
+```
+POST /_synapse/admin/v1/account_validity/validity
+```
+
+with the following body:
+
+```json
+{
+    "user_id": "<user ID for the account to renew>",
+    "expiration_ts": 0,
+    "enable_renewal_emails": true
+}
+```
+
+
+`expiration_ts` is an optional parameter and overrides the expiration date,
+which otherwise defaults to now + validity period.
+
+`enable_renewal_emails` is also an optional parameter and enables/disables
+sending renewal emails to the user. Defaults to true.
+
+The API returns with the new expiration date for this account, as a timestamp in
+milliseconds since epoch:
+
+```json
+{
+    "expiration_ts": 0
+}
+```
diff --git a/docs/admin_api/account_validity.rst b/docs/admin_api/account_validity.rst
deleted file mode 100644
index 7559de4c57..0000000000
--- a/docs/admin_api/account_validity.rst
+++ /dev/null
@@ -1,42 +0,0 @@
-Account validity API
-====================
-
-This API allows a server administrator to manage the validity of an account. To
-use it, you must enable the account validity feature (under
-``account_validity``) in Synapse's configuration.
-
-Renew account
--------------
-
-This API extends the validity of an account by as much time as configured in the
-``period`` parameter from the ``account_validity`` configuration.
-
-The API is::
-
-    POST /_synapse/admin/v1/account_validity/validity
-
-with the following body:
-
-.. code:: json
-
-    {
-        "user_id": "<user ID for the account to renew>",
-        "expiration_ts": 0,
-        "enable_renewal_emails": true
-    }
-
-
-``expiration_ts`` is an optional parameter and overrides the expiration date,
-which otherwise defaults to now + validity period.
-
-``enable_renewal_emails`` is also an optional parameter and enables/disables
-sending renewal emails to the user. Defaults to true.
-
-The API returns with the new expiration date for this account, as a timestamp in
-milliseconds since epoch:
-
-.. code:: json
-
-    {
-        "expiration_ts": 0
-    }
diff --git a/docs/admin_api/delete_group.md b/docs/admin_api/delete_group.md
index c061678e75..2e0a1d2474 100644
--- a/docs/admin_api/delete_group.md
+++ b/docs/admin_api/delete_group.md
@@ -11,4 +11,4 @@ POST /_synapse/admin/v1/delete_group/<group_id>
 ```
 
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
diff --git a/docs/admin_api/event_reports.md b/docs/admin_api/event_reports.md
index 0159098138..3abb06099c 100644
--- a/docs/admin_api/event_reports.md
+++ b/docs/admin_api/event_reports.md
@@ -7,7 +7,7 @@ The api is:
 GET /_synapse/admin/v1/event_reports?from=0&limit=10
 ```
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 It returns a JSON body like the following:
 
@@ -75,9 +75,9 @@ The following fields are returned in the JSON response body:
 * `name`: string - The name of the room.
 * `event_id`: string - The ID of the reported event.
 * `user_id`: string - This is the user who reported the event and wrote the reason.
-* `reason`: string - Comment made by the `user_id` in this report. May be blank.
+* `reason`: string - Comment made by the `user_id` in this report. May be blank or `null`.
 * `score`: integer - Content is reported based upon a negative score, where -100 is
-  "most offensive" and 0 is "inoffensive".
+  "most offensive" and 0 is "inoffensive". May be `null`.
 * `sender`: string - This is the ID of the user who sent the original message/event that
   was reported.
 * `canonical_alias`: string - The canonical alias of the room. `null` if the room does not
@@ -95,7 +95,7 @@ The api is:
 GET /_synapse/admin/v1/event_reports/<report_id>
 ```
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 It returns a JSON body like the following:
 
diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md
index 9dbec68c19..ea05bd6e44 100644
--- a/docs/admin_api/media_admin_api.md
+++ b/docs/admin_api/media_admin_api.md
@@ -4,12 +4,15 @@
   * [List all media uploaded by a user](#list-all-media-uploaded-by-a-user)
 - [Quarantine media](#quarantine-media)
   * [Quarantining media by ID](#quarantining-media-by-id)
+  * [Remove media from quarantine by ID](#remove-media-from-quarantine-by-id)
   * [Quarantining media in a room](#quarantining-media-in-a-room)
   * [Quarantining all media of a user](#quarantining-all-media-of-a-user)
   * [Protecting media from being quarantined](#protecting-media-from-being-quarantined)
+  * [Unprotecting media from being quarantined](#unprotecting-media-from-being-quarantined)
 - [Delete local media](#delete-local-media)
   * [Delete a specific local media](#delete-a-specific-local-media)
   * [Delete local media by date or size](#delete-local-media-by-date-or-size)
+  * [Delete media uploaded by a user](#delete-media-uploaded-by-a-user)
 - [Purge Remote Media API](#purge-remote-media-api)
 
 # Querying media
@@ -26,7 +29,7 @@ The API is:
 GET /_synapse/admin/v1/room/<room_id>/media
 ```
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 The API returns a JSON body like the following:
 ```json
@@ -45,7 +48,8 @@ The API returns a JSON body like the following:
 ## List all media uploaded by a user
 
 Listing all media that has been uploaded by a local user can be achieved through
-the use of the [List media of a user](user_admin_api.rst#list-media-of-a-user)
+the use of the
+[List media uploaded by a user](user_admin_api.md#list-media-uploaded-by-a-user)
 Admin API.
 
 # Quarantine media
@@ -76,6 +80,27 @@ Response:
 {}
 ```
 
+## Remove media from quarantine by ID
+
+This API removes a single piece of local or remote media from quarantine.
+
+Request:
+
+```
+POST /_synapse/admin/v1/media/unquarantine/<server_name>/<media_id>
+
+{}
+```
+
+Where `server_name` is in the form of `example.org`, and `media_id` is in the
+form of `abcdefg12345...`.
+
+Response:
+
+```json
+{}
+```
+
 ## Quarantining media in a room
 
 This API quarantines all local and remote media in a room.
@@ -159,6 +184,26 @@ Response:
 {}
 ```
 
+## Unprotecting media from being quarantined
+
+This API reverts the protection of a media.
+
+Request:
+
+```
+POST /_synapse/admin/v1/media/unprotect/<media_id>
+
+{}
+```
+
+Where `media_id` is in the  form of `abcdefg12345...`.
+
+Response:
+
+```json
+{}
+```
+
 # Delete local media
 This API deletes the *local* media from the disk of your own server.
 This includes any local thumbnails and copies of media downloaded from
@@ -214,7 +259,7 @@ URL Parameters
 * `server_name`: string - The name of your local server (e.g `matrix.org`).
 * `before_ts`: string representing a positive integer - Unix timestamp in ms.
 Files that were last used before this timestamp will be deleted. It is the timestamp of
-last access and not the timestamp creation. 
+last access and not the timestamp creation.
 * `size_gt`: Optional - string representing a positive integer - Size of the media in bytes.
 Files that are larger will be deleted. Defaults to `0`.
 * `keep_profiles`: Optional - string representing a boolean - Switch to also delete files
@@ -238,6 +283,11 @@ The following fields are returned in the JSON response body:
 * `deleted_media`: an array of strings - List of deleted `media_id`
 * `total`: integer - Total number of deleted `media_id`
 
+## Delete media uploaded by a user
+
+You can find details of how to delete multiple media uploaded by a user in
+[User Admin API](user_admin_api.md#delete-media-uploaded-by-a-user).
+
 # Purge Remote Media API
 
 The purge remote media API allows server admins to purge old cached remote media.
@@ -268,7 +318,7 @@ The following fields are returned in the JSON response body:
 * `deleted`: integer - The number of media items successfully deleted
 
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 If the user re-requests purged remote media, synapse will re-request the media
 from the originating server.
diff --git a/docs/admin_api/purge_history_api.rst b/docs/admin_api/purge_history_api.md
similarity index 56%
rename from docs/admin_api/purge_history_api.rst
rename to docs/admin_api/purge_history_api.md
index 92cd05f2a0..13b991eacf 100644
--- a/docs/admin_api/purge_history_api.rst
+++ b/docs/admin_api/purge_history_api.md
@@ -1,5 +1,4 @@
-Purge History API
-=================
+# Purge History API
 
 The purge history API allows server admins to purge historic events from their
 database, reclaiming disk space.
@@ -13,10 +12,12 @@ delete the last message in a room.
 
 The API is:
 
-``POST /_synapse/admin/v1/purge_history/<room_id>[/<event_id>]``
+```
+POST /_synapse/admin/v1/purge_history/<room_id>[/<event_id>]
+```
 
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
 
 By default, events sent by local users are not deleted, as they may represent
 the only copies of this content in existence. (Events sent by remote users are
@@ -24,54 +25,54 @@ deleted.)
 
 Room state data (such as joins, leaves, topic) is always preserved.
 
-To delete local message events as well, set ``delete_local_events`` in the body:
+To delete local message events as well, set `delete_local_events` in the body:
 
-.. code:: json
-
-   {
-       "delete_local_events": true
-   }
+```
+{
+   "delete_local_events": true
+}
+```
 
 The caller must specify the point in the room to purge up to. This can be
 specified by including an event_id in the URI, or by setting a
-``purge_up_to_event_id`` or ``purge_up_to_ts`` in the request body. If an event
+`purge_up_to_event_id` or `purge_up_to_ts` in the request body. If an event
 id is given, that event (and others at the same graph depth) will be retained.
-If ``purge_up_to_ts`` is given, it should be a timestamp since the unix epoch,
+If `purge_up_to_ts` is given, it should be a timestamp since the unix epoch,
 in milliseconds.
 
 The API starts the purge running, and returns immediately with a JSON body with
 a purge id:
 
-.. code:: json
-
-    {
-        "purge_id": "<opaque id>"
-    }
+```json
+{
+    "purge_id": "<opaque id>"
+}
+```
 
-Purge status query
-------------------
+## Purge status query
 
 It is possible to poll for updates on recent purges with a second API;
 
-``GET /_synapse/admin/v1/purge_history_status/<purge_id>``
+```
+GET /_synapse/admin/v1/purge_history_status/<purge_id>
+```
 
-Again, you will need to authenticate by providing an ``access_token`` for a
+Again, you will need to authenticate by providing an `access_token` for a
 server admin.
 
 This API returns a JSON body like the following:
 
-.. code:: json
-
-    {
-        "status": "active"
-    }
+```json
+{
+    "status": "active"
+}
+```
 
-The status will be one of ``active``, ``complete``, or ``failed``.
+The status will be one of `active`, `complete`, or `failed`.
 
-Reclaim disk space (Postgres)
------------------------------
+## Reclaim disk space (Postgres)
 
 To reclaim the disk space and return it to the operating system, you need to run
 `VACUUM FULL;` on the database.
 
-https://www.postgresql.org/docs/current/sql-vacuum.html
+<https://www.postgresql.org/docs/current/sql-vacuum.html>
diff --git a/docs/admin_api/register_api.md b/docs/admin_api/register_api.md
new file mode 100644
index 0000000000..c346090bb1
--- /dev/null
+++ b/docs/admin_api/register_api.md
@@ -0,0 +1,73 @@
+# Shared-Secret Registration
+
+This API allows for the creation of users in an administrative and
+non-interactive way. This is generally used for bootstrapping a Synapse
+instance with administrator accounts.
+
+To authenticate yourself to the server, you will need both the shared secret
+(`registration_shared_secret` in the homeserver configuration), and a
+one-time nonce. If the registration shared secret is not configured, this API
+is not enabled.
+
+To fetch the nonce, you need to request one from the API:
+
+```
+> GET /_synapse/admin/v1/register
+
+< {"nonce": "thisisanonce"}
+```
+
+Once you have the nonce, you can make a `POST` to the same URL with a JSON
+body containing the nonce, username, password, whether they are an admin
+(optional, False by default), and a HMAC digest of the content. Also you can
+set the displayname (optional, `username` by default).
+
+As an example:
+
+```
+> POST /_synapse/admin/v1/register
+> {
+   "nonce": "thisisanonce",
+   "username": "pepper_roni",
+   "displayname": "Pepper Roni",
+   "password": "pizza",
+   "admin": true,
+   "mac": "mac_digest_here"
+  }
+
+< {
+   "access_token": "token_here",
+   "user_id": "@pepper_roni:localhost",
+   "home_server": "test",
+   "device_id": "device_id_here"
+  }
+```
+
+The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the key being
+the shared secret and the content being the nonce, user, password, either the
+string "admin" or "notadmin", and optionally the user_type
+each separated by NULs. For an example of generation in Python:
+
+```python
+import hmac, hashlib
+
+def generate_mac(nonce, user, password, admin=False, user_type=None):
+
+    mac = hmac.new(
+      key=shared_secret,
+      digestmod=hashlib.sha1,
+    )
+
+    mac.update(nonce.encode('utf8'))
+    mac.update(b"\x00")
+    mac.update(user.encode('utf8'))
+    mac.update(b"\x00")
+    mac.update(password.encode('utf8'))
+    mac.update(b"\x00")
+    mac.update(b"admin" if admin else b"notadmin")
+    if user_type:
+        mac.update(b"\x00")
+        mac.update(user_type.encode('utf8'))
+
+    return mac.hexdigest()
+```
\ No newline at end of file
diff --git a/docs/admin_api/register_api.rst b/docs/admin_api/register_api.rst
deleted file mode 100644
index c3057b204b..0000000000
--- a/docs/admin_api/register_api.rst
+++ /dev/null
@@ -1,68 +0,0 @@
-Shared-Secret Registration
-==========================
-
-This API allows for the creation of users in an administrative and
-non-interactive way. This is generally used for bootstrapping a Synapse
-instance with administrator accounts.
-
-To authenticate yourself to the server, you will need both the shared secret
-(``registration_shared_secret`` in the homeserver configuration), and a
-one-time nonce. If the registration shared secret is not configured, this API
-is not enabled.
-
-To fetch the nonce, you need to request one from the API::
-
-  > GET /_synapse/admin/v1/register
-
-  < {"nonce": "thisisanonce"}
-
-Once you have the nonce, you can make a ``POST`` to the same URL with a JSON
-body containing the nonce, username, password, whether they are an admin
-(optional, False by default), and a HMAC digest of the content. Also you can
-set the displayname (optional, ``username`` by default).
-
-As an example::
-
-  > POST /_synapse/admin/v1/register
-  > {
-     "nonce": "thisisanonce",
-     "username": "pepper_roni",
-     "displayname": "Pepper Roni",
-     "password": "pizza",
-     "admin": true,
-     "mac": "mac_digest_here"
-    }
-
-  < {
-     "access_token": "token_here",
-     "user_id": "@pepper_roni:localhost",
-     "home_server": "test",
-     "device_id": "device_id_here"
-    }
-
-The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the key being
-the shared secret and the content being the nonce, user, password, either the
-string "admin" or "notadmin", and optionally the user_type
-each separated by NULs. For an example of generation in Python::
-
-  import hmac, hashlib
-
-  def generate_mac(nonce, user, password, admin=False, user_type=None):
-
-      mac = hmac.new(
-        key=shared_secret,
-        digestmod=hashlib.sha1,
-      )
-
-      mac.update(nonce.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(user.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(password.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(b"admin" if admin else b"notadmin")
-      if user_type:
-          mac.update(b"\x00")
-          mac.update(user_type.encode('utf8'))
-
-      return mac.hexdigest()
diff --git a/docs/admin_api/room_membership.md b/docs/admin_api/room_membership.md
index b6746ff5e4..8a5ce191df 100644
--- a/docs/admin_api/room_membership.md
+++ b/docs/admin_api/room_membership.md
@@ -24,7 +24,7 @@ POST /_synapse/admin/v1/join/<room_id_or_alias>
 ```
 
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 Response:
 
diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md
index bc737b30f5..48777dd231 100644
--- a/docs/admin_api/rooms.md
+++ b/docs/admin_api/rooms.md
@@ -1,12 +1,9 @@
 # Contents
 - [List Room API](#list-room-api)
-  * [Parameters](#parameters)
-  * [Usage](#usage)
 - [Room Details API](#room-details-api)
 - [Room Members API](#room-members-api)
+- [Room State API](#room-state-api)
 - [Delete Room API](#delete-room-api)
-  * [Parameters](#parameters-1)
-  * [Response](#response)
   * [Undoing room shutdowns](#undoing-room-shutdowns)
 - [Make Room Admin API](#make-room-admin-api)
 - [Forward Extremities Admin API](#forward-extremities-admin-api)
@@ -18,7 +15,7 @@ The List Room admin API allows server admins to get a list of rooms on their
 server. There are various parameters available that allow for filtering and
 sorting the returned list. This API supports pagination.
 
-## Parameters
+**Parameters**
 
 The following query parameters are available:
 
@@ -45,6 +42,8 @@ The following query parameters are available:
 * `search_term` - Filter rooms by their room name. Search term can be contained in any
                   part of the room name. Defaults to no filtering.
 
+**Response**
+
 The following fields are possible in the JSON response body:
 
 * `rooms` - An array of objects, each containing information about a room.
@@ -78,17 +77,15 @@ The following fields are possible in the JSON response body:
                  Use `prev_batch` for the `from` value in the next request to
                  get the "previous page" of results.
 
-## Usage
+The API is:
 
 A standard request with no filtering:
 
 ```
 GET /_synapse/admin/v1/rooms
-
-{}
 ```
 
-Response:
+A response body like the following is returned:
 
 ```jsonc
 {
@@ -136,11 +133,9 @@ Filtering by room name:
 
 ```
 GET /_synapse/admin/v1/rooms?search_term=TWIM
-
-{}
 ```
 
-Response:
+A response body like the following is returned:
 
 ```json
 {
@@ -171,11 +166,9 @@ Paginating through a list of rooms:
 
 ```
 GET /_synapse/admin/v1/rooms?order_by=size
-
-{}
 ```
 
-Response:
+A response body like the following is returned:
 
 ```jsonc
 {
@@ -227,11 +220,9 @@ parameter to the value of `next_token`.
 
 ```
 GET /_synapse/admin/v1/rooms?order_by=size&from=100
-
-{}
 ```
 
-Response:
+A response body like the following is returned:
 
 ```jsonc
 {
@@ -303,17 +294,13 @@ The following fields are possible in the JSON response body:
 * `history_visibility` - Who can see the room history. One of: ["invited", "joined", "shared", "world_readable"].
 * `state_events` - Total number of state_events of a room. Complexity of the room.
 
-## Usage
-
-A standard request:
+The API is:
 
 ```
 GET /_synapse/admin/v1/rooms/<room_id>
-
-{}
 ```
 
-Response:
+A response body like the following is returned:
 
 ```json
 {
@@ -346,17 +333,13 @@ The response includes the following fields:
 * `members` - A list of all the members that are present in the room, represented by their ids.
 * `total` - Total number of members in the room.
 
-## Usage
-
-A standard request:
+The API is:
 
 ```
 GET /_synapse/admin/v1/rooms/<room_id>/members
-
-{}
 ```
 
-Response:
+A response body like the following is returned:
 
 ```json
 {
@@ -377,17 +360,13 @@ The response includes the following fields:
 
 * `state` - The current state of the room at the time of request.
 
-## Usage
-
-A standard request:
+The API is:
 
 ```
 GET /_synapse/admin/v1/rooms/<room_id>/state
-
-{}
 ```
 
-Response:
+A response body like the following is returned:
 
 ```json
 {
@@ -427,10 +406,11 @@ the new room. Users on other servers will be unaffected.
 The API is:
 
 ```
-POST /_synapse/admin/v1/rooms/<room_id>/delete
+DELETE /_synapse/admin/v1/rooms/<room_id>
 ```
 
 with a body of:
+
 ```json
 {
     "new_room_user_id": "@someuser:example.com",
@@ -442,7 +422,7 @@ with a body of:
 ```
 
 To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 A response body like the following is returned:
 
@@ -460,7 +440,7 @@ A response body like the following is returned:
 }
 ```
 
-## Parameters
+**Parameters**
 
 The following parameters should be set in the URL:
 
@@ -490,7 +470,7 @@ The following JSON body parameters are available:
 
 The JSON body must not be empty. The body must be at least `{}`.
 
-## Response
+**Response**
 
 The following fields are returned in the JSON response body:
 
@@ -528,6 +508,15 @@ You will have to manually handle, if you so choose, the following:
 * Users that would have been booted from the room (and will have been force-joined to the Content Violation room).
 * Removal of the Content Violation room if desired.
 
+## Deprecated endpoint
+
+The previous deprecated API will be removed in a future release, it was:
+
+```
+POST /_synapse/admin/v1/rooms/<room_id>/delete
+```
+
+It behaves the same way than the current endpoint except the path and the method.
 
 # Make Room Admin API
 
@@ -538,10 +527,10 @@ By default the server admin (the caller) is granted power, but another user can
 optionally be specified, e.g.:
 
 ```
-    POST /_synapse/admin/v1/rooms/<room_id_or_alias>/make_room_admin
-    {
-        "user_id": "@foo:example.com"
-    }
+POST /_synapse/admin/v1/rooms/<room_id_or_alias>/make_room_admin
+{
+    "user_id": "@foo:example.com"
+}
 ```
 
 # Forward Extremities Admin API
@@ -555,7 +544,7 @@ extremities accumulate in a room, performance can become degraded. For details,
 To check the status of forward extremities for a room:
 
 ```
-    GET /_synapse/admin/v1/rooms/<room_id_or_alias>/forward_extremities
+GET /_synapse/admin/v1/rooms/<room_id_or_alias>/forward_extremities
 ```
 
 A response as follows will be returned:
@@ -571,7 +560,7 @@ A response as follows will be returned:
       "received_ts": 1611263016761
     }
   ]
-}    
+}
 ```
 
 ## Deleting forward extremities
@@ -584,7 +573,7 @@ If a room has lots of forward extremities, the extra can be
 deleted as follows:
 
 ```
-    DELETE /_synapse/admin/v1/rooms/<room_id_or_alias>/forward_extremities
+DELETE /_synapse/admin/v1/rooms/<room_id_or_alias>/forward_extremities
 ```
 
 A response as follows will be returned, indicating the amount of forward extremities
diff --git a/docs/admin_api/server_notices.md b/docs/admin_api/server_notices.md
index 858b052b84..323138491a 100644
--- a/docs/admin_api/server_notices.md
+++ b/docs/admin_api/server_notices.md
@@ -45,4 +45,4 @@ Once the notice has been sent, the API will return the following response:
 ```
 
 Note that server notices must be enabled in `homeserver.yaml` before this API
-can be used. See [server_notices.md](../server_notices.md) for more information.
+can be used. See [the server notices documentation](../server_notices.md) for more information.
diff --git a/docs/admin_api/statistics.md b/docs/admin_api/statistics.md
index d398a120fb..1901f1eea0 100644
--- a/docs/admin_api/statistics.md
+++ b/docs/admin_api/statistics.md
@@ -10,7 +10,7 @@ GET /_synapse/admin/v1/statistics/users/media
 ```
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [README.rst](README.rst).
+for a server admin: see [Admin API](../usage/administration/admin_api).
 
 A response body like the following is returned:
 
diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md
new file mode 100644
index 0000000000..6a9335d6ec
--- /dev/null
+++ b/docs/admin_api/user_admin_api.md
@@ -0,0 +1,1097 @@
+# User Admin API
+
+## Query User Account
+
+This API returns information about a specific user account.
+
+The api is:
+
+```
+GET /_synapse/admin/v2/users/<user_id>
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+It returns a JSON body like the following:
+
+```json
+{
+    "displayname": "User",
+    "threepids": [
+        {
+            "medium": "email",
+            "address": "<user_mail_1>"
+        },
+        {
+            "medium": "email",
+            "address": "<user_mail_2>"
+        }
+    ],
+    "avatar_url": "<avatar_url>",
+    "admin": 0,
+    "deactivated": 0,
+    "shadow_banned": 0,
+    "password_hash": "$2b$12$p9B4GkqYdRTPGD",
+    "creation_ts": 1560432506,
+    "appservice_id": null,
+    "consent_server_notice_sent": null,
+    "consent_version": null,
+    "external_ids": [
+        {
+            "auth_provider": "<provider1>",
+            "external_id": "<user_id_provider_1>"
+        },
+        {
+            "auth_provider": "<provider2>",
+            "external_id": "<user_id_provider_2>"
+        }
+    ]
+}
+```
+
+URL parameters:
+
+- `user_id`: fully-qualified user id: for example, `@user:server.com`.
+
+## Create or modify Account
+
+This API allows an administrator to create or modify a user account with a
+specific `user_id`.
+
+This api is:
+
+```
+PUT /_synapse/admin/v2/users/<user_id>
+```
+
+with a body of:
+
+```json
+{
+    "password": "user_password",
+    "displayname": "User",
+    "threepids": [
+        {
+            "medium": "email",
+            "address": "<user_mail_1>"
+        },
+        {
+            "medium": "email",
+            "address": "<user_mail_2>"
+        }
+    ],
+    "external_ids": [
+        {
+            "auth_provider": "<provider1>",
+            "external_id": "<user_id_provider_1>"
+        },
+        {
+            "auth_provider": "<provider2>",
+            "external_id": "<user_id_provider_2>"
+        }
+    ],
+    "avatar_url": "<avatar_url>",
+    "admin": false,
+    "deactivated": false
+}
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+Returns HTTP status code:
+- `201` - When a new user object was created.
+- `200` - When a user was modified.
+
+URL parameters:
+
+- `user_id`: fully-qualified user id: for example, `@user:server.com`.
+
+Body parameters:
+
+- `password` - string, optional. If provided, the user's password is updated and all
+  devices are logged out.
+- `displayname` - string, optional, defaults to the value of `user_id`.
+- `threepids` - array, optional, allows setting the third-party IDs (email, msisdn)
+  - `medium` - string. Kind of third-party ID, either `email` or `msisdn`.
+  - `address` - string. Value of third-party ID.
+  belonging to a user.
+- `external_ids` - array, optional. Allow setting the identifier of the external identity
+  provider for SSO (Single sign-on). Details in
+  [Sample Configuration File](../usage/configuration/homeserver_sample_config.html)
+  section `sso` and `oidc_providers`.
+  - `auth_provider` - string. ID of the external identity provider. Value of `idp_id`
+    in homeserver configuration.
+  - `external_id` - string, user ID in the external identity provider.
+- `avatar_url` - string, optional, must be a
+  [MXC URI](https://matrix.org/docs/spec/client_server/r0.6.0#matrix-content-mxc-uris).
+- `admin` - bool, optional, defaults to `false`.
+- `deactivated` - bool, optional. If unspecified, deactivation state will be left
+  unchanged on existing accounts and set to `false` for new accounts.
+  A user cannot be erased by deactivating with this API. For details on
+  deactivating users see [Deactivate Account](#deactivate-account).
+
+If the user already exists then optional parameters default to the current value.
+
+In order to re-activate an account `deactivated` must be set to `false`. If
+users do not login via single-sign-on, a new `password` must be provided.
+
+## List Accounts
+
+This API returns all local user accounts.
+By default, the response is ordered by ascending user ID.
+
+```
+GET /_synapse/admin/v2/users?from=0&limit=10&guests=false
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+{
+    "users": [
+        {
+            "name": "<user_id1>",
+            "is_guest": 0,
+            "admin": 0,
+            "user_type": null,
+            "deactivated": 0,
+            "shadow_banned": 0,
+            "displayname": "<User One>",
+            "avatar_url": null,
+            "creation_ts": 1560432668000
+        }, {
+            "name": "<user_id2>",
+            "is_guest": 0,
+            "admin": 1,
+            "user_type": null,
+            "deactivated": 0,
+            "shadow_banned": 0,
+            "displayname": "<User Two>",
+            "avatar_url": "<avatar_url>",
+            "creation_ts": 1561550621000
+        }
+    ],
+    "next_token": "100",
+    "total": 200
+}
+```
+
+To paginate, check for `next_token` and if present, call the endpoint again
+with `from` set to the value of `next_token`. This will return a new page.
+
+If the endpoint does not return a `next_token` then there are no more users
+to paginate through.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - Is optional and filters to only return users with user IDs
+  that contain this value. This parameter is ignored when using the `name` parameter.
+- `name` - Is optional and filters to only return users with user ID localparts
+  **or** displaynames that contain this value.
+- `guests` - string representing a bool - Is optional and if `false` will **exclude** guest users.
+  Defaults to `true` to include guest users.
+- `deactivated` - string representing a bool - Is optional and if `true` will **include** deactivated users.
+  Defaults to `false` to exclude deactivated users.
+- `limit` - string representing a positive integer - Is optional but is used for pagination,
+  denoting the maximum number of items to return in this call. Defaults to `100`.
+- `from` - string representing a positive integer - Is optional but used for pagination,
+  denoting the offset in the returned results. This should be treated as an opaque value and
+  not explicitly set to anything other than the return value of `next_token` from a previous call.
+  Defaults to `0`.
+- `order_by` - The method by which to sort the returned list of users.
+  If the ordered field has duplicates, the second order is always by ascending `name`,
+  which guarantees a stable ordering. Valid values are:
+
+  - `name` - Users are ordered alphabetically by `name`. This is the default.
+  - `is_guest` - Users are ordered by `is_guest` status.
+  - `admin` - Users are ordered by `admin` status.
+  - `user_type` - Users are ordered alphabetically by `user_type`.
+  - `deactivated` - Users are ordered by `deactivated` status.
+  - `shadow_banned` - Users are ordered by `shadow_banned` status.
+  - `displayname` - Users are ordered alphabetically by `displayname`.
+  - `avatar_url` - Users are ordered alphabetically by avatar URL.
+  - `creation_ts` - Users are ordered by when the users was created in ms.
+
+- `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
+  Setting this value to `b` will reverse the above sort order. Defaults to `f`.
+
+Caution. The database only has indexes on the columns `name` and `creation_ts`.
+This means that if a different sort order is used (`is_guest`, `admin`,
+`user_type`, `deactivated`, `shadow_banned`, `avatar_url` or `displayname`),
+this can cause a large load on the database, especially for large environments.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- `users` - An array of objects, each containing information about an user.
+  User objects contain the following fields:
+
+  - `name` - string - Fully-qualified user ID (ex. `@user:server.com`).
+  - `is_guest` - bool - Status if that user is a guest account.
+  - `admin` - bool - Status if that user is a server administrator.
+  - `user_type` - string - Type of the user. Normal users are type `None`.
+    This allows user type specific behaviour. There are also types `support` and `bot`. 
+  - `deactivated` - bool - Status if that user has been marked as deactivated.
+  - `shadow_banned` - bool - Status if that user has been marked as shadow banned.
+  - `displayname` - string - The user's display name if they have set one.
+  - `avatar_url` - string -  The user's avatar URL if they have set one.
+  - `creation_ts` - integer - The user's creation timestamp in ms.
+
+- `next_token`: string representing a positive integer - Indication for pagination. See above.
+- `total` - integer - Total number of media.
+
+
+## Query current sessions for a user
+
+This API returns information about the active sessions for a specific user.
+
+The endpoints are:
+
+```
+GET /_synapse/admin/v1/whois/<user_id>
+```
+
+and:
+
+```
+GET /_matrix/client/r0/admin/whois/<userId>
+```
+
+See also: [Client Server
+API Whois](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid).
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+It returns a JSON body like the following:
+
+```json
+{
+    "user_id": "<user_id>",
+    "devices": {
+        "": {
+            "sessions": [
+                {
+                    "connections": [
+                        {
+                            "ip": "1.2.3.4",
+                            "last_seen": 1417222374433,
+                            "user_agent": "Mozilla/5.0 ..."
+                        },
+                        {
+                            "ip": "1.2.3.10",
+                            "last_seen": 1417222374500,
+                            "user_agent": "Dalvik/2.1.0 ..."
+                        }
+                    ]
+                }
+            ]
+        }
+    }
+}
+```
+
+`last_seen` is measured in milliseconds since the Unix epoch.
+
+## Deactivate Account
+
+This API deactivates an account. It removes active access tokens, resets the
+password, and deletes third-party IDs (to prevent the user requesting a
+password reset).
+
+It can also mark the user as GDPR-erased. This means messages sent by the
+user will still be visible by anyone that was in the room when these messages
+were sent, but hidden from users joining the room afterwards.
+
+The api is:
+
+```
+POST /_synapse/admin/v1/deactivate/<user_id>
+```
+
+with a body of:
+
+```json
+{
+    "erase": true
+}
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+The erase parameter is optional and defaults to `false`.
+An empty body may be passed for backwards compatibility.
+
+The following actions are performed when deactivating an user:
+
+- Try to unpind 3PIDs from the identity server
+- Remove all 3PIDs from the homeserver
+- Delete all devices and E2EE keys
+- Delete all access tokens
+- Delete the password hash
+- Removal from all rooms the user is a member of
+- Remove the user from the user directory
+- Reject all pending invites
+- Remove all account validity information related to the user
+
+The following additional actions are performed during deactivation if `erase`
+is set to `true`:
+
+- Remove the user's display name
+- Remove the user's avatar URL
+- Mark the user as erased
+
+
+## Reset password
+
+Changes the password of another user. This will automatically log the user out of all their devices.
+
+The api is:
+
+```
+POST /_synapse/admin/v1/reset_password/<user_id>
+```
+
+with a body of:
+
+```json
+{
+   "new_password": "<secret>",
+   "logout_devices": true
+}
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+The parameter `new_password` is required.
+The parameter `logout_devices` is optional and defaults to `true`.
+
+
+## Get whether a user is a server administrator or not
+
+The api is:
+
+```
+GET /_synapse/admin/v1/users/<user_id>/admin
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+{
+    "admin": true
+}
+```
+
+
+## Change whether a user is a server administrator or not
+
+Note that you cannot demote yourself.
+
+The api is:
+
+```
+PUT /_synapse/admin/v1/users/<user_id>/admin
+```
+
+with a body of:
+
+```json
+{
+    "admin": true
+}
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+
+## List room memberships of a user
+
+Gets a list of all `room_id` that a specific `user_id` is member.
+
+The API is:
+
+```
+GET /_synapse/admin/v1/users/<user_id>/joined_rooms
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+    {
+        "joined_rooms": [
+            "!DuGcnbhHGaSZQoNQR:matrix.org",
+            "!ZtSaPCawyWtxfWiIy:matrix.org"
+        ],
+        "total": 2
+    }
+```
+
+The server returns the list of rooms of which the user and the server
+are member. If the user is local, all the rooms of which the user is
+member are returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - fully qualified: for example, `@user:server.com`.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- `joined_rooms` - An array of `room_id`.
+- `total` - Number of rooms.
+
+## User media
+
+### List media uploaded by a user
+Gets a list of all local media that a specific `user_id` has created.
+By default, the response is ordered by descending creation date and ascending media ID.
+The newest media is on top. You can change the order with parameters
+`order_by` and `dir`.
+
+The API is:
+
+```
+GET /_synapse/admin/v1/users/<user_id>/media
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+{
+  "media": [
+    {
+      "created_ts": 100400,
+      "last_access_ts": null,
+      "media_id": "qXhyRzulkwLsNHTbpHreuEgo",
+      "media_length": 67,
+      "media_type": "image/png",
+      "quarantined_by": null,
+      "safe_from_quarantine": false,
+      "upload_name": "test1.png"
+    },
+    {
+      "created_ts": 200400,
+      "last_access_ts": null,
+      "media_id": "FHfiSnzoINDatrXHQIXBtahw",
+      "media_length": 67,
+      "media_type": "image/png",
+      "quarantined_by": null,
+      "safe_from_quarantine": false,
+      "upload_name": "test2.png"
+    }
+  ],
+  "next_token": 3,
+  "total": 2
+}
+```
+
+To paginate, check for `next_token` and if present, call the endpoint again
+with `from` set to the value of `next_token`. This will return a new page.
+
+If the endpoint does not return a `next_token` then there are no more
+reports to paginate through.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - string - fully qualified: for example, `@user:server.com`.
+- `limit`: string representing a positive integer - Is optional but is used for pagination,
+  denoting the maximum number of items to return in this call. Defaults to `100`.
+- `from`: string representing a positive integer - Is optional but used for pagination,
+  denoting the offset in the returned results. This should be treated as an opaque value and
+  not explicitly set to anything other than the return value of `next_token` from a previous call.
+  Defaults to `0`.
+- `order_by` - The method by which to sort the returned list of media.
+  If the ordered field has duplicates, the second order is always by ascending `media_id`,
+  which guarantees a stable ordering. Valid values are:
+
+  - `media_id` - Media are ordered alphabetically by `media_id`.
+  - `upload_name` - Media are ordered alphabetically by name the media was uploaded with.
+  - `created_ts` - Media are ordered by when the content was uploaded in ms.
+    Smallest to largest. This is the default.
+  - `last_access_ts` - Media are ordered by when the content was last accessed in ms.
+    Smallest to largest.
+  - `media_length` - Media are ordered by length of the media in bytes.
+    Smallest to largest.
+  - `media_type` - Media are ordered alphabetically by MIME-type.
+  - `quarantined_by` - Media are ordered alphabetically by the user ID that
+    initiated the quarantine request for this media.
+  - `safe_from_quarantine` - Media are ordered by the status if this media is safe
+    from quarantining.
+
+- `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
+  Setting this value to `b` will reverse the above sort order. Defaults to `f`.
+
+If neither `order_by` nor `dir` is set, the default order is newest media on top
+(corresponds to `order_by` = `created_ts` and `dir` = `b`).
+
+Caution. The database only has indexes on the columns `media_id`,
+`user_id` and `created_ts`. This means that if a different sort order is used
+(`upload_name`, `last_access_ts`, `media_length`, `media_type`,
+`quarantined_by` or `safe_from_quarantine`), this can cause a large load on the
+database, especially for large environments.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- `media` - An array of objects, each containing information about a media.
+  Media objects contain the following fields:
+  - `created_ts` - integer - Timestamp when the content was uploaded in ms.
+  - `last_access_ts` - integer - Timestamp when the content was last accessed in ms.
+  - `media_id` - string - The id used to refer to the media.
+  - `media_length` - integer - Length of the media in bytes.
+  - `media_type` - string - The MIME-type of the media.
+  - `quarantined_by` - string - The user ID that initiated the quarantine request
+    for this media.
+  - `safe_from_quarantine` - bool - Status if this media is safe from quarantining.
+  - `upload_name` - string - The name the media was uploaded with.
+- `next_token`: integer - Indication for pagination. See above.
+- `total` - integer - Total number of media.
+
+### Delete media uploaded by a user
+
+This API deletes the *local* media from the disk of your own server
+that a specific `user_id` has created. This includes any local thumbnails.
+
+This API will not affect media that has been uploaded to external
+media repositories (e.g https://github.com/turt2live/matrix-media-repo/).
+
+By default, the API deletes media ordered by descending creation date and ascending media ID.
+The newest media is deleted first. You can change the order with parameters
+`order_by` and `dir`. If no `limit` is set the API deletes `100` files per request.
+
+The API is:
+
+```
+DELETE /_synapse/admin/v1/users/<user_id>/media
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+{
+  "deleted_media": [
+    "abcdefghijklmnopqrstuvwx"
+  ],
+  "total": 1
+}
+```
+
+The following fields are returned in the JSON response body:
+
+* `deleted_media`: an array of strings - List of deleted `media_id`
+* `total`: integer - Total number of deleted `media_id`
+
+**Note**: There is no `next_token`. This is not useful for deleting media, because
+after deleting media the remaining media have a new order.
+
+**Parameters**
+
+This API has the same parameters as
+[List media uploaded by a user](#list-media-uploaded-by-a-user).
+With the parameters you can for example limit the number of files to delete at once or
+delete largest/smallest or newest/oldest files first.
+
+## Login as a user
+
+Get an access token that can be used to authenticate as that user. Useful for
+when admins wish to do actions on behalf of a user.
+
+The API is:
+
+```
+POST /_synapse/admin/v1/users/<user_id>/login
+{}
+```
+
+An optional `valid_until_ms` field can be specified in the request body as an
+integer timestamp that specifies when the token should expire. By default tokens
+do not expire.
+
+A response body like the following is returned:
+
+```json
+{
+    "access_token": "<opaque_access_token_string>"
+}
+```
+
+This API does *not* generate a new device for the user, and so will not appear
+their `/devices` list, and in general the target user should not be able to
+tell they have been logged in as.
+
+To expire the token call the standard `/logout` API with the token.
+
+Note: The token will expire if the *admin* user calls `/logout/all` from any
+of their devices, but the token will *not* expire if the target user does the
+same.
+
+
+## User devices
+
+### List all devices
+Gets information about all devices for a specific `user_id`.
+
+The API is:
+
+```
+GET /_synapse/admin/v2/users/<user_id>/devices
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+{
+  "devices": [
+    {
+      "device_id": "QBUAZIFURK",
+      "display_name": "android",
+      "last_seen_ip": "1.2.3.4",
+      "last_seen_ts": 1474491775024,
+      "user_id": "<user_id>"
+    },
+    {
+      "device_id": "AUIECTSRND",
+      "display_name": "ios",
+      "last_seen_ip": "1.2.3.5",
+      "last_seen_ts": 1474491775025,
+      "user_id": "<user_id>"
+    }
+  ],
+  "total": 2
+}
+```
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - fully qualified: for example, `@user:server.com`.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- `devices` - An array of objects, each containing information about a device.
+  Device objects contain the following fields:
+
+  - `device_id` - Identifier of device.
+  - `display_name` - Display name set by the user for this device.
+    Absent if no name has been set.
+  - `last_seen_ip` - The IP address where this device was last seen.
+    (May be a few minutes out of date, for efficiency reasons).
+  - `last_seen_ts` - The timestamp (in milliseconds since the unix epoch) when this
+    devices was last seen. (May be a few minutes out of date, for efficiency reasons).
+  - `user_id` - Owner of  device.
+
+- `total` - Total number of user's devices.
+
+### Delete multiple devices
+Deletes the given devices for a specific `user_id`, and invalidates
+any access token associated with them.
+
+The API is:
+
+```
+POST /_synapse/admin/v2/users/<user_id>/delete_devices
+
+{
+  "devices": [
+    "QBUAZIFURK",
+    "AUIECTSRND"
+  ],
+}
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+An empty JSON dict is returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - fully qualified: for example, `@user:server.com`.
+
+The following fields are required in the JSON request body:
+
+- `devices` - The list of device IDs to delete.
+
+### Show a device
+Gets information on a single device, by `device_id` for a specific `user_id`.
+
+The API is:
+
+```
+GET /_synapse/admin/v2/users/<user_id>/devices/<device_id>
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+{
+  "device_id": "<device_id>",
+  "display_name": "android",
+  "last_seen_ip": "1.2.3.4",
+  "last_seen_ts": 1474491775024,
+  "user_id": "<user_id>"
+}
+```
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - fully qualified: for example, `@user:server.com`.
+- `device_id` - The device to retrieve.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- `device_id` - Identifier of device.
+- `display_name` - Display name set by the user for this device.
+  Absent if no name has been set.
+- `last_seen_ip` - The IP address where this device was last seen.
+  (May be a few minutes out of date, for efficiency reasons).
+- `last_seen_ts` - The timestamp (in milliseconds since the unix epoch) when this
+  devices was last seen. (May be a few minutes out of date, for efficiency reasons).
+- `user_id` - Owner of  device.
+
+### Update a device
+Updates the metadata on the given `device_id` for a specific `user_id`.
+
+The API is:
+
+```
+PUT /_synapse/admin/v2/users/<user_id>/devices/<device_id>
+
+{
+  "display_name": "My other phone"
+}
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+An empty JSON dict is returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - fully qualified: for example, `@user:server.com`.
+- `device_id` - The device to update.
+
+The following fields are required in the JSON request body:
+
+- `display_name` - The new display name for this device. If not given,
+  the display name is unchanged.
+
+### Delete a device
+Deletes the given `device_id` for a specific `user_id`,
+and invalidates any access token associated with it.
+
+The API is:
+
+```
+DELETE /_synapse/admin/v2/users/<user_id>/devices/<device_id>
+
+{}
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+An empty JSON dict is returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - fully qualified: for example, `@user:server.com`.
+- `device_id` - The device to delete.
+
+## List all pushers
+Gets information about all pushers for a specific `user_id`.
+
+The API is:
+
+```
+GET /_synapse/admin/v1/users/<user_id>/pushers
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+{
+  "pushers": [
+    {
+      "app_display_name":"HTTP Push Notifications",
+      "app_id":"m.http",
+      "data": {
+        "url":"example.com"
+      },
+      "device_display_name":"pushy push",
+      "kind":"http",
+      "lang":"None",
+      "profile_tag":"",
+      "pushkey":"a@example.com"
+    }
+  ],
+  "total": 1
+}
+```
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - fully qualified: for example, `@user:server.com`.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- `pushers` - An array containing the current pushers for the user
+
+  - `app_display_name` - string - A string that will allow the user to identify
+    what application owns this pusher.
+
+  - `app_id` - string - This is a reverse-DNS style identifier for the application.
+    Max length, 64 chars.
+
+  - `data` - A dictionary of information for the pusher implementation itself.
+
+    - `url` - string - Required if `kind` is `http`. The URL to use to send
+      notifications to.
+
+    - `format` - string - The format to use when sending notifications to the
+      Push Gateway.
+
+  - `device_display_name` - string -  A string that will allow the user to identify
+    what device owns this pusher.
+
+  - `profile_tag` - string - This string determines which set of device specific rules
+    this pusher executes.
+
+  - `kind` - string -  The kind of pusher. "http" is a pusher that sends HTTP pokes.
+  - `lang` - string - The preferred language for receiving notifications
+    (e.g. 'en' or 'en-US')
+
+  - `profile_tag` - string - This string determines which set of device specific rules
+    this pusher executes.
+
+  - `pushkey` - string - This is a unique identifier for this pusher.
+    Max length, 512 bytes.
+
+- `total` - integer - Number of pushers.
+
+See also the
+[Client-Server API Spec on pushers](https://matrix.org/docs/spec/client_server/latest#get-matrix-client-r0-pushers).
+
+## Shadow-banning users
+
+Shadow-banning is a useful tool for moderating malicious or egregiously abusive users.
+A shadow-banned users receives successful responses to their client-server API requests,
+but the events are not propagated into rooms. This can be an effective tool as it
+(hopefully) takes longer for the user to realise they are being moderated before
+pivoting to another account.
+
+Shadow-banning a user should be used as a tool of last resort and may lead to confusing
+or broken behaviour for the client. A shadow-banned user will not receive any
+notification and it is generally more appropriate to ban or kick abusive users.
+A shadow-banned user will be unable to contact anyone on the server.
+
+The API is:
+
+```
+POST /_synapse/admin/v1/users/<user_id>/shadow_ban
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+An empty JSON dict is returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
+  be local.
+
+## Override ratelimiting for users
+
+This API allows to override or disable ratelimiting for a specific user.
+There are specific APIs to set, get and delete a ratelimit.
+
+### Get status of ratelimit
+
+The API is:
+
+```
+GET /_synapse/admin/v1/users/<user_id>/override_ratelimit
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+{
+  "messages_per_second": 0,
+  "burst_count": 0
+}
+```
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
+  be local.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- `messages_per_second` - integer - The number of actions that can
+  be performed in a second. `0` mean that ratelimiting is disabled for this user.
+- `burst_count` - integer - How many actions that can be performed before
+  being limited.
+
+If **no** custom ratelimit is set, an empty JSON dict is returned.
+
+```json
+{}
+```
+
+### Set ratelimit
+
+The API is:
+
+```
+POST /_synapse/admin/v1/users/<user_id>/override_ratelimit
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+{
+  "messages_per_second": 0,
+  "burst_count": 0
+}
+```
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
+  be local.
+
+Body parameters:
+
+- `messages_per_second` - positive integer, optional. The number of actions that can
+  be performed in a second. Defaults to `0`.
+- `burst_count` - positive integer, optional. How many actions that can be performed
+  before being limited. Defaults to `0`.
+
+To disable users' ratelimit set both values to `0`.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- `messages_per_second` - integer - The number of actions that can
+  be performed in a second.
+- `burst_count` - integer - How many actions that can be performed before
+  being limited.
+
+### Delete ratelimit
+
+The API is:
+
+```
+DELETE /_synapse/admin/v1/users/<user_id>/override_ratelimit
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+An empty JSON dict is returned.
+
+```json
+{}
+```
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
+  be local.
+
+### Check username availability
+
+Checks to see if a username is available, and valid, for the server. See [the client-server 
+API](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available)
+for more information.
+
+This endpoint will work even if registration is disabled on the server, unlike 
+`/_matrix/client/r0/register/available`.
+
+The API is:
+
+```
+POST /_synapse/admin/v1/username_availabile?username=$localpart
+```
+
+The request and response format is the same as the [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
diff --git a/docs/admin_api/user_admin_api.rst b/docs/admin_api/user_admin_api.rst
deleted file mode 100644
index 8d4ec5a6f9..0000000000
--- a/docs/admin_api/user_admin_api.rst
+++ /dev/null
@@ -1,825 +0,0 @@
-.. contents::
-
-Query User Account
-==================
-
-This API returns information about a specific user account.
-
-The api is::
-
-    GET /_synapse/admin/v2/users/<user_id>
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-It returns a JSON body like the following:
-
-.. code:: json
-
-    {
-        "displayname": "User",
-        "threepids": [
-            {
-                "medium": "email",
-                "address": "<user_mail_1>"
-            },
-            {
-                "medium": "email",
-                "address": "<user_mail_2>"
-            }
-        ],
-        "avatar_url": "<avatar_url>",
-        "admin": 0,
-        "deactivated": 0,
-        "shadow_banned": 0,
-        "password_hash": "$2b$12$p9B4GkqYdRTPGD",
-        "creation_ts": 1560432506,
-        "appservice_id": null,
-        "consent_server_notice_sent": null,
-        "consent_version": null
-    }
-
-URL parameters:
-
-- ``user_id``: fully-qualified user id: for example, ``@user:server.com``.
-
-Create or modify Account
-========================
-
-This API allows an administrator to create or modify a user account with a
-specific ``user_id``.
-
-This api is::
-
-    PUT /_synapse/admin/v2/users/<user_id>
-
-with a body of:
-
-.. code:: json
-
-    {
-        "password": "user_password",
-        "displayname": "User",
-        "threepids": [
-            {
-                "medium": "email",
-                "address": "<user_mail_1>"
-            },
-            {
-                "medium": "email",
-                "address": "<user_mail_2>"
-            }
-        ],
-        "avatar_url": "<avatar_url>",
-        "admin": false,
-        "deactivated": false
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-URL parameters:
-
-- ``user_id``: fully-qualified user id: for example, ``@user:server.com``.
-
-Body parameters:
-
-- ``password``, optional. If provided, the user's password is updated and all
-  devices are logged out.
-
-- ``displayname``, optional, defaults to the value of ``user_id``.
-
-- ``threepids``, optional, allows setting the third-party IDs (email, msisdn)
-  belonging to a user.
-
-- ``avatar_url``, optional, must be a
-  `MXC URI <https://matrix.org/docs/spec/client_server/r0.6.0#matrix-content-mxc-uris>`_.
-
-- ``admin``, optional, defaults to ``false``.
-
-- ``deactivated``, optional. If unspecified, deactivation state will be left
-  unchanged on existing accounts and set to ``false`` for new accounts.
-  A user cannot be erased by deactivating with this API. For details on deactivating users see
-  `Deactivate Account <#deactivate-account>`_.
-
-If the user already exists then optional parameters default to the current value.
-
-In order to re-activate an account ``deactivated`` must be set to ``false``. If
-users do not login via single-sign-on, a new ``password`` must be provided.
-
-List Accounts
-=============
-
-This API returns all local user accounts.
-
-The api is::
-
-    GET /_synapse/admin/v2/users?from=0&limit=10&guests=false
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-The parameter ``from`` is optional but used for pagination, denoting the
-offset in the returned results. This should be treated as an opaque value and
-not explicitly set to anything other than the return value of ``next_token``
-from a previous call.
-
-The parameter ``limit`` is optional but is used for pagination, denoting the
-maximum number of items to return in this call. Defaults to ``100``.
-
-The parameter ``user_id`` is optional and filters to only return users with user IDs
-that contain this value. This parameter is ignored when using the ``name`` parameter.
-
-The parameter ``name`` is optional and filters to only return users with user ID localparts
-**or** displaynames that contain this value.
-
-The parameter ``guests`` is optional and if ``false`` will **exclude** guest users.
-Defaults to ``true`` to include guest users.
-
-The parameter ``deactivated`` is optional and if ``true`` will **include** deactivated users.
-Defaults to ``false`` to exclude deactivated users.
-
-A JSON body is returned with the following shape:
-
-.. code:: json
-
-    {
-        "users": [
-            {
-                "name": "<user_id1>",
-                "is_guest": 0,
-                "admin": 0,
-                "user_type": null,
-                "deactivated": 0,
-                "shadow_banned": 0,
-                "displayname": "<User One>",
-                "avatar_url": null
-            }, {
-                "name": "<user_id2>",
-                "is_guest": 0,
-                "admin": 1,
-                "user_type": null,
-                "deactivated": 0,
-                "shadow_banned": 0,
-                "displayname": "<User Two>",
-                "avatar_url": "<avatar_url>"
-            }
-        ],
-        "next_token": "100",
-        "total": 200
-    }
-
-To paginate, check for ``next_token`` and if present, call the endpoint again
-with ``from`` set to the value of ``next_token``. This will return a new page.
-
-If the endpoint does not return a ``next_token`` then there are no more users
-to paginate through.
-
-Query current sessions for a user
-=================================
-
-This API returns information about the active sessions for a specific user.
-
-The api is::
-
-    GET /_synapse/admin/v1/whois/<user_id>
-
-and::
-
-    GET /_matrix/client/r0/admin/whois/<userId>
-
-See also: `Client Server API Whois
-<https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid>`_
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-It returns a JSON body like the following:
-
-.. code:: json
-
-    {
-        "user_id": "<user_id>",
-        "devices": {
-            "": {
-                "sessions": [
-                    {
-                        "connections": [
-                            {
-                                "ip": "1.2.3.4",
-                                "last_seen": 1417222374433,
-                                "user_agent": "Mozilla/5.0 ..."
-                            },
-                            {
-                                "ip": "1.2.3.10",
-                                "last_seen": 1417222374500,
-                                "user_agent": "Dalvik/2.1.0 ..."
-                            }
-                        ]
-                    }
-                ]
-            }
-        }
-    }
-
-``last_seen`` is measured in milliseconds since the Unix epoch.
-
-Deactivate Account
-==================
-
-This API deactivates an account. It removes active access tokens, resets the
-password, and deletes third-party IDs (to prevent the user requesting a
-password reset).
-
-It can also mark the user as GDPR-erased. This means messages sent by the
-user will still be visible by anyone that was in the room when these messages
-were sent, but hidden from users joining the room afterwards.
-
-The api is::
-
-    POST /_synapse/admin/v1/deactivate/<user_id>
-
-with a body of:
-
-.. code:: json
-
-    {
-        "erase": true
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-The erase parameter is optional and defaults to ``false``.
-An empty body may be passed for backwards compatibility.
-
-The following actions are performed when deactivating an user:
-
-- Try to unpind 3PIDs from the identity server
-- Remove all 3PIDs from the homeserver
-- Delete all devices and E2EE keys
-- Delete all access tokens
-- Delete the password hash
-- Removal from all rooms the user is a member of
-- Remove the user from the user directory
-- Reject all pending invites
-- Remove all account validity information related to the user
-
-The following additional actions are performed during deactivation if ``erase``
-is set to ``true``:
-
-- Remove the user's display name
-- Remove the user's avatar URL
-- Mark the user as erased
-
-
-Reset password
-==============
-
-Changes the password of another user. This will automatically log the user out of all their devices.
-
-The api is::
-
-    POST /_synapse/admin/v1/reset_password/<user_id>
-
-with a body of:
-
-.. code:: json
-
-   {
-       "new_password": "<secret>",
-       "logout_devices": true
-   }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-The parameter ``new_password`` is required.
-The parameter ``logout_devices`` is optional and defaults to ``true``.
-
-Get whether a user is a server administrator or not
-===================================================
-
-
-The api is::
-
-    GET /_synapse/admin/v1/users/<user_id>/admin
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-        "admin": true
-    }
-
-
-Change whether a user is a server administrator or not
-======================================================
-
-Note that you cannot demote yourself.
-
-The api is::
-
-    PUT /_synapse/admin/v1/users/<user_id>/admin
-
-with a body of:
-
-.. code:: json
-
-    {
-        "admin": true
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-
-List room memberships of an user
-================================
-Gets a list of all ``room_id`` that a specific ``user_id`` is member.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/joined_rooms
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-        "joined_rooms": [
-            "!DuGcnbhHGaSZQoNQR:matrix.org",
-            "!ZtSaPCawyWtxfWiIy:matrix.org"
-        ],
-        "total": 2
-    }
-
-The server returns the list of rooms of which the user and the server
-are member. If the user is local, all the rooms of which the user is
-member are returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``joined_rooms`` - An array of ``room_id``.
-- ``total`` - Number of rooms.
-
-
-List media of a user
-====================
-Gets a list of all local media that a specific ``user_id`` has created.
-By default, the response is ordered by descending creation date and ascending media ID.
-The newest media is on top. You can change the order with parameters
-``order_by`` and ``dir``.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/media
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "media": [
-        {
-          "created_ts": 100400,
-          "last_access_ts": null,
-          "media_id": "qXhyRzulkwLsNHTbpHreuEgo",
-          "media_length": 67,
-          "media_type": "image/png",
-          "quarantined_by": null,
-          "safe_from_quarantine": false,
-          "upload_name": "test1.png"
-        },
-        {
-          "created_ts": 200400,
-          "last_access_ts": null,
-          "media_id": "FHfiSnzoINDatrXHQIXBtahw",
-          "media_length": 67,
-          "media_type": "image/png",
-          "quarantined_by": null,
-          "safe_from_quarantine": false,
-          "upload_name": "test2.png"
-        }
-      ],
-      "next_token": 3,
-      "total": 2
-    }
-
-To paginate, check for ``next_token`` and if present, call the endpoint again
-with ``from`` set to the value of ``next_token``. This will return a new page.
-
-If the endpoint does not return a ``next_token`` then there are no more
-reports to paginate through.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - string - fully qualified: for example, ``@user:server.com``.
-- ``limit``: string representing a positive integer - Is optional but is used for pagination,
-  denoting the maximum number of items to return in this call. Defaults to ``100``.
-- ``from``: string representing a positive integer - Is optional but used for pagination,
-  denoting the offset in the returned results. This should be treated as an opaque value and
-  not explicitly set to anything other than the return value of ``next_token`` from a previous call.
-  Defaults to ``0``.
-- ``order_by`` - The method by which to sort the returned list of media.
-  If the ordered field has duplicates, the second order is always by ascending ``media_id``,
-  which guarantees a stable ordering. Valid values are:
-
-  - ``media_id`` - Media are ordered alphabetically by ``media_id``.
-  - ``upload_name`` - Media are ordered alphabetically by name the media was uploaded with.
-  - ``created_ts`` - Media are ordered by when the content was uploaded in ms.
-    Smallest to largest. This is the default.
-  - ``last_access_ts`` - Media are ordered by when the content was last accessed in ms.
-    Smallest to largest.
-  - ``media_length`` - Media are ordered by length of the media in bytes.
-    Smallest to largest.
-  - ``media_type`` - Media are ordered alphabetically by MIME-type.
-  - ``quarantined_by`` - Media are ordered alphabetically by the user ID that
-    initiated the quarantine request for this media.
-  - ``safe_from_quarantine`` - Media are ordered by the status if this media is safe
-    from quarantining.
-
-- ``dir`` - Direction of media order. Either ``f`` for forwards or ``b`` for backwards.
-  Setting this value to ``b`` will reverse the above sort order. Defaults to ``f``.
-
-If neither ``order_by`` nor ``dir`` is set, the default order is newest media on top
-(corresponds to ``order_by`` = ``created_ts`` and ``dir`` = ``b``).
-
-Caution. The database only has indexes on the columns ``media_id``,
-``user_id`` and ``created_ts``. This means that if a different sort order is used
-(``upload_name``, ``last_access_ts``, ``media_length``, ``media_type``,
-``quarantined_by`` or ``safe_from_quarantine``), this can cause a large load on the
-database, especially for large environments.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``media`` - An array of objects, each containing information about a media.
-  Media objects contain the following fields:
-
-  - ``created_ts`` - integer - Timestamp when the content was uploaded in ms.
-  - ``last_access_ts`` - integer - Timestamp when the content was last accessed in ms.
-  - ``media_id`` - string - The id used to refer to the media.
-  - ``media_length`` - integer - Length of the media in bytes.
-  - ``media_type`` - string - The MIME-type of the media.
-  - ``quarantined_by`` - string - The user ID that initiated the quarantine request
-    for this media.
-
-  - ``safe_from_quarantine`` - bool - Status if this media is safe from quarantining.
-  - ``upload_name`` - string - The name the media was uploaded with.
-
-- ``next_token``: integer - Indication for pagination. See above.
-- ``total`` - integer - Total number of media.
-
-Login as a user
-===============
-
-Get an access token that can be used to authenticate as that user. Useful for
-when admins wish to do actions on behalf of a user.
-
-The API is::
-
-    POST /_synapse/admin/v1/users/<user_id>/login
-    {}
-
-An optional ``valid_until_ms`` field can be specified in the request body as an
-integer timestamp that specifies when the token should expire. By default tokens
-do not expire.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-        "access_token": "<opaque_access_token_string>"
-    }
-
-
-This API does *not* generate a new device for the user, and so will not appear
-their ``/devices`` list, and in general the target user should not be able to
-tell they have been logged in as.
-
-To expire the token call the standard ``/logout`` API with the token.
-
-Note: The token will expire if the *admin* user calls ``/logout/all`` from any
-of their devices, but the token will *not* expire if the target user does the
-same.
-
-
-User devices
-============
-
-List all devices
-----------------
-Gets information about all devices for a specific ``user_id``.
-
-The API is::
-
-  GET /_synapse/admin/v2/users/<user_id>/devices
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "devices": [
-        {
-          "device_id": "QBUAZIFURK",
-          "display_name": "android",
-          "last_seen_ip": "1.2.3.4",
-          "last_seen_ts": 1474491775024,
-          "user_id": "<user_id>"
-        },
-        {
-          "device_id": "AUIECTSRND",
-          "display_name": "ios",
-          "last_seen_ip": "1.2.3.5",
-          "last_seen_ts": 1474491775025,
-          "user_id": "<user_id>"
-        }
-      ],
-      "total": 2
-    }
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``devices`` - An array of objects, each containing information about a device.
-  Device objects contain the following fields:
-
-  - ``device_id`` - Identifier of device.
-  - ``display_name`` - Display name set by the user for this device.
-    Absent if no name has been set.
-  - ``last_seen_ip`` - The IP address where this device was last seen.
-    (May be a few minutes out of date, for efficiency reasons).
-  - ``last_seen_ts`` - The timestamp (in milliseconds since the unix epoch) when this
-    devices was last seen. (May be a few minutes out of date, for efficiency reasons).
-  - ``user_id`` - Owner of  device.
-
-- ``total`` - Total number of user's devices.
-
-Delete multiple devices
-------------------
-Deletes the given devices for a specific ``user_id``, and invalidates
-any access token associated with them.
-
-The API is::
-
-    POST /_synapse/admin/v2/users/<user_id>/delete_devices
-
-    {
-      "devices": [
-        "QBUAZIFURK",
-        "AUIECTSRND"
-      ],
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-The following fields are required in the JSON request body:
-
-- ``devices`` - The list of device IDs to delete.
-
-Show a device
----------------
-Gets information on a single device, by ``device_id`` for a specific ``user_id``.
-
-The API is::
-
-    GET /_synapse/admin/v2/users/<user_id>/devices/<device_id>
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "device_id": "<device_id>",
-      "display_name": "android",
-      "last_seen_ip": "1.2.3.4",
-      "last_seen_ts": 1474491775024,
-      "user_id": "<user_id>"
-    }
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-- ``device_id`` - The device to retrieve.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``device_id`` - Identifier of device.
-- ``display_name`` - Display name set by the user for this device.
-  Absent if no name has been set.
-- ``last_seen_ip`` - The IP address where this device was last seen.
-  (May be a few minutes out of date, for efficiency reasons).
-- ``last_seen_ts`` - The timestamp (in milliseconds since the unix epoch) when this
-  devices was last seen. (May be a few minutes out of date, for efficiency reasons).
-- ``user_id`` - Owner of  device.
-
-Update a device
----------------
-Updates the metadata on the given ``device_id`` for a specific ``user_id``.
-
-The API is::
-
-    PUT /_synapse/admin/v2/users/<user_id>/devices/<device_id>
-
-    {
-      "display_name": "My other phone"
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-- ``device_id`` - The device to update.
-
-The following fields are required in the JSON request body:
-
-- ``display_name`` - The new display name for this device. If not given,
-  the display name is unchanged.
-
-Delete a device
----------------
-Deletes the given ``device_id`` for a specific ``user_id``,
-and invalidates any access token associated with it.
-
-The API is::
-
-    DELETE /_synapse/admin/v2/users/<user_id>/devices/<device_id>
-
-    {}
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-- ``device_id`` - The device to delete.
-
-List all pushers
-================
-Gets information about all pushers for a specific ``user_id``.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/pushers
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "pushers": [
-        {
-          "app_display_name":"HTTP Push Notifications",
-          "app_id":"m.http",
-          "data": {
-            "url":"example.com"
-          },
-          "device_display_name":"pushy push",
-          "kind":"http",
-          "lang":"None",
-          "profile_tag":"",
-          "pushkey":"a@example.com"
-        }
-      ],
-      "total": 1
-    }
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``pushers`` - An array containing the current pushers for the user
-
-  - ``app_display_name`` - string - A string that will allow the user to identify
-    what application owns this pusher.
-
-  - ``app_id`` - string - This is a reverse-DNS style identifier for the application.
-    Max length, 64 chars.
-
-  - ``data`` - A dictionary of information for the pusher implementation itself.
-
-    - ``url`` - string - Required if ``kind`` is ``http``. The URL to use to send
-      notifications to.
-
-    - ``format`` - string - The format to use when sending notifications to the
-      Push Gateway.
-
-  - ``device_display_name`` - string -  A string that will allow the user to identify
-    what device owns this pusher.
-
-  - ``profile_tag`` - string - This string determines which set of device specific rules
-    this pusher executes.
-
-  - ``kind`` - string -  The kind of pusher. "http" is a pusher that sends HTTP pokes.
-  - ``lang`` - string - The preferred language for receiving notifications
-    (e.g. 'en' or 'en-US')
-
-  - ``profile_tag`` - string - This string determines which set of device specific rules
-    this pusher executes.
-
-  - ``pushkey`` - string - This is a unique identifier for this pusher.
-    Max length, 512 bytes.
-
-- ``total`` - integer - Number of pushers.
-
-See also `Client-Server API Spec <https://matrix.org/docs/spec/client_server/latest#get-matrix-client-r0-pushers>`_
-
-Shadow-banning users
-====================
-
-Shadow-banning is a useful tool for moderating malicious or egregiously abusive users.
-A shadow-banned users receives successful responses to their client-server API requests,
-but the events are not propagated into rooms. This can be an effective tool as it
-(hopefully) takes longer for the user to realise they are being moderated before
-pivoting to another account.
-
-Shadow-banning a user should be used as a tool of last resort and may lead to confusing
-or broken behaviour for the client. A shadow-banned user will not receive any
-notification and it is generally more appropriate to ban or kick abusive users.
-A shadow-banned user will be unable to contact anyone on the server.
-
-The API is::
-
-  POST /_synapse/admin/v1/users/<user_id>/shadow_ban
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
-  be local.
diff --git a/docs/admin_api/version_api.rst b/docs/admin_api/version_api.md
similarity index 59%
rename from docs/admin_api/version_api.rst
rename to docs/admin_api/version_api.md
index 833d9028be..efb4a0c0f7 100644
--- a/docs/admin_api/version_api.rst
+++ b/docs/admin_api/version_api.md
@@ -1,20 +1,21 @@
-Version API
-===========
+# Version API
 
 This API returns the running Synapse version and the Python version
 on which Synapse is being run. This is useful when a Synapse instance
 is behind a proxy that does not forward the 'Server' header (which also
 contains Synapse version information).
 
-The api is::
+The api is:
 
-    GET /_synapse/admin/v1/server_version
+```
+GET /_synapse/admin/v1/server_version
+```
 
 It returns a JSON body like the following:
 
-.. code:: json
-
-    {
-        "server_version": "0.99.2rc1 (b=develop, abcdef123)",
-        "python_version": "3.6.8"
-    }
+```json
+{
+    "server_version": "0.99.2rc1 (b=develop, abcdef123)",
+    "python_version": "3.6.8"
+}
+```
diff --git a/docs/code_style.md b/docs/code_style.md
index 190f8ab2de..28fb7277c4 100644
--- a/docs/code_style.md
+++ b/docs/code_style.md
@@ -128,6 +128,9 @@ Some guidelines follow:
     will be if no sub-options are enabled).
 -   Lines should be wrapped at 80 characters.
 -   Use two-space indents.
+-   `true` and `false` are spelt thus (as opposed to `True`, etc.)
+-   Use single quotes (`'`) rather than double-quotes (`"`) or backticks
+    (`` ` ``) to refer to configuration options.
 
 Example:
 
diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md
index c586b5f0b6..911a1f95db 100644
--- a/docs/consent_tracking.md
+++ b/docs/consent_tracking.md
@@ -24,8 +24,8 @@ To enable this, first create templates for the policy and success pages.
 These should be stored on the local filesystem.
 
 These templates use the [Jinja2](http://jinja.pocoo.org) templating language,
-and [docs/privacy_policy_templates](privacy_policy_templates) gives
-examples of the sort of thing that can be done.
+and [docs/privacy_policy_templates](https://github.com/matrix-org/synapse/tree/develop/docs/privacy_policy_templates/)
+gives examples of the sort of thing that can be done.
 
 Note that the templates must be stored under a name giving the language of the
 template - currently this must always be `en` (for "English");
@@ -152,7 +152,7 @@ version of the policy. To do so:
 
  * ensure that the consent resource is configured, as in the previous section
 
- * ensure that server notices are configured, as in [server_notices.md](server_notices.md).
+ * ensure that server notices are configured, as in [the server notice documentation](server_notices.md).
 
  * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For
    example:
diff --git a/docs/delegate.md b/docs/delegate.md
index 208ddb6277..05cb635047 100644
--- a/docs/delegate.md
+++ b/docs/delegate.md
@@ -74,7 +74,7 @@ We no longer actively recommend against using a reverse proxy. Many admins will
 find it easier to direct federation traffic to a reverse proxy and manage their
 own TLS certificates, and this is a supported configuration.
 
-See [reverse_proxy.md](reverse_proxy.md) for information on setting up a
+See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
 reverse proxy.
 
 ### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?
diff --git a/docs/dev/cas.md b/docs/development/cas.md
similarity index 100%
rename from docs/dev/cas.md
rename to docs/development/cas.md
diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md
new file mode 100644
index 0000000000..ddf0887123
--- /dev/null
+++ b/docs/development/contributing_guide.md
@@ -0,0 +1,7 @@
+<!--
+  Include the contents of CONTRIBUTING.md from the project root (where GitHub likes it 
+  to be)
+-->
+# Contributing
+
+{{#include ../../CONTRIBUTING.md}}
diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md
new file mode 100644
index 0000000000..20740cf5ac
--- /dev/null
+++ b/docs/development/database_schema.md
@@ -0,0 +1,137 @@
+# Synapse database schema files
+
+Synapse's database schema is stored in the `synapse.storage.schema` module.
+
+## Logical databases
+
+Synapse supports splitting its datastore across multiple physical databases (which can
+be useful for large installations), and the schema files are therefore split according
+to the logical database they apply to.
+
+At the time of writing, the following "logical" databases are supported:
+
+* `state` - used to store Matrix room state (more specifically, `state_groups`,
+  their relationships and contents).
+* `main` - stores everything else.
+
+Additionally, the `common` directory contains schema files for tables which must be
+present on *all* physical databases.
+
+## Synapse schema versions
+
+Synapse manages its database schema via "schema versions". These are mainly used to
+help avoid confusion if the Synapse codebase is rolled back after the database is
+updated. They work as follows:
+
+ * The Synapse codebase defines a constant `synapse.storage.schema.SCHEMA_VERSION`
+   which represents the expectations made about the database by that version. For
+   example, as of Synapse v1.36, this is `59`.
+
+ * The database stores a "compatibility version" in
+   `schema_compat_version.compat_version` which defines the `SCHEMA_VERSION` of the
+   oldest version of Synapse which will work with the database. On startup, if
+   `compat_version` is found to be newer than `SCHEMA_VERSION`, Synapse will refuse to
+   start.
+
+   Synapse automatically updates this field from
+   `synapse.storage.schema.SCHEMA_COMPAT_VERSION`.
+
+ * Whenever a backwards-incompatible change is made to the database format (normally
+   via a `delta` file), `synapse.storage.schema.SCHEMA_COMPAT_VERSION` is also updated
+   so that administrators can not accidentally roll back to a too-old version of Synapse.
+
+Generally, the goal is to maintain compatibility with at least one or two previous
+releases of Synapse, so any substantial change tends to require multiple releases and a
+bit of forward-planning to get right.
+
+As a worked example: we want to remove the `room_stats_historical` table. Here is how it
+might pan out.
+
+ 1. Replace any code that *reads* from `room_stats_historical` with alternative
+    implementations, but keep writing to it in case of rollback to an earlier version.
+    Also, increase `synapse.storage.schema.SCHEMA_VERSION`.  In this
+    instance, there is no existing code which reads from `room_stats_historical`, so
+    our starting point is:
+
+    v1.36.0: `SCHEMA_VERSION=59`, `SCHEMA_COMPAT_VERSION=59`
+
+ 2. Next (say in Synapse v1.37.0): remove the code that *writes* to
+    `room_stats_historical`, but don’t yet remove the table in case of rollback to
+    v1.36.0. Again, we increase `synapse.storage.schema.SCHEMA_VERSION`, but
+    because we have not broken compatibility with v1.36, we do not yet update
+    `SCHEMA_COMPAT_VERSION`. We now have:
+
+    v1.37.0: `SCHEMA_VERSION=60`, `SCHEMA_COMPAT_VERSION=59`.
+
+ 3. Later (say in Synapse v1.38.0): we can remove the table altogether. This will
+    break compatibility with v1.36.0, so we must update `SCHEMA_COMPAT_VERSION` accordingly.
+    There is no need to update `synapse.storage.schema.SCHEMA_VERSION`, since there is no
+    change to the Synapse codebase here. So we end up with:
+
+    v1.38.0: `SCHEMA_VERSION=60`, `SCHEMA_COMPAT_VERSION=60`.
+
+If in doubt about whether to update `SCHEMA_VERSION` or not, it is generally best to
+lean towards doing so.
+
+## Full schema dumps
+
+In the `full_schemas` directories, only the most recently-numbered snapshot is used
+(`54` at the time of writing). Older snapshots (eg, `16`) are present for historical
+reference only.
+
+### Building full schema dumps
+
+If you want to recreate these schemas, they need to be made from a database that
+has had all background updates run.
+
+To do so, use `scripts-dev/make_full_schema.sh`. This will produce new
+`full.sql.postgres` and `full.sql.sqlite` files.
+
+Ensure postgres is installed, then run:
+
+    ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
+
+NB at the time of writing, this script predates the split into separate `state`/`main`
+databases so will require updates to handle that correctly.
+
+## Boolean columns
+
+Boolean columns require special treatment, since SQLite treats booleans the
+same as integers.
+
+There are three separate aspects to this:
+
+ * Any new boolean column must be added to the `BOOLEAN_COLUMNS` list in
+   `scripts/synapse_port_db`. This tells the port script to cast the integer
+   value from SQLite to a boolean before writing the value to the postgres
+   database.
+
+ * Before SQLite 3.23, `TRUE` and `FALSE` were not recognised as constants by
+   SQLite, and the `IS [NOT] TRUE`/`IS [NOT] FALSE` operators were not
+   supported. This makes it necessary to avoid using `TRUE` and `FALSE`
+   constants in SQL commands.
+
+   For example, to insert a `TRUE` value into the database, write:
+
+   ```python
+   txn.execute("INSERT INTO tbl(col) VALUES (?)", (True, ))
+   ```
+
+ * Default values for new boolean columns present a particular
+   difficulty. Generally it is best to create separate schema files for
+   Postgres and SQLite. For example:
+
+   ```sql
+   # in 00delta.sql.postgres:
+   ALTER TABLE tbl ADD COLUMN col BOOLEAN DEFAULT FALSE;
+   ```
+
+   ```sql
+   # in 00delta.sql.sqlite:
+   ALTER TABLE tbl ADD COLUMN col BOOLEAN DEFAULT 0;
+   ```
+
+   Note that there is a particularly insidious failure mode here: the Postgres
+   flavour will be accepted by SQLite 3.22, but will give a column whose
+   default value is the **string** `"FALSE"` - which, when cast back to a boolean
+   in Python, evaluates to `True`.
diff --git a/docs/dev/git.md b/docs/development/git.md
similarity index 94%
rename from docs/dev/git.md
rename to docs/development/git.md
index b747ff20c9..9b1ed54b65 100644
--- a/docs/dev/git.md
+++ b/docs/development/git.md
@@ -9,7 +9,7 @@ commits each of which contains a single change building on what came
 before. Here, by way of an arbitrary example, is the top of `git log --graph
 b2dba0607`:
 
-<img src="git/clean.png" alt="clean git graph" width="500px">
+<img src="img/git/clean.png" alt="clean git graph" width="500px">
 
 Note how the commit comment explains clearly what is changing and why. Also
 note the *absence* of merge commits, as well as the absence of commits called
@@ -61,7 +61,7 @@ Ok, so that's what we'd like to achieve. How do we achieve it?
 The TL;DR is: when you come to merge a pull request, you *probably* want to
 “squash and merge”:
 
-![squash and merge](git/squash.png).
+![squash and merge](img/git/squash.png).
 
 (This applies whether you are merging your own PR, or that of another
 contributor.)
@@ -105,7 +105,7 @@ complicated. Here's how we do it.
 
 Let's start with a picture:
 
-![branching model](git/branches.jpg)
+![branching model](img/git/branches.jpg)
 
 It looks complicated, but it's really not. There's one basic rule: *anyone* is
 free to merge from *any* more-stable branch to *any* less-stable branch at
@@ -122,15 +122,15 @@ So, what counts as a more- or less-stable branch? A little reflection will show
 that our active branches are ordered thus, from more-stable to less-stable:
 
  * `master` (tracks our last release).
- * `release-vX.Y.Z` (the branch where we prepare the next release)<sup
+ * `release-vX.Y` (the branch where we prepare the next release)<sup
    id="a3">[3](#f3)</sup>.
  * PR branches which are targeting the release.
  * `develop` (our "mainline" branch containing our bleeding-edge).
  * regular PR branches.
 
 The corollary is: if you have a bugfix that needs to land in both
-`release-vX.Y.Z` *and* `develop`, then you should base your PR on
-`release-vX.Y.Z`, get it merged there, and then merge from `release-vX.Y.Z` to
+`release-vX.Y` *and* `develop`, then you should base your PR on
+`release-vX.Y`, get it merged there, and then merge from `release-vX.Y` to
 `develop`. (If a fix lands in `develop` and we later need it in a
 release-branch, we can of course cherry-pick it, but landing it in the release
 branch first helps reduce the chance of annoying conflicts.)
@@ -145,4 +145,4 @@ most intuitive name. [^](#a1)
 
 <b id="f3">[3]</b>: Very, very occasionally (I think this has happened once in
 the history of Synapse), we've had two releases in flight at once. Obviously,
-`release-v1.2.3` is more-stable than `release-v1.3.0`. [^](#a3)
+`release-v1.2` is more-stable than `release-v1.3`. [^](#a3)
diff --git a/docs/dev/git/branches.jpg b/docs/development/img/git/branches.jpg
similarity index 100%
rename from docs/dev/git/branches.jpg
rename to docs/development/img/git/branches.jpg
diff --git a/docs/dev/git/clean.png b/docs/development/img/git/clean.png
similarity index 100%
rename from docs/dev/git/clean.png
rename to docs/development/img/git/clean.png
diff --git a/docs/dev/git/squash.png b/docs/development/img/git/squash.png
similarity index 100%
rename from docs/dev/git/squash.png
rename to docs/development/img/git/squash.png
diff --git a/docs/development/internal_documentation/README.md b/docs/development/internal_documentation/README.md
new file mode 100644
index 0000000000..51c5fb94d5
--- /dev/null
+++ b/docs/development/internal_documentation/README.md
@@ -0,0 +1,12 @@
+# Internal Documentation
+
+This section covers implementation documentation for various parts of Synapse.
+
+If a developer is planning to make a change to a feature of Synapse, it can be useful for
+general documentation of how that feature is implemented to be available. This saves the
+developer time in place of needing to understand how the feature works by reading the
+code.
+
+Documentation that would be more useful for the perspective of a system administrator,
+rather than a developer who's intending to change to code, should instead be placed
+under the Usage section of the documentation.
\ No newline at end of file
diff --git a/docs/development/room-dag-concepts.md b/docs/development/room-dag-concepts.md
new file mode 100644
index 0000000000..5eed72bec6
--- /dev/null
+++ b/docs/development/room-dag-concepts.md
@@ -0,0 +1,79 @@
+# Room DAG concepts
+
+## Edges
+
+The word "edge" comes from graph theory lingo. An edge is just a connection
+between two events. In Synapse, we connect events by specifying their
+`prev_events`. A subsequent event points back at a previous event.
+
+```
+A (oldest) <---- B <---- C (most recent)
+```
+
+
+## Depth and stream ordering
+
+Events are normally sorted by `(topological_ordering, stream_ordering)` where
+`topological_ordering` is just `depth`. In other words, we first sort by `depth`
+and then tie-break based on `stream_ordering`. `depth` is incremented as new
+messages are added to the DAG. Normally, `stream_ordering` is an auto
+incrementing integer, but backfilled events start with `stream_ordering=-1` and decrement.
+
+---
+
+ - `/sync` returns things in the order they arrive at the server (`stream_ordering`).
+ - `/messages` (and `/backfill` in the federation API) return them in the order determined by the event graph `(topological_ordering, stream_ordering)`.
+
+The general idea is that, if you're following a room in real-time (i.e.
+`/sync`), you probably want to see the messages as they arrive at your server,
+rather than skipping any that arrived late; whereas if you're looking at a
+historical section of timeline (i.e. `/messages`), you want to see the best
+representation of the state of the room as others were seeing it at the time.
+
+
+## Forward extremity
+
+Most-recent-in-time events in the DAG which are not referenced by any other events' `prev_events` yet.
+
+The forward extremities of a room are used as the `prev_events` when the next event is sent.
+
+
+## Backwards extremity
+
+The current marker of where we have backfilled up to and will generally be the
+oldest-in-time events we know of in the DAG.
+
+This is an event where we haven't fetched all of the `prev_events` for.
+
+Once we have fetched all of its `prev_events`, it's unmarked as a backwards
+extremity (although we may have formed new backwards extremities from the prev
+events during the backfilling process).
+
+
+## Outliers
+
+We mark an event as an `outlier` when we haven't figured out the state for the
+room at that point in the DAG yet.
+
+We won't *necessarily* have the `prev_events` of an `outlier` in the database,
+but it's entirely possible that we *might*. The status of whether we have all of
+the `prev_events` is marked as a [backwards extremity](#backwards-extremity).
+
+For example, when we fetch the event auth chain or state for a given event, we
+mark all of those claimed auth events as outliers because we haven't done the
+state calculation ourself.
+
+
+## State groups
+
+For every non-outlier event we need to know the state at that event. Instead of
+storing the full state for each event in the DB (i.e. a `event_id -> state`
+mapping), which is *very* space inefficient when state doesn't change, we
+instead assign each different set of state a "state group" and then have
+mappings of `event_id -> state_group` and `state_group -> state`.
+
+
+### Stage group edges
+
+TODO: `state_group_edges` is a further optimization...
+      notes from @Azrenbeth, https://pastebin.com/seUGVGeT
diff --git a/docs/dev/saml.md b/docs/development/saml.md
similarity index 100%
rename from docs/dev/saml.md
rename to docs/development/saml.md
diff --git a/docs/favicon.png b/docs/favicon.png
new file mode 100644
index 0000000000..5f18bf641f
Binary files /dev/null and b/docs/favicon.png differ
diff --git a/docs/favicon.svg b/docs/favicon.svg
new file mode 100644
index 0000000000..e571aeb3ed
--- /dev/null
+++ b/docs/favicon.svg
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   viewBox="0 0 199.7 184.2"
+   version="1.1"
+   id="svg62"
+   sodipodi:docname="mdbook-favicon.svg"
+   inkscape:version="1.0.2 (e86c870879, 2021-01-15, custom)">
+  <metadata
+     id="metadata68">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs66" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1920"
+     inkscape:window-height="1026"
+     id="namedview64"
+     showgrid="false"
+     inkscape:zoom="3.2245912"
+     inkscape:cx="84.790185"
+     inkscape:cy="117.96478"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg62" />
+  <style
+     id="style58">
+    @media (prefers-color-scheme: dark) {
+      svg { fill: white; }
+    }
+  </style>
+  <path
+     d="m 189.5,36.8 c 0.2,2.8 0,5.1 -0.6,6.8 L 153,162 c -0.6,2.1 -2,3.7 -4.2,5 -2.2,1.2 -4.4,1.9 -6.7,1.9 H 31.4 c -9.6,0 -15.3,-2.8 -17.3,-8.4 -0.8,-2.2 -0.8,-3.9 0.1,-5.2 0.9,-1.2 2.4,-1.8 4.6,-1.8 H 123 c 7.4,0 12.6,-1.4 15.4,-4.1 2.8,-2.7 5.7,-8.9 8.6,-18.4 L 179.9,22.4 c 1.8,-5.9 1,-11.1 -2.2,-15.6 C 174.5,2.3 169.9,0 164,0 H 72.7 c -1,0 -3.1,0.4 -6.1,1.1 L 66.7,0.7 C 64.5,0.2 62.6,0 61,0.1 c -1.6,0.1 -3,0.5 -4.3,1.4 -1.3,0.9 -2.4,1.8 -3.2,2.8 -0.8,1 -1.5,2.2 -2.3,3.8 -0.8,1.6 -1.4,3 -1.9,4.3 -0.5,1.3 -1.1,2.7 -1.8,4.2 -0.7,1.5 -1.3,2.7 -2,3.7 -0.5,0.6 -1.2,1.5 -2,2.5 -0.8,1 -1.6,2 -2.2,2.8 -0.6,0.8 -0.9,1.5 -1.1,2.2 -0.2,0.7 -0.1,1.8 0.2,3.2 0.3,1.4 0.4,2.4 0.4,3.1 -0.3,3 -1.4,6.9 -3.3,11.6 -1.9,4.7 -3.6,8.1 -5.1,10.1 -0.3,0.4 -1.2,1.3 -2.6,2.7 -1.4,1.4 -2.3,2.6 -2.6,3.7 -0.3,0.4 -0.3,1.5 -0.1,3.4 0.3,1.8 0.4,3.1 0.3,3.8 -0.3,2.7 -1.3,6.3 -3,10.8 -2.406801,6.370944 -3.4,8.2 -5,11 -0.2,0.5 -0.9,1.4 -2,2.8 -1.1,1.4 -1.8,2.5 -2,3.4 -0.2,0.6 -0.1,1.8 0.1,3.4 0.2,1.6 0.2,2.8 -0.1,3.6 -0.6,3 -1.8,6.7 -3.6,11 -1.8,4.3 -3.6,7.9 -5.4,11 -0.5,0.8 -1.1,1.7 -2,2.8 -0.8,1.1 -1.5,2 -2,2.8 -0.5,0.8 -0.8,1.6 -1,2.5 -0.1,0.5 0,1.3 0.4,2.3 0.3,1.1 0.4,1.9 0.4,2.6 -0.1,1.1 -0.2,2.6 -0.5,4.4 -0.2,1.8 -0.4,2.9 -0.4,3.2 -1.8,4.8 -1.7,9.9 0.2,15.2 2.2,6.2 6.2,11.5 11.9,15.8 5.7,4.3 11.7,6.4 17.8,6.4 h 110.7 c 5.2,0 10.1,-1.7 14.7,-5.2 4.6,-3.5 7.7,-7.8 9.2,-12.9 l 33,-108.6 c 1.8,-5.8 1,-10.9 -2.2,-15.5 -1.7,-2.5 -4,-4.2 -7.1,-5.4 z M 38.14858,105.59813 60.882735,41.992545 h 10.8 c 6.340631,0 33.351895,0.778957 70.804135,0.970479 -18.18245,63.254766 0,0 -18.18245,63.254766 -23.00947,-0.10382 -63.362955,-0.6218 -72.55584,-0.51966 -18,0.2 -13.6,-0.1 -13.6,-0.1 z m 80.621,-5.891206 c 15.19043,-50.034423 0,1e-5 15.19043,-50.034423 l -11.90624,-0.13228 2.73304,-9.302941 -44.32863,0.07339 -2.532953,8.036036 -11.321128,-0.18864 -17.955519,51.440073 c 0.02698,0.027 4.954586,0.0514 12.187488,0.0717 l -2.997994,9.804886 c 11.36463,0.0271 1.219679,-0.0736 46.117666,-0.31499 l 2.65246,-9.571696 c 7.08021,0.14819 11.59705,0.13117 12.16138,0.1189 z m -56.149615,-3.855606 13.7,-42.5 h 9.8 l 1.194896,32.99936 23.205109,-32.99936 h 9.9 l -13.6,42.5 h -7.099996 l 12.499996,-35.4 -24.50001,35.4 h -6.799995 l -0.8,-35 -10.8,35 z"
+     id="path60"
+     sodipodi:nodetypes="ccccssccsssccsssccsssssscsssscssscccscscscsccsccccccssssccccccsccsccccccccccccccccccccccccccccc" />
+</svg>
diff --git a/docs/federate.md b/docs/federate.md
index b15cd724d1..5107f995be 100644
--- a/docs/federate.md
+++ b/docs/federate.md
@@ -14,7 +14,7 @@ you set the `server_name` to match your machine's public DNS hostname.
 
 For this default configuration to work, you will need to listen for TLS
 connections on port 8448. The preferred way to do that is by using a
-reverse proxy: see [reverse_proxy.md](<reverse_proxy.md>) for instructions
+reverse proxy: see [the reverse proxy documentation](reverse_proxy.md) for instructions
 on how to correctly set one up.
 
 In some cases you might not want to run Synapse on the machine that has
@@ -23,7 +23,7 @@ traffic to use a different port than 8448. For example, you might want to
 have your user names look like `@user:example.com`, but you want to run
 Synapse on `synapse.example.com` on port 443. This can be done using
 delegation, which allows an admin to control where federation traffic should
-be sent. See [delegate.md](delegate.md) for instructions on how to set this up.
+be sent. See [the delegation documentation](delegate.md) for instructions on how to set this up.
 
 Once federation has been configured, you should be able to join a room over
 federation. A good place to start is `#synapse:matrix.org` - a room for
@@ -44,8 +44,8 @@ a complicated dance which requires connections in both directions).
 
 Another common problem is that people on other servers can't join rooms that
 you invite them to. This can be caused by an incorrectly-configured reverse
-proxy: see [reverse_proxy.md](<reverse_proxy.md>) for instructions on how to correctly
-configure a reverse proxy.
+proxy: see [the reverse proxy documentation](reverse_proxy.md) for instructions on how
+to correctly configure a reverse proxy.
 
 ### Known issues
 
@@ -63,4 +63,4 @@ release of Synapse.
 
 If you want to get up and running quickly with a trio of homeservers in a
 private federation, there is a script in the `demo` directory. This is mainly
-useful just for development purposes. See [demo/README](<../demo/README>).
+useful just for development purposes. See [demo/README](https://github.com/matrix-org/synapse/tree/develop/demo/).
diff --git a/docs/log_contexts.md b/docs/log_contexts.md
index fe30ca2791..d49dce8830 100644
--- a/docs/log_contexts.md
+++ b/docs/log_contexts.md
@@ -14,12 +14,16 @@ The `synapse.logging.context` module provides a facilities for managing
 the current log context (as well as providing the `LoggingContextFilter`
 class).
 
-Deferreds make the whole thing complicated, so this document describes
+Asynchronous functions make the whole thing complicated, so this document describes
 how it all works, and how to write code which follows the rules.
 
-##Logcontexts without Deferreds
+In this document, "awaitable" refers to any object which can be `await`ed. In the context of
+Synapse, that normally means either a coroutine or a Twisted 
+[`Deferred`](https://twistedmatrix.com/documents/current/api/twisted.internet.defer.Deferred.html).
 
-In the absence of any Deferred voodoo, things are simple enough. As with
+## Logcontexts without asynchronous code
+
+In the absence of any asynchronous voodoo, things are simple enough. As with
 any code of this nature, the rule is that our function should leave
 things as it found them:
 
@@ -55,126 +59,109 @@ def do_request_handling():
     logger.debug("phew")
 ```
 
-## Using logcontexts with Deferreds
+## Using logcontexts with awaitables
 
-Deferreds --- and in particular, `defer.inlineCallbacks` --- break the
-linear flow of code so that there is no longer a single entry point
-where we should set the logcontext and a single exit point where we
-should remove it.
+Awaitables break the linear flow of code so that there is no longer a single entry point
+where we should set the logcontext and a single exit point where we should remove it.
 
 Consider the example above, where `do_request_handling` needs to do some
-blocking operation, and returns a deferred:
+blocking operation, and returns an awaitable:
 
 ```python
-@defer.inlineCallbacks
-def handle_request(request_id):
+async def handle_request(request_id):
     with context.LoggingContext() as request_context:
         request_context.request = request_id
-        yield do_request_handling()
+        await do_request_handling()
         logger.debug("finished")
 ```
 
 In the above flow:
 
 -   The logcontext is set
--   `do_request_handling` is called, and returns a deferred
--   `handle_request` yields the deferred
--   The `inlineCallbacks` wrapper of `handle_request` returns a deferred
+-   `do_request_handling` is called, and returns an awaitable
+-   `handle_request` awaits the awaitable
+-   Execution of `handle_request` is suspended
 
 So we have stopped processing the request (and will probably go on to
 start processing the next), without clearing the logcontext.
 
 To circumvent this problem, synapse code assumes that, wherever you have
-a deferred, you will want to yield on it. To that end, whereever
-functions return a deferred, we adopt the following conventions:
+an awaitable, you will want to `await` it. To that end, whereever
+functions return awaitables, we adopt the following conventions:
 
-**Rules for functions returning deferreds:**
+**Rules for functions returning awaitables:**
 
-> -   If the deferred is already complete, the function returns with the
+> -   If the awaitable is already complete, the function returns with the
 >     same logcontext it started with.
-> -   If the deferred is incomplete, the function clears the logcontext
->     before returning; when the deferred completes, it restores the
+> -   If the awaitable is incomplete, the function clears the logcontext
+>     before returning; when the awaitable completes, it restores the
 >     logcontext before running any callbacks.
 
 That sounds complicated, but actually it means a lot of code (including
 the example above) "just works". There are two cases:
 
--   If `do_request_handling` returns a completed deferred, then the
+-   If `do_request_handling` returns a completed awaitable, then the
     logcontext will still be in place. In this case, execution will
-    continue immediately after the `yield`; the "finished" line will
+    continue immediately after the `await`; the "finished" line will
     be logged against the right context, and the `with` block restores
     the original context before we return to the caller.
--   If the returned deferred is incomplete, `do_request_handling` clears
+-   If the returned awaitable is incomplete, `do_request_handling` clears
     the logcontext before returning. The logcontext is therefore clear
-    when `handle_request` yields the deferred. At that point, the
-    `inlineCallbacks` wrapper adds a callback to the deferred, and
-    returns another (incomplete) deferred to the caller, and it is safe
-    to begin processing the next request.
-
-    Once `do_request_handling`'s deferred completes, it will reinstate
-    the logcontext, before running the callback added by the
-    `inlineCallbacks` wrapper. That callback runs the second half of
-    `handle_request`, so again the "finished" line will be logged
-    against the right context, and the `with` block restores the
-    original context.
+    when `handle_request` `await`s the awaitable.
+
+    Once `do_request_handling`'s awaitable completes, it will reinstate
+    the logcontext, before running the second half of `handle_request`,
+    so again the "finished" line will be logged against the right context,
+    and the `with` block restores the original context.
 
 As an aside, it's worth noting that `handle_request` follows our rules
--though that only matters if the caller has its own logcontext which it
+- though that only matters if the caller has its own logcontext which it
 cares about.
 
 The following sections describe pitfalls and helpful patterns when
 implementing these rules.
 
-Always yield your deferreds
----------------------------
+Always await your awaitables
+----------------------------
 
-Whenever you get a deferred back from a function, you should `yield` on
-it as soon as possible. (Returning it directly to your caller is ok too,
-if you're not doing `inlineCallbacks`.) Do not pass go; do not do any
-logging; do not call any other functions.
+Whenever you get an awaitable back from a function, you should `await` on
+it as soon as possible. Do not pass go; do not do any logging; do not
+call any other functions.
 
 ```python
-@defer.inlineCallbacks
-def fun():
+async def fun():
     logger.debug("starting")
-    yield do_some_stuff()       # just like this
+    await do_some_stuff()       # just like this
 
-    d = more_stuff()
-    result = yield d            # also fine, of course
+    coro = more_stuff()
+    result = await coro         # also fine, of course
 
     return result
-
-def nonInlineCallbacksFun():
-    logger.debug("just a wrapper really")
-    return do_some_stuff()      # this is ok too - the caller will yield on
-                                # it anyway.
 ```
 
 Provided this pattern is followed all the way back up to the callchain
 to where the logcontext was set, this will make things work out ok:
 provided `do_some_stuff` and `more_stuff` follow the rules above, then
-so will `fun` (as wrapped by `inlineCallbacks`) and
-`nonInlineCallbacksFun`.
+so will `fun`.
 
-It's all too easy to forget to `yield`: for instance if we forgot that
-`do_some_stuff` returned a deferred, we might plough on regardless. This
+It's all too easy to forget to `await`: for instance if we forgot that
+`do_some_stuff` returned an awaitable, we might plough on regardless. This
 leads to a mess; it will probably work itself out eventually, but not
 before a load of stuff has been logged against the wrong context.
 (Normally, other things will break, more obviously, if you forget to
-`yield`, so this tends not to be a major problem in practice.)
+`await`, so this tends not to be a major problem in practice.)
 
 Of course sometimes you need to do something a bit fancier with your
-Deferreds - not all code follows the linear A-then-B-then-C pattern.
+awaitable - not all code follows the linear A-then-B-then-C pattern.
 Notes on implementing more complex patterns are in later sections.
 
-## Where you create a new Deferred, make it follow the rules
+## Where you create a new awaitable, make it follow the rules
 
-Most of the time, a Deferred comes from another synapse function.
-Sometimes, though, we need to make up a new Deferred, or we get a
-Deferred back from external code. We need to make it follow our rules.
+Most of the time, an awaitable comes from another synapse function.
+Sometimes, though, we need to make up a new awaitable, or we get an awaitable
+back from external code. We need to make it follow our rules.
 
-The easy way to do it is with a combination of `defer.inlineCallbacks`,
-and `context.PreserveLoggingContext`. Suppose we want to implement
+The easy way to do it is by using `context.make_deferred_yieldable`. Suppose we want to implement
 `sleep`, which returns a deferred which will run its callbacks after a
 given number of seconds. That might look like:
 
@@ -186,25 +173,12 @@ def get_sleep_deferred(seconds):
     return d
 ```
 
-That doesn't follow the rules, but we can fix it by wrapping it with
-`PreserveLoggingContext` and `yield` ing on it:
+That doesn't follow the rules, but we can fix it by calling it through
+`context.make_deferred_yieldable`:
 
 ```python
-@defer.inlineCallbacks
-def sleep(seconds):
-    with PreserveLoggingContext():
-        yield get_sleep_deferred(seconds)
-```
-
-This technique works equally for external functions which return
-deferreds, or deferreds we have made ourselves.
-
-You can also use `context.make_deferred_yieldable`, which just does the
-boilerplate for you, so the above could be written:
-
-```python
-def sleep(seconds):
-    return context.make_deferred_yieldable(get_sleep_deferred(seconds))
+async def sleep(seconds):
+    return await context.make_deferred_yieldable(get_sleep_deferred(seconds))
 ```
 
 ## Fire-and-forget
@@ -213,20 +187,18 @@ Sometimes you want to fire off a chain of execution, but not wait for
 its result. That might look a bit like this:
 
 ```python
-@defer.inlineCallbacks
-def do_request_handling():
-    yield foreground_operation()
+async def do_request_handling():
+    await foreground_operation()
 
     # *don't* do this
     background_operation()
 
     logger.debug("Request handling complete")
 
-@defer.inlineCallbacks
-def background_operation():
-    yield first_background_step()
+async def background_operation():
+    await first_background_step()
     logger.debug("Completed first step")
-    yield second_background_step()
+    await second_background_step()
     logger.debug("Completed second step")
 ```
 
@@ -235,13 +207,13 @@ The above code does a couple of steps in the background after
 against the `request_context` logcontext, which may or may not be
 desirable. There are two big problems with the above, however. The first
 problem is that, if `background_operation` returns an incomplete
-Deferred, it will expect its caller to `yield` immediately, so will have
+awaitable, it will expect its caller to `await` immediately, so will have
 cleared the logcontext. In this example, that means that 'Request
 handling complete' will be logged without any context.
 
 The second problem, which is potentially even worse, is that when the
-Deferred returned by `background_operation` completes, it will restore
-the original logcontext. There is nothing waiting on that Deferred, so
+awaitable returned by `background_operation` completes, it will restore
+the original logcontext. There is nothing waiting on that awaitable, so
 the logcontext will leak into the reactor and possibly get attached to
 some arbitrary future operation.
 
@@ -254,9 +226,8 @@ deferred completes will be the empty logcontext), and will restore the
 current logcontext before continuing the foreground process:
 
 ```python
-@defer.inlineCallbacks
-def do_request_handling():
-    yield foreground_operation()
+async def do_request_handling():
+    await foreground_operation()
 
     # start background_operation off in the empty logcontext, to
     # avoid leaking the current context into the reactor.
@@ -274,16 +245,15 @@ Obviously that option means that the operations done in
 
 The second option is to use `context.run_in_background`, which wraps a
 function so that it doesn't reset the logcontext even when it returns
-an incomplete deferred, and adds a callback to the returned deferred to
+an incomplete awaitable, and adds a callback to the returned awaitable to
 reset the logcontext. In other words, it turns a function that follows
-the Synapse rules about logcontexts and Deferreds into one which behaves
+the Synapse rules about logcontexts and awaitables into one which behaves
 more like an external function --- the opposite operation to that
 described in the previous section. It can be used like this:
 
 ```python
-@defer.inlineCallbacks
-def do_request_handling():
-    yield foreground_operation()
+async def do_request_handling():
+    await foreground_operation()
 
     context.run_in_background(background_operation)
 
@@ -294,152 +264,53 @@ def do_request_handling():
 ## Passing synapse deferreds into third-party functions
 
 A typical example of this is where we want to collect together two or
-more deferred via `defer.gatherResults`:
+more awaitables via `defer.gatherResults`:
 
 ```python
-d1 = operation1()
-d2 = operation2()
-d3 = defer.gatherResults([d1, d2])
+a1 = operation1()
+a2 = operation2()
+a3 = defer.gatherResults([a1, a2])
 ```
 
 This is really a variation of the fire-and-forget problem above, in that
-we are firing off `d1` and `d2` without yielding on them. The difference
+we are firing off `a1` and `a2` without awaiting on them. The difference
 is that we now have third-party code attached to their callbacks. Anyway
 either technique given in the [Fire-and-forget](#fire-and-forget)
 section will work.
 
-Of course, the new Deferred returned by `gatherResults` needs to be
+Of course, the new awaitable returned by `gather` needs to be
 wrapped in order to make it follow the logcontext rules before we can
-yield it, as described in [Where you create a new Deferred, make it
+yield it, as described in [Where you create a new awaitable, make it
 follow the
-rules](#where-you-create-a-new-deferred-make-it-follow-the-rules).
+rules](#where-you-create-a-new-awaitable-make-it-follow-the-rules).
 
 So, option one: reset the logcontext before starting the operations to
 be gathered:
 
 ```python
-@defer.inlineCallbacks
-def do_request_handling():
+async def do_request_handling():
     with PreserveLoggingContext():
-        d1 = operation1()
-        d2 = operation2()
-        result = yield defer.gatherResults([d1, d2])
+        a1 = operation1()
+        a2 = operation2()
+        result = await defer.gatherResults([a1, a2])
 ```
 
 In this case particularly, though, option two, of using
-`context.preserve_fn` almost certainly makes more sense, so that
+`context.run_in_background` almost certainly makes more sense, so that
 `operation1` and `operation2` are both logged against the original
 logcontext. This looks like:
 
 ```python
-@defer.inlineCallbacks
-def do_request_handling():
-    d1 = context.preserve_fn(operation1)()
-    d2 = context.preserve_fn(operation2)()
+async def do_request_handling():
+    a1 = context.run_in_background(operation1)
+    a2 = context.run_in_background(operation2)
 
-    with PreserveLoggingContext():
-        result = yield defer.gatherResults([d1, d2])
+    result = await make_deferred_yieldable(defer.gatherResults([a1, a2]))
 ```
 
-## Was all this really necessary?
-
-The conventions used work fine for a linear flow where everything
-happens in series via `defer.inlineCallbacks` and `yield`, but are
-certainly tricky to follow for any more exotic flows. It's hard not to
-wonder if we could have done something else.
-
-We're not going to rewrite Synapse now, so the following is entirely of
-academic interest, but I'd like to record some thoughts on an
-alternative approach.
-
-I briefly prototyped some code following an alternative set of rules. I
-think it would work, but I certainly didn't get as far as thinking how
-it would interact with concepts as complicated as the cache descriptors.
-
-My alternative rules were:
-
--   functions always preserve the logcontext of their caller, whether or
-    not they are returning a Deferred.
--   Deferreds returned by synapse functions run their callbacks in the
-    same context as the function was orignally called in.
-
-The main point of this scheme is that everywhere that sets the
-logcontext is responsible for clearing it before returning control to
-the reactor.
-
-So, for example, if you were the function which started a
-`with LoggingContext` block, you wouldn't `yield` within it --- instead
-you'd start off the background process, and then leave the `with` block
-to wait for it:
-
-```python
-def handle_request(request_id):
-    with context.LoggingContext() as request_context:
-        request_context.request = request_id
-        d = do_request_handling()
-
-    def cb(r):
-        logger.debug("finished")
-
-    d.addCallback(cb)
-    return d
-```
-
-(in general, mixing `with LoggingContext` blocks and
-`defer.inlineCallbacks` in the same function leads to slighly
-counter-intuitive code, under this scheme).
-
-Because we leave the original `with` block as soon as the Deferred is
-returned (as opposed to waiting for it to be resolved, as we do today),
-the logcontext is cleared before control passes back to the reactor; so
-if there is some code within `do_request_handling` which needs to wait
-for a Deferred to complete, there is no need for it to worry about
-clearing the logcontext before doing so:
-
-```python
-def handle_request():
-    r = do_some_stuff()
-    r.addCallback(do_some_more_stuff)
-    return r
-```
-
---- and provided `do_some_stuff` follows the rules of returning a
-Deferred which runs its callbacks in the original logcontext, all is
-happy.
-
-The business of a Deferred which runs its callbacks in the original
-logcontext isn't hard to achieve --- we have it today, in the shape of
-`context._PreservingContextDeferred`:
-
-```python
-def do_some_stuff():
-    deferred = do_some_io()
-    pcd = _PreservingContextDeferred(LoggingContext.current_context())
-    deferred.chainDeferred(pcd)
-    return pcd
-```
-
-It turns out that, thanks to the way that Deferreds chain together, we
-automatically get the property of a context-preserving deferred with
-`defer.inlineCallbacks`, provided the final Defered the function
-`yields` on has that property. So we can just write:
-
-```python
-@defer.inlineCallbacks
-def handle_request():
-    yield do_some_stuff()
-    yield do_some_more_stuff()
-```
-
-To conclude: I think this scheme would have worked equally well, with
-less danger of messing it up, and probably made some more esoteric code
-easier to write. But again --- changing the conventions of the entire
-Synapse codebase is not a sensible option for the marginal improvement
-offered.
-
-## A note on garbage-collection of Deferred chains
+## A note on garbage-collection of awaitable chains
 
-It turns out that our logcontext rules do not play nicely with Deferred
+It turns out that our logcontext rules do not play nicely with awaitable
 chains which get orphaned and garbage-collected.
 
 Imagine we have some code that looks like this:
@@ -451,13 +322,12 @@ def on_something_interesting():
     for d in listener_queue:
         d.callback("foo")
 
-@defer.inlineCallbacks
-def await_something_interesting():
-    new_deferred = defer.Deferred()
-    listener_queue.append(new_deferred)
+async def await_something_interesting():
+    new_awaitable = defer.Deferred()
+    listener_queue.append(new_awaitable)
 
     with PreserveLoggingContext():
-        yield new_deferred
+        await new_awaitable
 ```
 
 Obviously, the idea here is that we have a bunch of things which are
@@ -476,18 +346,19 @@ def reset_listener_queue():
     listener_queue.clear()
 ```
 
-So, both ends of the deferred chain have now dropped their references,
-and the deferred chain is now orphaned, and will be garbage-collected at
-some point. Note that `await_something_interesting` is a generator
-function, and when Python garbage-collects generator functions, it gives
-them a chance to clean up by making the `yield` raise a `GeneratorExit`
+So, both ends of the awaitable chain have now dropped their references,
+and the awaitable chain is now orphaned, and will be garbage-collected at
+some point. Note that `await_something_interesting` is a coroutine, 
+which Python implements as a generator function.  When Python
+garbage-collects generator functions, it gives them a chance to 
+clean up by making the `async` (or `yield`) raise a `GeneratorExit`
 exception. In our case, that means that the `__exit__` handler of
 `PreserveLoggingContext` will carefully restore the request context, but
 there is now nothing waiting for its return, so the request context is
 never cleared.
 
-To reiterate, this problem only arises when *both* ends of a deferred
-chain are dropped. Dropping the the reference to a deferred you're
-supposed to be calling is probably bad practice, so this doesn't
+To reiterate, this problem only arises when *both* ends of a awaitable
+chain are dropped. Dropping the the reference to an awaitable you're
+supposed to be awaiting is bad practice, so this doesn't
 actually happen too much. Unfortunately, when it does happen, it will
 lead to leaked logcontexts which are incredibly hard to track down.
diff --git a/docs/manhole.md b/docs/manhole.md
index 37d1d7823c..db92df88dc 100644
--- a/docs/manhole.md
+++ b/docs/manhole.md
@@ -67,7 +67,7 @@ This gives a Python REPL in which `hs` gives access to the
 `synapse.server.HomeServer` object - which in turn gives access to many other
 parts of the process.
 
-Note that any call which returns a coroutine will need to be wrapped in `ensureDeferred`.
+Note that, prior to Synapse 1.41, any call which returns a coroutine will need to be wrapped in `ensureDeferred`.
 
 As a simple example, retrieving an event from the database:
 
diff --git a/docs/message_retention_policies.md b/docs/message_retention_policies.md
index 75d2028e17..ea3d46cc10 100644
--- a/docs/message_retention_policies.md
+++ b/docs/message_retention_policies.md
@@ -51,7 +51,7 @@ clients.
 
 Support for this feature can be enabled and configured in the
 `retention` section of the Synapse configuration file (see the
-[sample file](https://github.com/matrix-org/synapse/blob/v1.7.3/docs/sample_config.yaml#L332-L393)).
+[sample file](https://github.com/matrix-org/synapse/blob/v1.36.0/docs/sample_config.yaml#L451-L518)).
 
 To enable support for message retention policies, set the setting
 `enabled` in this section to `true`.
@@ -87,7 +87,7 @@ expired events from the database. They are only run if support for
 message retention policies is enabled in the server's configuration. If
 no configuration for purge jobs is configured by the server admin,
 Synapse will use a default configuration, which is described in the
-[sample configuration file](https://github.com/matrix-org/synapse/blob/master/docs/sample_config.yaml#L332-L393).
+[sample configuration file](https://github.com/matrix-org/synapse/blob/v1.36.0/docs/sample_config.yaml#L451-L518).
 
 Some server admins might want a finer control on when events are removed
 depending on an event's room's policy. This can be done by setting the
diff --git a/docs/metrics-howto.md b/docs/metrics-howto.md
index 6b84153274..4a77d5604c 100644
--- a/docs/metrics-howto.md
+++ b/docs/metrics-howto.md
@@ -72,8 +72,7 @@
 
 ## Monitoring workers
 
-To monitor a Synapse installation using
-[workers](https://github.com/matrix-org/synapse/blob/master/docs/workers.md),
+To monitor a Synapse installation using [workers](workers.md),
 every worker needs to be monitored independently, in addition to
 the main homeserver process. This is because workers don't send
 their metrics to the main homeserver process, but expose them
diff --git a/docs/modules.md b/docs/modules.md
new file mode 100644
index 0000000000..9a430390a4
--- /dev/null
+++ b/docs/modules.md
@@ -0,0 +1,353 @@
+# Modules
+
+Synapse supports extending its functionality by configuring external modules.
+
+## Using modules
+
+To use a module on Synapse, add it to the `modules` section of the configuration file:
+
+```yaml
+modules:
+  - module: my_super_module.MySuperClass
+    config:
+      do_thing: true
+  - module: my_other_super_module.SomeClass
+    config: {}
+```
+
+Each module is defined by a path to a Python class as well as a configuration. This
+information for a given module should be available in the module's own documentation.
+
+**Note**: When using third-party modules, you effectively allow someone else to run
+custom code on your Synapse homeserver. Server admins are encouraged to verify the
+provenance of the modules they use on their homeserver and make sure the modules aren't
+running malicious code on their instance.
+
+Also note that we are currently in the process of migrating module interfaces to this
+system. While some interfaces might be compatible with it, others still require
+configuring modules in another part of Synapse's configuration file. Currently, only the
+spam checker interface is compatible with this new system.
+
+## Writing a module
+
+A module is a Python class that uses Synapse's module API to interact with the
+homeserver. It can register callbacks that Synapse will call on specific operations, as
+well as web resources to attach to Synapse's web server.
+
+When instantiated, a module is given its parsed configuration as well as an instance of
+the `synapse.module_api.ModuleApi` class. The configuration is a dictionary, and is
+either the output of the module's `parse_config` static method (see below), or the
+configuration associated with the module in Synapse's configuration file.
+
+See the documentation for the `ModuleApi` class
+[here](https://github.com/matrix-org/synapse/blob/master/synapse/module_api/__init__.py).
+
+### Handling the module's configuration
+
+A module can implement the following static method:
+
+```python
+@staticmethod
+def parse_config(config: dict) -> dict
+```
+
+This method is given a dictionary resulting from parsing the YAML configuration for the
+module. It may modify it (for example by parsing durations expressed as strings (e.g.
+"5d") into milliseconds, etc.), and return the modified dictionary. It may also verify
+that the configuration is correct, and raise an instance of
+`synapse.module_api.errors.ConfigError` if not.
+
+### Registering a web resource
+
+Modules can register web resources onto Synapse's web server using the following module
+API method:
+
+```python
+def ModuleApi.register_web_resource(path: str, resource: IResource) -> None
+```
+
+The path is the full absolute path to register the resource at. For example, if you
+register a resource for the path `/_synapse/client/my_super_module/say_hello`, Synapse
+will serve it at `http(s)://[HS_URL]/_synapse/client/my_super_module/say_hello`. Note
+that Synapse does not allow registering resources for several sub-paths in the `/_matrix`
+namespace (such as anything under `/_matrix/client` for example). It is strongly
+recommended that modules register their web resources under the `/_synapse/client`
+namespace.
+
+The provided resource is a Python class that implements Twisted's [IResource](https://twistedmatrix.com/documents/current/api/twisted.web.resource.IResource.html)
+interface (such as [Resource](https://twistedmatrix.com/documents/current/api/twisted.web.resource.Resource.html)).
+
+Only one resource can be registered for a given path. If several modules attempt to
+register a resource for the same path, the module that appears first in Synapse's
+configuration file takes priority.
+
+Modules **must** register their web resources in their `__init__` method.
+
+### Registering a callback
+
+Modules can use Synapse's module API to register callbacks. Callbacks are functions that
+Synapse will call when performing specific actions. Callbacks must be asynchronous, and
+are split in categories. A single module may implement callbacks from multiple categories,
+and is under no obligation to implement all callbacks from the categories it registers
+callbacks for.
+
+Modules can register callbacks using one of the module API's `register_[...]_callbacks`
+methods. The callback functions are passed to these methods as keyword arguments, with
+the callback name as the argument name and the function as its value. This is demonstrated
+in the example below. A `register_[...]_callbacks` method exists for each module type
+documented in this section.
+
+#### Spam checker callbacks
+
+Spam checker callbacks allow module developers to implement spam mitigation actions for
+Synapse instances. Spam checker callbacks can be registered using the module API's
+`register_spam_checker_callbacks` method.
+
+The available spam checker callbacks are:
+
+```python
+async def check_event_for_spam(event: "synapse.events.EventBase") -> Union[bool, str]
+```
+
+Called when receiving an event from a client or via federation. The module can return
+either a `bool` to indicate whether the event must be rejected because of spam, or a `str`
+to indicate the event must be rejected because of spam and to give a rejection reason to
+forward to clients.
+
+```python
+async def user_may_invite(inviter: str, invitee: str, room_id: str) -> bool
+```
+
+Called when processing an invitation. The module must return a `bool` indicating whether
+the inviter can invite the invitee to the given room. Both inviter and invitee are
+represented by their Matrix user ID (e.g. `@alice:example.com`).
+
+```python
+async def user_may_create_room(user: str) -> bool
+```
+
+Called when processing a room creation request. The module must return a `bool` indicating
+whether the given user (represented by their Matrix user ID) is allowed to create a room.
+
+```python
+async def user_may_create_room_alias(user: str, room_alias: "synapse.types.RoomAlias") -> bool
+```
+
+Called when trying to associate an alias with an existing room. The module must return a
+`bool` indicating whether the given user (represented by their Matrix user ID) is allowed
+to set the given alias.
+
+```python
+async def user_may_publish_room(user: str, room_id: str) -> bool
+```
+
+Called when trying to publish a room to the homeserver's public rooms directory. The
+module must return a `bool` indicating whether the given user (represented by their
+Matrix user ID) is allowed to publish the given room.
+
+```python
+async def check_username_for_spam(user_profile: Dict[str, str]) -> bool
+```
+
+Called when computing search results in the user directory. The module must return a
+`bool` indicating whether the given user profile can appear in search results. The profile
+is represented as a dictionary with the following keys:
+
+* `user_id`: The Matrix ID for this user.
+* `display_name`: The user's display name.
+* `avatar_url`: The `mxc://` URL to the user's avatar.
+
+The module is given a copy of the original dictionary, so modifying it from within the
+module cannot modify a user's profile when included in user directory search results.
+
+```python
+async def check_registration_for_spam(
+    email_threepid: Optional[dict],
+    username: Optional[str],
+    request_info: Collection[Tuple[str, str]],
+    auth_provider_id: Optional[str] = None,
+) -> "synapse.spam_checker_api.RegistrationBehaviour"
+```
+
+Called when registering a new user. The module must return a `RegistrationBehaviour`
+indicating whether the registration can go through or must be denied, or whether the user
+may be allowed to register but will be shadow banned.
+
+The arguments passed to this callback are:
+
+* `email_threepid`: The email address used for registering, if any.
+* `username`: The username the user would like to register. Can be `None`, meaning that
+  Synapse will generate one later.
+* `request_info`: A collection of tuples, which first item is a user agent, and which
+  second item is an IP address. These user agents and IP addresses are the ones that were
+  used during the registration process.
+* `auth_provider_id`: The identifier of the SSO authentication provider, if any.
+
+```python
+async def check_media_file_for_spam(
+    file_wrapper: "synapse.rest.media.v1.media_storage.ReadableFileWrapper",
+    file_info: "synapse.rest.media.v1._base.FileInfo",
+) -> bool
+```
+
+Called when storing a local or remote file. The module must return a boolean indicating
+whether the given file can be stored in the homeserver's media store.
+
+#### Account validity callbacks
+
+Account validity callbacks allow module developers to add extra steps to verify the
+validity on an account, i.e. see if a user can be granted access to their account on the
+Synapse instance. Account validity callbacks can be registered using the module API's
+`register_account_validity_callbacks` method.
+
+The available account validity callbacks are:
+
+```python
+async def is_user_expired(user: str) -> Optional[bool]
+```
+
+Called when processing any authenticated request (except for logout requests). The module
+can return a `bool` to indicate whether the user has expired and should be locked out of
+their account, or `None` if the module wasn't able to figure it out. The user is
+represented by their Matrix user ID (e.g. `@alice:example.com`).
+
+If the module returns `True`, the current request will be denied with the error code
+`ORG_MATRIX_EXPIRED_ACCOUNT` and the HTTP status code 403. Note that this doesn't
+invalidate the user's access token.
+
+```python
+async def on_user_registration(user: str) -> None
+```
+
+Called after successfully registering a user, in case the module needs to perform extra
+operations to keep track of them. (e.g. add them to a database table). The user is
+represented by their Matrix user ID.
+
+#### Third party rules callbacks
+
+Third party rules callbacks allow module developers to add extra checks to verify the
+validity of incoming events. Third party event rules callbacks can be registered using
+the module API's `register_third_party_rules_callbacks` method.
+
+The available third party rules callbacks are:
+
+```python
+async def check_event_allowed(
+    event: "synapse.events.EventBase",
+    state_events: "synapse.types.StateMap",
+) -> Tuple[bool, Optional[dict]]
+```
+
+**<span style="color:red">
+This callback is very experimental and can and will break without notice. Module developers
+are encouraged to implement `check_event_for_spam` from the spam checker category instead.
+</span>**
+
+Called when processing any incoming event, with the event and a `StateMap`
+representing the current state of the room the event is being sent into. A `StateMap` is
+a dictionary that maps tuples containing an event type and a state key to the
+corresponding state event. For example retrieving the room's `m.room.create` event from
+the `state_events` argument would look like this: `state_events.get(("m.room.create", ""))`.
+The module must return a boolean indicating whether the event can be allowed.
+
+Note that this callback function processes incoming events coming via federation
+traffic (on top of client traffic). This means denying an event might cause the local
+copy of the room's history to diverge from that of remote servers. This may cause
+federation issues in the room. It is strongly recommended to only deny events using this
+callback function if the sender is a local user, or in a private federation in which all
+servers are using the same module, with the same configuration.
+
+If the boolean returned by the module is `True`, it may also tell Synapse to replace the
+event with new data by returning the new event's data as a dictionary. In order to do
+that, it is recommended the module calls `event.get_dict()` to get the current event as a
+dictionary, and modify the returned dictionary accordingly.
+
+Note that replacing the event only works for events sent by local users, not for events
+received over federation.
+
+```python
+async def on_create_room(
+    requester: "synapse.types.Requester",
+    request_content: dict,
+    is_requester_admin: bool,
+) -> None
+```
+
+Called when processing a room creation request, with the `Requester` object for the user
+performing the request, a dictionary representing the room creation request's JSON body
+(see [the spec](https://matrix.org/docs/spec/client_server/latest#post-matrix-client-r0-createroom)
+for a list of possible parameters), and a boolean indicating whether the user performing
+the request is a server admin.
+
+Modules can modify the `request_content` (by e.g. adding events to its `initial_state`),
+or deny the room's creation by raising a `module_api.errors.SynapseError`.
+
+
+### Porting an existing module that uses the old interface
+
+In order to port a module that uses Synapse's old module interface, its author needs to:
+
+* ensure the module's callbacks are all asynchronous.
+* register their callbacks using one or more of the `register_[...]_callbacks` methods
+  from the `ModuleApi` class in the module's `__init__` method (see [this section](#registering-a-callback)
+  for more info).
+
+Additionally, if the module is packaged with an additional web resource, the module
+should register this resource in its `__init__` method using the `register_web_resource`
+method from the `ModuleApi` class (see [this section](#registering-a-web-resource) for
+more info).
+
+The module's author should also update any example in the module's configuration to only
+use the new `modules` section in Synapse's configuration file (see [this section](#using-modules)
+for more info).
+
+### Example
+
+The example below is a module that implements the spam checker callback
+`user_may_create_room` to deny room creation to user `@evilguy:example.com`, and registers
+a web resource to the path `/_synapse/client/demo/hello` that returns a JSON object.
+
+```python
+import json
+
+from twisted.web.resource import Resource
+from twisted.web.server import Request
+
+from synapse.module_api import ModuleApi
+
+
+class DemoResource(Resource):
+    def __init__(self, config):
+        super(DemoResource, self).__init__()
+        self.config = config
+
+    def render_GET(self, request: Request):
+        name = request.args.get(b"name")[0]
+        request.setHeader(b"Content-Type", b"application/json")
+        return json.dumps({"hello": name})
+
+
+class DemoModule:
+    def __init__(self, config: dict, api: ModuleApi):
+        self.config = config
+        self.api = api
+
+        self.api.register_web_resource(
+            path="/_synapse/client/demo/hello",
+            resource=DemoResource(self.config),
+        )
+
+        self.api.register_spam_checker_callbacks(
+            user_may_create_room=self.user_may_create_room,
+        )
+
+    @staticmethod
+    def parse_config(config):
+        return config
+
+    async def user_may_create_room(self, user: str) -> bool:
+        if user == "@evilguy:example.com":
+            return False
+
+        return True
+```
diff --git a/docs/openid.md b/docs/openid.md
index cfaafc5015..f685fd551a 100644
--- a/docs/openid.md
+++ b/docs/openid.md
@@ -410,7 +410,7 @@ oidc_providers:
         display_name_template: "{{ user.name }}"
 ```
 
-## Apple
+### Apple
 
 Configuring "Sign in with Apple" (SiWA) requires an Apple Developer account.
 
diff --git a/docs/opentracing.md b/docs/opentracing.md
index 4c7a56a5d7..f91362f112 100644
--- a/docs/opentracing.md
+++ b/docs/opentracing.md
@@ -42,17 +42,17 @@ To receive OpenTracing spans, start up a Jaeger server. This can be done
 using docker like so:
 
 ```sh
-docker run -d --name jaeger
+docker run -d --name jaeger \
   -p 6831:6831/udp \
   -p 6832:6832/udp \
   -p 5778:5778 \
   -p 16686:16686 \
   -p 14268:14268 \
-  jaegertracing/all-in-one:1.13
+  jaegertracing/all-in-one:1
 ```
 
 Latest documentation is probably at
-<https://www.jaegertracing.io/docs/1.13/getting-started/>
+https://www.jaegertracing.io/docs/latest/getting-started.
 
 ## Enable OpenTracing in Synapse
 
@@ -62,7 +62,7 @@ as shown in the [sample config](./sample_config.yaml). For example:
 
 ```yaml
 opentracing:
-  tracer_enabled: true
+  enabled: true
   homeserver_whitelist:
     - "mytrustedhomeserver.org"
     - "*.myotherhomeservers.com"
@@ -90,4 +90,4 @@ to two problems, namely:
 ## Configuring Jaeger
 
 Sampling strategies can be set as in this document:
-<https://www.jaegertracing.io/docs/1.13/sampling/>
+<https://www.jaegertracing.io/docs/latest/sampling/>.
diff --git a/docs/postgres.md b/docs/postgres.md
index 680685d04e..2c0a5b803a 100644
--- a/docs/postgres.md
+++ b/docs/postgres.md
@@ -1,6 +1,6 @@
 # Using Postgres
 
-Postgres version 9.5 or later is known to work.
+Synapse supports PostgreSQL versions 9.6 or later.
 
 ## Install postgres client libraries
 
@@ -8,14 +8,14 @@ Synapse will require the python postgres client library in order to
 connect to a postgres database.
 
 -   If you are using the [matrix.org debian/ubuntu
-    packages](../INSTALL.md#matrixorg-packages), the necessary python
+    packages](setup/installation.md#matrixorg-packages), the necessary python
     library will already be installed, but you will need to ensure the
     low-level postgres library is installed, which you can do with
     `apt install libpq5`.
 -   For other pre-built packages, please consult the documentation from
     the relevant package.
 -   If you installed synapse [in a
-    virtualenv](../INSTALL.md#installing-from-source), you can install
+    virtualenv](setup/installation.md#installing-from-source), you can install
     the library with:
 
         ~/synapse/env/bin/pip install "matrix-synapse[postgres]"
@@ -33,28 +33,15 @@ Assuming your PostgreSQL database user is called `postgres`, first authenticate
     # Or, if your system uses sudo to get administrative rights
     sudo -u postgres bash
 
-Then, create a user ``synapse_user`` with:
+Then, create a postgres user and a database with:
 
+    # this will prompt for a password for the new user
     createuser --pwprompt synapse_user
 
-Before you can authenticate with the `synapse_user`, you must create a
-database that it can access. To create a database, first connect to the
-database with your database user:
+    createdb --encoding=UTF8 --locale=C --template=template0 --owner=synapse_user synapse
 
-    su - postgres # Or: sudo -u postgres bash
-    psql
-
-and then run:
-
-    CREATE DATABASE synapse
-     ENCODING 'UTF8'
-     LC_COLLATE='C'
-     LC_CTYPE='C'
-     template=template0
-     OWNER synapse_user;
-
-This would create an appropriate database named `synapse` owned by the
-`synapse_user` user (which must already have been created as above).
+The above will create a user called `synapse_user`, and a database called
+`synapse`.
 
 Note that the PostgreSQL database *must* have the correct encoding set
 (as shown above), otherwise it will not be able to store UTF8 strings.
@@ -63,79 +50,6 @@ You may need to enable password authentication so `synapse_user` can
 connect to the database. See
 <https://www.postgresql.org/docs/current/auth-pg-hba-conf.html>.
 
-If you get an error along the lines of `FATAL:  Ident authentication failed for
-user "synapse_user"`, you may need to use an authentication method other than
-`ident`:
-
-* If the `synapse_user` user has a password, add the password to the `database:`
-  section of `homeserver.yaml`. Then add the following to `pg_hba.conf`:
-
-  ```
-  host    synapse     synapse_user    ::1/128     md5  # or `scram-sha-256` instead of `md5` if you use that
-  ```
-
-* If the `synapse_user` user does not have a password, then a password doesn't
-  have to be added to `homeserver.yaml`. But the following does need to be added
-  to `pg_hba.conf`:
-
-  ```
-  host    synapse     synapse_user    ::1/128     trust
-  ```
-
-Note that line order matters in `pg_hba.conf`, so make sure that if you do add a
-new line, it is inserted before:
-
-```
-host    all         all             ::1/128     ident
-```
-
-### Fixing incorrect `COLLATE` or `CTYPE`
-
-Synapse will refuse to set up a new database if it has the wrong values of
-`COLLATE` and `CTYPE` set, and will log warnings on existing databases. Using
-different locales can cause issues if the locale library is updated from
-underneath the database, or if a different version of the locale is used on any
-replicas.
-
-The safest way to fix the issue is to take a dump and recreate the database with
-the correct `COLLATE` and `CTYPE` parameters (as shown above). It is also possible to change the
-parameters on a live database and run a `REINDEX` on the entire database,
-however extreme care must be taken to avoid database corruption.
-
-Note that the above may fail with an error about duplicate rows if corruption
-has already occurred, and such duplicate rows will need to be manually removed.
-
-
-## Fixing inconsistent sequences error
-
-Synapse uses Postgres sequences to generate IDs for various tables. A sequence
-and associated table can get out of sync if, for example, Synapse has been
-downgraded and then upgraded again.
-
-To fix the issue shut down Synapse (including any and all workers) and run the
-SQL command included in the error message. Once done Synapse should start
-successfully.
-
-
-## Tuning Postgres
-
-The default settings should be fine for most deployments. For larger
-scale deployments tuning some of the settings is recommended, details of
-which can be found at
-<https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server>.
-
-In particular, we've found tuning the following values helpful for
-performance:
-
--   `shared_buffers`
--   `effective_cache_size`
--   `work_mem`
--   `maintenance_work_mem`
--   `autovacuum_work_mem`
-
-Note that the appropriate values for those fields depend on the amount
-of free memory the database host has available.
-
 ## Synapse config
 
 When you are ready to start using PostgreSQL, edit the `database`
@@ -165,18 +79,42 @@ may block for an extended period while it waits for a response from the
 database server. Example values might be:
 
 ```yaml
-# seconds of inactivity after which TCP should send a keepalive message to the server
-keepalives_idle: 10
+database:
+  args:
+    # ... as above
+
+    # seconds of inactivity after which TCP should send a keepalive message to the server
+    keepalives_idle: 10
 
-# the number of seconds after which a TCP keepalive message that is not
-# acknowledged by the server should be retransmitted
-keepalives_interval: 10
+    # the number of seconds after which a TCP keepalive message that is not
+    # acknowledged by the server should be retransmitted
+    keepalives_interval: 10
 
-# the number of TCP keepalives that can be lost before the client's connection
-# to the server is considered dead
-keepalives_count: 3
+    # the number of TCP keepalives that can be lost before the client's connection
+    # to the server is considered dead
+    keepalives_count: 3
 ```
 
+## Tuning Postgres
+
+The default settings should be fine for most deployments. For larger
+scale deployments tuning some of the settings is recommended, details of
+which can be found at
+<https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server>.
+
+In particular, we've found tuning the following values helpful for
+performance:
+
+-   `shared_buffers`
+-   `effective_cache_size`
+-   `work_mem`
+-   `maintenance_work_mem`
+-   `autovacuum_work_mem`
+
+Note that the appropriate values for those fields depend on the amount
+of free memory the database host has available.
+
+
 ## Porting from SQLite
 
 ### Overview
@@ -185,9 +123,8 @@ The script `synapse_port_db` allows porting an existing synapse server
 backed by SQLite to using PostgreSQL. This is done in as a two phase
 process:
 
-1.  Copy the existing SQLite database to a separate location (while the
-    server is down) and running the port script against that offline
-    database.
+1.  Copy the existing SQLite database to a separate location and run
+    the port script against that offline database.
 2.  Shut down the server. Rerun the port script to port any data that
     has come in since taking the first snapshot. Restart server against
     the PostgreSQL database.
@@ -245,3 +182,60 @@ PostgreSQL database configuration file `homeserver-postgres.yaml`:
     ./synctl start
 
 Synapse should now be running against PostgreSQL.
+
+
+## Troubleshooting
+
+### Alternative auth methods
+
+If you get an error along the lines of `FATAL:  Ident authentication failed for
+user "synapse_user"`, you may need to use an authentication method other than
+`ident`:
+
+* If the `synapse_user` user has a password, add the password to the `database:`
+  section of `homeserver.yaml`. Then add the following to `pg_hba.conf`:
+
+  ```
+  host    synapse     synapse_user    ::1/128     md5  # or `scram-sha-256` instead of `md5` if you use that
+  ```
+
+* If the `synapse_user` user does not have a password, then a password doesn't
+  have to be added to `homeserver.yaml`. But the following does need to be added
+  to `pg_hba.conf`:
+
+  ```
+  host    synapse     synapse_user    ::1/128     trust
+  ```
+
+Note that line order matters in `pg_hba.conf`, so make sure that if you do add a
+new line, it is inserted before:
+
+```
+host    all         all             ::1/128     ident
+```
+
+### Fixing incorrect `COLLATE` or `CTYPE`
+
+Synapse will refuse to set up a new database if it has the wrong values of
+`COLLATE` and `CTYPE` set, and will log warnings on existing databases. Using
+different locales can cause issues if the locale library is updated from
+underneath the database, or if a different version of the locale is used on any
+replicas.
+
+The safest way to fix the issue is to dump the database and recreate it with
+the correct locale parameter (as shown above). It is also possible to change the
+parameters on a live database and run a `REINDEX` on the entire database,
+however extreme care must be taken to avoid database corruption.
+
+Note that the above may fail with an error about duplicate rows if corruption
+has already occurred, and such duplicate rows will need to be manually removed.
+
+### Fixing inconsistent sequences error
+
+Synapse uses Postgres sequences to generate IDs for various tables. A sequence
+and associated table can get out of sync if, for example, Synapse has been
+downgraded and then upgraded again.
+
+To fix the issue shut down Synapse (including any and all workers) and run the
+SQL command included in the error message. Once done Synapse should start
+successfully.
diff --git a/docs/presence_router_module.md b/docs/presence_router_module.md
new file mode 100644
index 0000000000..4a3e720240
--- /dev/null
+++ b/docs/presence_router_module.md
@@ -0,0 +1,241 @@
+# Presence Router Module
+
+Synapse supports configuring a module that can specify additional users
+(local or remote) to should receive certain presence updates from local
+users.
+
+Note that routing presence via Application Service transactions is not
+currently supported.
+
+The presence routing module is implemented as a Python class, which will
+be imported by the running Synapse.
+
+## Python Presence Router Class
+
+The Python class is instantiated with two objects:
+
+* A configuration object of some type (see below).
+* An instance of `synapse.module_api.ModuleApi`.
+
+It then implements methods related to presence routing.
+
+Note that one method of `ModuleApi` that may be useful is:
+
+```python
+async def ModuleApi.send_local_online_presence_to(users: Iterable[str]) -> None
+```
+
+which can be given a list of local or remote MXIDs to broadcast known, online user
+presence to (for those users that the receiving user is considered interested in). 
+It does not include state for users who are currently offline, and it can only be
+called on workers that support sending federation. Additionally, this method must
+only be called from the process that has been configured to write to the
+the [presence stream](workers.md#stream-writers).
+By default, this is the main process, but another worker can be configured to do
+so.
+
+### Module structure
+
+Below is a list of possible methods that can be implemented, and whether they are
+required.
+
+#### `parse_config`
+
+```python
+def parse_config(config_dict: dict) -> Any
+```
+
+**Required.** A static method that is passed a dictionary of config options, and
+  should return a validated config object. This method is described further in
+  [Configuration](#configuration).
+
+#### `get_users_for_states`
+
+```python
+async def get_users_for_states(
+    self,
+    state_updates: Iterable[UserPresenceState],
+) -> Dict[str, Set[UserPresenceState]]:
+```
+
+**Required.** An asynchronous method that is passed an iterable of user presence
+state. This method can determine whether a given presence update should be sent to certain
+users. It does this by returning a dictionary with keys representing local or remote
+Matrix User IDs, and values being a python set
+of `synapse.handlers.presence.UserPresenceState` instances.
+
+Synapse will then attempt to send the specified presence updates to each user when
+possible.
+
+#### `get_interested_users`
+
+```python
+async def get_interested_users(self, user_id: str) -> Union[Set[str], str]
+```
+
+**Required.** An asynchronous method that is passed a single Matrix User ID. This
+method is expected to return the users that the passed in user may be interested in the
+presence of. Returned users may be local or remote. The presence routed as a result of
+what this method returns is sent in addition to the updates already sent between users
+that share a room together. Presence updates are deduplicated.
+
+This method should return a python set of Matrix User IDs, or the object
+`synapse.events.presence_router.PresenceRouter.ALL_USERS` to indicate that the passed
+user should receive presence information for *all* known users.
+
+For clarity, if the user `@alice:example.org` is passed to this method, and the Set
+`{"@bob:example.com", "@charlie:somewhere.org"}` is returned, this signifies that Alice
+should receive presence updates sent by Bob and Charlie, regardless of whether these
+users share a room.
+
+### Example
+
+Below is an example implementation of a presence router class.
+
+```python
+from typing import Dict, Iterable, Set, Union
+from synapse.events.presence_router import PresenceRouter
+from synapse.handlers.presence import UserPresenceState
+from synapse.module_api import ModuleApi
+
+class PresenceRouterConfig:
+    def __init__(self):
+        # Config options with their defaults
+        # A list of users to always send all user presence updates to
+        self.always_send_to_users = []  # type: List[str]
+        
+        # A list of users to ignore presence updates for. Does not affect
+        # shared-room presence relationships
+        self.blacklisted_users = []  # type: List[str]
+
+class ExamplePresenceRouter:
+    """An example implementation of synapse.presence_router.PresenceRouter.
+    Supports routing all presence to a configured set of users, or a subset
+    of presence from certain users to members of certain rooms.
+
+    Args:
+        config: A configuration object.
+        module_api: An instance of Synapse's ModuleApi.
+    """
+    def __init__(self, config: PresenceRouterConfig, module_api: ModuleApi):
+        self._config = config
+        self._module_api = module_api
+
+    @staticmethod
+    def parse_config(config_dict: dict) -> PresenceRouterConfig:
+        """Parse a configuration dictionary from the homeserver config, do
+        some validation and return a typed PresenceRouterConfig.
+
+        Args:
+            config_dict: The configuration dictionary.
+
+        Returns:
+            A validated config object.
+        """
+        # Initialise a typed config object
+        config = PresenceRouterConfig()
+        always_send_to_users = config_dict.get("always_send_to_users")
+        blacklisted_users = config_dict.get("blacklisted_users")
+
+        # Do some validation of config options... otherwise raise a
+        # synapse.config.ConfigError.
+        config.always_send_to_users = always_send_to_users
+        config.blacklisted_users = blacklisted_users
+
+        return config
+
+    async def get_users_for_states(
+        self,
+        state_updates: Iterable[UserPresenceState],
+    ) -> Dict[str, Set[UserPresenceState]]:
+        """Given an iterable of user presence updates, determine where each one
+        needs to go. Returned results will not affect presence updates that are
+        sent between users who share a room.
+
+        Args:
+            state_updates: An iterable of user presence state updates.
+
+        Returns:
+          A dictionary of user_id -> set of UserPresenceState that the user should 
+          receive.
+        """
+        destination_users = {}  # type: Dict[str, Set[UserPresenceState]
+
+        # Ignore any updates for blacklisted users
+        desired_updates = set()
+        for update in state_updates:
+            if update.state_key not in self._config.blacklisted_users:
+                desired_updates.add(update)
+
+        # Send all presence updates to specific users
+        for user_id in self._config.always_send_to_users:
+            destination_users[user_id] = desired_updates
+
+        return destination_users
+
+    async def get_interested_users(
+        self,
+        user_id: str,
+    ) -> Union[Set[str], PresenceRouter.ALL_USERS]:
+        """
+        Retrieve a list of users that `user_id` is interested in receiving the
+        presence of. This will be in addition to those they share a room with.
+        Optionally, the object PresenceRouter.ALL_USERS can be returned to indicate
+        that this user should receive all incoming local and remote presence updates.
+
+        Note that this method will only be called for local users.
+
+        Args:
+          user_id: A user requesting presence updates.
+
+        Returns:
+          A set of user IDs to return additional presence updates for, or
+          PresenceRouter.ALL_USERS to return presence updates for all other users.
+        """
+        if user_id in self._config.always_send_to_users:
+            return PresenceRouter.ALL_USERS
+
+        return set()
+```
+
+#### A note on `get_users_for_states` and `get_interested_users`
+
+Both of these methods are effectively two different sides of the same coin. The logic
+regarding which users should receive updates for other users should be the same 
+between them.
+
+`get_users_for_states` is called when presence updates come in from either federation 
+or local users, and is used to either direct local presence to remote users, or to
+wake up the sync streams of local users to collect remote presence.
+
+In contrast, `get_interested_users` is used to determine the users that presence should
+be fetched for when a local user is syncing. This presence is then retrieved, before
+being fed through `get_users_for_states` once again, with only the syncing user's
+routing information pulled from the resulting dictionary.
+
+Their routing logic should thus line up, else you may run into unintended behaviour.
+
+## Configuration
+
+Once you've crafted your module and installed it into the same Python environment as
+Synapse, amend your homeserver config file with the following.
+
+```yaml
+presence:
+  enabled: true
+
+  presence_router:
+    module: my_module.ExamplePresenceRouter
+    config:
+      # Any configuration options for your module. The below is an example.
+      # of setting options for ExamplePresenceRouter.
+      always_send_to_users: ["@presence_gobbler:example.org"]
+      blacklisted_users:
+        - "@alice:example.com"
+        - "@bob:example.com"
+      ...
+```
+
+The contents of `config` will be passed as a Python dictionary to the static
+`parse_config` method of your class. The object returned by this method will
+then be passed to the `__init__` method of your module as `config`.
diff --git a/docs/replication.md b/docs/replication.md
index ed88233157..e82df0de8a 100644
--- a/docs/replication.md
+++ b/docs/replication.md
@@ -28,7 +28,7 @@ minimal.
 
 ### The Replication Protocol
 
-See [tcp_replication.md](tcp_replication.md)
+See [the TCP replication documentation](tcp_replication.md).
 
 ### The Slaved DataStore
 
diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md
index cf1b835b9d..5f8d20129e 100644
--- a/docs/reverse_proxy.md
+++ b/docs/reverse_proxy.md
@@ -21,7 +21,7 @@ port 8448. Where these are different, we refer to the 'client port' and the
 'federation port'. See [the Matrix
 specification](https://matrix.org/docs/spec/server_server/latest#resolving-server-names)
 for more details of the algorithm used for federation connections, and
-[delegate.md](<delegate.md>) for instructions on setting up delegation.
+[Delegation](delegate.md) for instructions on setting up delegation.
 
 **NOTE**: Your reverse proxy must not `canonicalise` or `normalise`
 the requested URI in any way (for example, by decoding `%xx` escapes).
@@ -33,6 +33,19 @@ Let's assume that we expect clients to connect to our server at
 `https://example.com:8448`.  The following sections detail the configuration of
 the reverse proxy and the homeserver.
 
+
+## Homeserver Configuration
+
+The HTTP configuration will need to be updated for Synapse to correctly record 
+client IP addresses and generate redirect URLs while behind a reverse proxy. 
+
+In `homeserver.yaml` set `x_forwarded: true` in the port 8008 section and 
+consider setting `bind_addresses: ['127.0.0.1']` so that the server only
+listens to traffic on localhost. (Do not change `bind_addresses` to `127.0.0.1` 
+when using a containerized Synapse, as that will prevent it from responding
+to proxied traffic.)
+
+
 ## Reverse-proxy configuration examples
 
 **NOTE**: You only need one of these.
@@ -98,6 +111,33 @@ example.com:8448 {
   reverse_proxy http://localhost:8008
 }
 ```
+[Delegation](delegate.md) example:
+```
+(matrix-well-known-header) {
+    # Headers
+    header Access-Control-Allow-Origin "*"
+    header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
+    header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization"
+    header Content-Type "application/json"
+}
+
+example.com {
+    handle /.well-known/matrix/server {
+        import matrix-well-known-header
+        respond `{"m.server":"matrix.example.com:443"}`
+    }
+
+    handle /.well-known/matrix/client {
+        import matrix-well-known-header
+        respond `{"m.homeserver":{"base_url":"https://matrix.example.com"},"m.identity_server":{"base_url":"https://identity.example.com"}}`
+    }
+}
+
+matrix.example.com {
+    reverse_proxy /_matrix/* http://localhost:8008
+    reverse_proxy /_synapse/client/* http://localhost:8008
+}
+```
 
 ### Apache
 
@@ -212,16 +252,6 @@ relay "matrix_federation" {
 }
 ```
 
-## Homeserver Configuration
-
-You will also want to set `bind_addresses: ['127.0.0.1']` and
-`x_forwarded: true` for port 8008 in `homeserver.yaml` to ensure that
-client IP addresses are recorded correctly.
-
-Having done so, you can then use `https://matrix.example.com` (instead
-of `https://matrix.example.com:8448`) as the "Custom server" when
-connecting to Synapse from a client.
-
 
 ## Health check endpoint
 
diff --git a/docs/room_and_user_statistics.md b/docs/room_and_user_statistics.md
index e1facb38d4..cc38c890bb 100644
--- a/docs/room_and_user_statistics.md
+++ b/docs/room_and_user_statistics.md
@@ -1,9 +1,9 @@
 Room and User Statistics
 ========================
 
-Synapse maintains room and user statistics (as well as a cache of room state),
-in various tables. These can be used for administrative purposes but are also
-used when generating the public room directory.
+Synapse maintains room and user statistics in various tables. These can be used
+for administrative purposes but are also used when generating the public room
+directory.
 
 
 # Synapse Developer Documentation
@@ -15,48 +15,8 @@ used when generating the public room directory.
 * **subject**: Something we are tracking stats about – currently a room or user.
 * **current row**: An entry for a subject in the appropriate current statistics
     table. Each subject can have only one.
-* **historical row**: An entry for a subject in the appropriate historical
-    statistics table. Each subject can have any number of these.
 
 ### Overview
 
-Stats are maintained as time series. There are two kinds of column:
-
-* absolute columns – where the value is correct for the time given by `end_ts`
-    in the stats row. (Imagine a line graph for these values)
-    * They can also be thought of as 'gauges' in Prometheus, if you are familiar.
-* per-slice columns – where the value corresponds to how many of the occurrences
-    occurred within the time slice given by `(end_ts − bucket_size)…end_ts`
-    or `start_ts…end_ts`. (Imagine a histogram for these values)
-
-Stats are maintained in two tables (for each type): current and historical.
-
-Current stats correspond to the present values. Each subject can only have one
-entry.
-
-Historical stats correspond to values in the past. Subjects may have multiple
-entries.
-
-## Concepts around the management of stats
-
-### Current rows
-
-Current rows contain the most up-to-date statistics for a room.
-They only contain absolute columns
-
-### Historical rows
-
-Historical rows can always be considered to be valid for the time slice and
-end time specified.
-
-* historical rows will not exist for every time slice – they will be omitted
-    if there were no changes. In this case, the following assumptions can be
-    made to interpolate/recreate missing rows:
-    - absolute fields have the same values as in the preceding row
-    - per-slice fields are zero (`0`)
-* historical rows will not be retained forever – rows older than a configurable
-    time will be purged.
-
-#### Purge
-
-The purging of historical rows is not yet implemented.
+Stats correspond to the present values. Current rows contain the most up-to-date
+statistics for a room. Each subject can only have one entry.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index be5e84f0ad..1c72de536a 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -8,7 +8,8 @@
 #
 # It is *not* intended to be copied and used as the basis for a real
 # homeserver.yaml. Instead, if you are starting from scratch, please generate
-# a fresh config using Synapse by following the instructions in INSTALL.md.
+# a fresh config using Synapse by following the instructions in
+# https://matrix-org.github.io/synapse/latest/setup/installation.html.
 
 # Configuration options that take a time period can be set using a number
 # followed by a letter. Letters have the following meanings:
@@ -31,6 +32,22 @@
 #
 # [1] https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html
 
+
+## Modules ##
+
+# Server admins can expand Synapse's functionality with external modules.
+#
+# See https://matrix-org.github.io/synapse/latest/modules.html for more
+# documentation on how to configure or create custom modules for Synapse.
+#
+modules:
+    # - module: my_super_module.MySuperClass
+    #   config:
+    #       do_thing: true
+    # - module: my_other_super_module.SomeClass
+    #   config: {}
+
+
 ## Server ##
 
 # The public-facing domain of the server
@@ -42,7 +59,7 @@
 # In most cases you should avoid using a matrix specific subdomain such as
 # matrix.example.com or synapse.example.com as the server_name for the same
 # reasons you wouldn't use user@email.example.com as your email address.
-# See https://github.com/matrix-org/synapse/blob/master/docs/delegate.md
+# See https://matrix-org.github.io/synapse/latest/delegate.html
 # for information on how to host Synapse on a subdomain while preserving
 # a clean server_name.
 #
@@ -82,9 +99,28 @@ pid_file: DATADIR/homeserver.pid
 #
 #soft_file_limit: 0
 
-# Set to false to disable presence tracking on this homeserver.
+# Presence tracking allows users to see the state (e.g online/offline)
+# of other local and remote users.
 #
-#use_presence: false
+presence:
+  # Uncomment to disable presence tracking on this homeserver. This option
+  # replaces the previous top-level 'use_presence' option.
+  #
+  #enabled: false
+
+  # Presence routers are third-party modules that can specify additional logic
+  # to where presence updates from users are routed.
+  #
+  presence_router:
+    # The custom module's class. Uncomment to use a custom presence router module.
+    #
+    #module: "my_custom_router.PresenceRouter"
+
+    # Configuration options of the custom module. Refer to your module's
+    # documentation for available options.
+    #
+    #config:
+    #  example_option: 'something'
 
 # Whether to require authentication to retrieve profile data (avatars,
 # display names) of other users through the client API. Defaults to
@@ -133,6 +169,16 @@ pid_file: DATADIR/homeserver.pid
 #
 #gc_thresholds: [700, 10, 10]
 
+# The minimum time in seconds between each GC for a generation, regardless of
+# the GC thresholds. This ensures that we don't do GC too frequently.
+#
+# A value of `[1s, 10s, 30s]` indicates that a second must pass between consecutive
+# generation 0 GCs, etc.
+#
+# Defaults to `[1s, 10s, 30s]`.
+#
+#gc_min_interval: [0.5s, 30s, 1m]
+
 # Set the limit on the returned events in the timeline in the get
 # and sync operations. The default value is 100. -1 means no upper limit.
 #
@@ -164,6 +210,8 @@ pid_file: DATADIR/homeserver.pid
 #
 # This option replaces federation_ip_range_blacklist in Synapse v1.25.0.
 #
+# Note: The value is ignored when an HTTP proxy is in use
+#
 #ip_range_blacklist:
 #  - '127.0.0.0/8'
 #  - '10.0.0.0/8'
@@ -208,9 +256,9 @@ pid_file: DATADIR/homeserver.pid
 #       'all local interfaces'.
 #
 #   type: the type of listener. Normally 'http', but other valid options are:
-#       'manhole' (see docs/manhole.md),
-#       'metrics' (see docs/metrics-howto.md),
-#       'replication' (see docs/workers.md).
+#       'manhole' (see https://matrix-org.github.io/synapse/latest/manhole.html),
+#       'metrics' (see https://matrix-org.github.io/synapse/latest/metrics-howto.html),
+#       'replication' (see https://matrix-org.github.io/synapse/latest/workers.html).
 #
 #   tls: set to true to enable TLS for this listener. Will use the TLS
 #       key/cert specified in tls_private_key_path / tls_certificate_path.
@@ -235,8 +283,8 @@ pid_file: DATADIR/homeserver.pid
 #   client: the client-server API (/_matrix/client), and the synapse admin
 #       API (/_synapse/admin). Also implies 'media' and 'static'.
 #
-#   consent: user consent forms (/_matrix/consent). See
-#       docs/consent_tracking.md.
+#   consent: user consent forms (/_matrix/consent).
+#       See https://matrix-org.github.io/synapse/latest/consent_tracking.html.
 #
 #   federation: the server-server API (/_matrix/federation). Also implies
 #       'media', 'keys', 'openid'
@@ -245,12 +293,13 @@ pid_file: DATADIR/homeserver.pid
 #
 #   media: the media API (/_matrix/media).
 #
-#   metrics: the metrics interface. See docs/metrics-howto.md.
+#   metrics: the metrics interface.
+#       See https://matrix-org.github.io/synapse/latest/metrics-howto.html.
 #
 #   openid: OpenID authentication.
 #
-#   replication: the HTTP replication API (/_synapse/replication). See
-#       docs/workers.md.
+#   replication: the HTTP replication API (/_synapse/replication).
+#       See https://matrix-org.github.io/synapse/latest/workers.html.
 #
 #   static: static resources under synapse/static (/_matrix/static). (Mostly
 #       useful for 'fallback authentication'.)
@@ -274,7 +323,7 @@ listeners:
   # that unwraps TLS.
   #
   # If you plan to use a reverse proxy, please see
-  # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md.
+  # https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
   #
   - port: 8008
     tls: false
@@ -584,6 +633,19 @@ retention:
 #
 #next_link_domain_whitelist: ["matrix.org"]
 
+# Templates to use when generating email or HTML page contents.
+#
+templates:
+  # Directory in which Synapse will try to find template files to use to generate
+  # email or HTML page contents.
+  # If not set, or a file is not found within the template directory, a default
+  # template from within the Synapse package will be used.
+  #
+  # See https://matrix-org.github.io/synapse/latest/templates.html for more
+  # information about using custom templates.
+  #
+  #custom_template_directory: /path/to/custom/templates/
+
 
 ## TLS ##
 
@@ -591,13 +653,9 @@ retention:
 # This certificate, as of Synapse 1.0, will need to be a valid and verifiable
 # certificate, signed by a recognised Certificate Authority.
 #
-# See 'ACME support' below to enable auto-provisioning this certificate via
-# Let's Encrypt.
-#
-# If supplying your own, be sure to use a `.pem` file that includes the
-# full certificate chain including any intermediate certificates (for
-# instance, if using certbot, use `fullchain.pem` as your certificate,
-# not `cert.pem`).
+# Be sure to use a `.pem` file that includes the full certificate chain including
+# any intermediate certificates (for instance, if using certbot, use
+# `fullchain.pem` as your certificate, not `cert.pem`).
 #
 #tls_certificate_path: "CONFDIR/SERVERNAME.tls.crt"
 
@@ -648,107 +706,6 @@ retention:
 #  - myCA2.pem
 #  - myCA3.pem
 
-# ACME support: This will configure Synapse to request a valid TLS certificate
-# for your configured `server_name` via Let's Encrypt.
-#
-# Note that ACME v1 is now deprecated, and Synapse currently doesn't support
-# ACME v2. This means that this feature currently won't work with installs set
-# up after November 2019. For more info, and alternative solutions, see
-# https://github.com/matrix-org/synapse/blob/master/docs/ACME.md#deprecation-of-acme-v1
-#
-# Note that provisioning a certificate in this way requires port 80 to be
-# routed to Synapse so that it can complete the http-01 ACME challenge.
-# By default, if you enable ACME support, Synapse will attempt to listen on
-# port 80 for incoming http-01 challenges - however, this will likely fail
-# with 'Permission denied' or a similar error.
-#
-# There are a couple of potential solutions to this:
-#
-#  * If you already have an Apache, Nginx, or similar listening on port 80,
-#    you can configure Synapse to use an alternate port, and have your web
-#    server forward the requests. For example, assuming you set 'port: 8009'
-#    below, on Apache, you would write:
-#
-#    ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge
-#
-#  * Alternatively, you can use something like `authbind` to give Synapse
-#    permission to listen on port 80.
-#
-acme:
-    # ACME support is disabled by default. Set this to `true` and uncomment
-    # tls_certificate_path and tls_private_key_path above to enable it.
-    #
-    enabled: false
-
-    # Endpoint to use to request certificates. If you only want to test,
-    # use Let's Encrypt's staging url:
-    #     https://acme-staging.api.letsencrypt.org/directory
-    #
-    #url: https://acme-v01.api.letsencrypt.org/directory
-
-    # Port number to listen on for the HTTP-01 challenge. Change this if
-    # you are forwarding connections through Apache/Nginx/etc.
-    #
-    port: 80
-
-    # Local addresses to listen on for incoming connections.
-    # Again, you may want to change this if you are forwarding connections
-    # through Apache/Nginx/etc.
-    #
-    bind_addresses: ['::', '0.0.0.0']
-
-    # How many days remaining on a certificate before it is renewed.
-    #
-    reprovision_threshold: 30
-
-    # The domain that the certificate should be for. Normally this
-    # should be the same as your Matrix domain (i.e., 'server_name'), but,
-    # by putting a file at 'https://<server_name>/.well-known/matrix/server',
-    # you can delegate incoming traffic to another server. If you do that,
-    # you should give the target of the delegation here.
-    #
-    # For example: if your 'server_name' is 'example.com', but
-    # 'https://example.com/.well-known/matrix/server' delegates to
-    # 'matrix.example.com', you should put 'matrix.example.com' here.
-    #
-    # If not set, defaults to your 'server_name'.
-    #
-    domain: matrix.example.com
-
-    # file to use for the account key. This will be generated if it doesn't
-    # exist.
-    #
-    # If unspecified, we will use CONFDIR/client.key.
-    #
-    account_key_file: DATADIR/acme_account.key
-
-# List of allowed TLS fingerprints for this server to publish along
-# with the signing keys for this server. Other matrix servers that
-# make HTTPS requests to this server will check that the TLS
-# certificates returned by this server match one of the fingerprints.
-#
-# Synapse automatically adds the fingerprint of its own certificate
-# to the list. So if federation traffic is handled directly by synapse
-# then no modification to the list is required.
-#
-# If synapse is run behind a load balancer that handles the TLS then it
-# will be necessary to add the fingerprints of the certificates used by
-# the loadbalancers to this list if they are different to the one
-# synapse is using.
-#
-# Homeservers are permitted to cache the list of TLS fingerprints
-# returned in the key responses up to the "valid_until_ts" returned in
-# key. It may be necessary to publish the fingerprints of a new
-# certificate and wait until the "valid_until_ts" of the previous key
-# responses have passed before deploying it.
-#
-# You can calculate a fingerprint from a given TLS listener via:
-# openssl s_client -connect $host:$port < /dev/null 2> /dev/null |
-#   openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '='
-# or by checking matrix.org/federationtester/api/report?server_name=$host
-#
-#tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
-
 
 ## Federation ##
 
@@ -780,6 +737,12 @@ acme:
 #
 #allow_profile_lookup_over_federation: false
 
+# Uncomment to disable device display name lookup over federation. By default, the
+# Federation API allows other homeservers to obtain device display names of any user
+# on this homeserver. Defaults to 'true'.
+#
+#allow_device_name_lookup_over_federation: false
+
 
 ## Caching ##
 
@@ -795,35 +758,50 @@ acme:
 #event_cache_size: 10K
 
 caches:
-   # Controls the global cache factor, which is the default cache factor
-   # for all caches if a specific factor for that cache is not otherwise
-   # set.
-   #
-   # This can also be set by the "SYNAPSE_CACHE_FACTOR" environment
-   # variable. Setting by environment variable takes priority over
-   # setting through the config file.
-   #
-   # Defaults to 0.5, which will half the size of all caches.
-   #
-   #global_factor: 1.0
+  # Controls the global cache factor, which is the default cache factor
+  # for all caches if a specific factor for that cache is not otherwise
+  # set.
+  #
+  # This can also be set by the "SYNAPSE_CACHE_FACTOR" environment
+  # variable. Setting by environment variable takes priority over
+  # setting through the config file.
+  #
+  # Defaults to 0.5, which will half the size of all caches.
+  #
+  #global_factor: 1.0
 
-   # A dictionary of cache name to cache factor for that individual
-   # cache. Overrides the global cache factor for a given cache.
-   #
-   # These can also be set through environment variables comprised
-   # of "SYNAPSE_CACHE_FACTOR_" + the name of the cache in capital
-   # letters and underscores. Setting by environment variable
-   # takes priority over setting through the config file.
-   # Ex. SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0
-   #
-   # Some caches have '*' and other characters that are not
-   # alphanumeric or underscores. These caches can be named with or
-   # without the special characters stripped. For example, to specify
-   # the cache factor for `*stateGroupCache*` via an environment
-   # variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`.
-   #
-   per_cache_factors:
-     #get_users_who_share_room_with_user: 2.0
+  # A dictionary of cache name to cache factor for that individual
+  # cache. Overrides the global cache factor for a given cache.
+  #
+  # These can also be set through environment variables comprised
+  # of "SYNAPSE_CACHE_FACTOR_" + the name of the cache in capital
+  # letters and underscores. Setting by environment variable
+  # takes priority over setting through the config file.
+  # Ex. SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0
+  #
+  # Some caches have '*' and other characters that are not
+  # alphanumeric or underscores. These caches can be named with or
+  # without the special characters stripped. For example, to specify
+  # the cache factor for `*stateGroupCache*` via an environment
+  # variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`.
+  #
+  per_cache_factors:
+    #get_users_who_share_room_with_user: 2.0
+
+  # Controls how long an entry can be in a cache without having been
+  # accessed before being evicted. Defaults to None, which means
+  # entries are never evicted based on time.
+  #
+  #expiry_time: 30m
+
+  # Controls how long the results of a /sync request are cached for after
+  # a successful response is returned. A higher duration can help clients with
+  # intermittent connections, at the cost of higher memory usage.
+  #
+  # By default, this is zero, which means that sync responses are not cached
+  # at all.
+  #
+  #sync_response_cache_duration: 2m
 
 
 ## Database ##
@@ -834,6 +812,9 @@ caches:
 # 'name' gives the database engine to use: either 'sqlite3' (for SQLite) or
 # 'psycopg2' (for PostgreSQL).
 #
+# 'txn_limit' gives the maximum number of transactions to run per connection
+# before reconnecting. Defaults to 0, which means no limit.
+#
 # 'args' gives options which are passed through to the database engine,
 # except for options starting 'cp_', which are used to configure the Twisted
 # connection pool. For a reference to valid arguments, see:
@@ -854,15 +835,18 @@ caches:
 #
 #database:
 #  name: psycopg2
+#  txn_limit: 10000
 #  args:
 #    user: synapse_user
 #    password: secretpassword
 #    database: synapse
 #    host: localhost
+#    port: 5432
 #    cp_min: 5
 #    cp_max: 10
 #
-# For more information on using Synapse with Postgres, see `docs/postgres.md`.
+# For more information on using Synapse with Postgres,
+# see https://matrix-org.github.io/synapse/latest/postgres.html.
 #
 database:
   name: sqlite3
@@ -1019,6 +1003,10 @@ media_store_path: "DATADIR/media_store"
 
 # The largest allowed upload size in bytes
 #
+# If you are using a reverse proxy you may also need to set this value in
+# your reverse proxy's config. Notably Nginx has a small max body size by default.
+# See https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
+#
 #max_upload_size: 50M
 
 # The largest allowed size for a user avatar. If not defined, no
@@ -1097,6 +1085,8 @@ media_store_path: "DATADIR/media_store"
 # This must be specified if url_preview_enabled is set. It is recommended that
 # you uncomment the following list as a starting point.
 #
+# Note: The value is ignored when an HTTP proxy is in use
+#
 #url_preview_ip_range_blacklist:
 #  - '127.0.0.0/8'
 #  - '10.0.0.0/8'
@@ -1304,9 +1294,9 @@ url_preview_accept_language:
 #
 #allowed_local_3pids:
 #  - medium: email
-#    pattern: '.*@matrix\.org'
+#    pattern: '^[^@]+@matrix\.org$'
 #  - medium: email
-#    pattern: '.*@vector\.im'
+#    pattern: '^[^@]+@vector\.im$'
 #  - medium: msisdn
 #    pattern: '\+44'
 
@@ -1525,72 +1515,6 @@ account_threepid_delegates:
 #bind_new_user_emails_to_sydent: https://example.com:8091
 
 
-## Account Validity ##
-#
-# Optional account validity configuration. This allows for accounts to be denied
-# any request after a given period.
-#
-# Once this feature is enabled, Synapse will look for registered users without an
-# expiration date at startup and will add one to every account it found using the
-# current settings at that time.
-# This means that, if a validity period is set, and Synapse is restarted (it will
-# then derive an expiration date from the current validity period), and some time
-# after that the validity period changes and Synapse is restarted, the users'
-# expiration dates won't be updated unless their account is manually renewed. This
-# date will be randomly selected within a range [now + period - d ; now + period],
-# where d is equal to 10% of the validity period.
-#
-account_validity:
-  # The account validity feature is disabled by default. Uncomment the
-  # following line to enable it.
-  #
-  #enabled: true
-
-  # The period after which an account is valid after its registration. When
-  # renewing the account, its validity period will be extended by this amount
-  # of time. This parameter is required when using the account validity
-  # feature.
-  #
-  #period: 6w
-
-  # The amount of time before an account's expiry date at which Synapse will
-  # send an email to the account's email address with a renewal link. By
-  # default, no such emails are sent.
-  #
-  # If you enable this setting, you will also need to fill out the 'email' and
-  # 'public_baseurl' configuration sections.
-  #
-  #renew_at: 1w
-
-  # The subject of the email sent out with the renewal link. '%(app)s' can be
-  # used as a placeholder for the 'app_name' parameter from the 'email'
-  # section.
-  #
-  # Note that the placeholder must be written '%(app)s', including the
-  # trailing 's'.
-  #
-  # If this is not set, a default value is used.
-  #
-  #renew_email_subject: "Renew your %(app)s account"
-
-  # Directory in which Synapse will try to find templates for the HTML files to
-  # serve to the user when trying to renew an account. If not set, default
-  # templates from within the Synapse package will be used.
-  #
-  #template_dir: "res/templates"
-
-  # File within 'template_dir' giving the HTML to be displayed to the user after
-  # they successfully renewed their account. If not set, default text is used.
-  #
-  #account_renewed_html_path: "account_renewed.html"
-
-  # File within 'template_dir' giving the HTML to be displayed when the user
-  # tries to renew an account with an invalid renewal token. If not set,
-  # default text is used.
-  #
-  #invalid_token_html_path: "invalid_token.html"
-
-
 ## Metrics ###
 
 # Enable collection and rendering of performance metrics
@@ -1629,16 +1553,32 @@ metrics_flags:
 
 ## API Configuration ##
 
-# A list of event types from a room that will be given to users when they
-# are invited to a room. This allows clients to display information about the
-# room that they've been invited to, without actually being in the room yet.
+# Controls for the state that is shared with users who receive an invite
+# to a room
 #
-#room_invite_state_types:
-#  - "m.room.join_rules"
-#  - "m.room.canonical_alias"
-#  - "m.room.avatar"
-#  - "m.room.encryption"
-#  - "m.room.name"
+room_prejoin_state:
+   # By default, the following state event types are shared with users who
+   # receive invites to the room:
+   #
+   # - m.room.join_rules
+   # - m.room.canonical_alias
+   # - m.room.avatar
+   # - m.room.encryption
+   # - m.room.name
+   # - m.room.create
+   #
+   # Uncomment the following to disable these defaults (so that only the event
+   # types listed in 'additional_event_types' are shared). Defaults to 'false'.
+   #
+   #disable_default_event_types: true
+
+   # Additional state event types to share with users when they are invited
+   # to a room.
+   #
+   # By default, this list is empty (so only the default event types are shared).
+   #
+   #additional_event_types:
+   #  - org.example.custom.event.type
 
 
 # A list of application service config files to use
@@ -2022,8 +1962,8 @@ saml2_config:
 #       sub-properties:
 #
 #       module: The class name of a custom mapping module. Default is
-#           'synapse.handlers.oidc_handler.JinjaOidcMappingProvider'.
-#           See https://github.com/matrix-org/synapse/blob/master/docs/sso_mapping_providers.md#openid-mapping-providers
+#           'synapse.handlers.oidc.JinjaOidcMappingProvider'.
+#           See https://matrix-org.github.io/synapse/latest/sso_mapping_providers.html#openid-mapping-providers
 #           for information on implementing a custom mapping provider.
 #
 #       config: Configuration for the mapping provider module. This section will
@@ -2074,7 +2014,7 @@ saml2_config:
 #     - attribute: groups
 #       value: "admin"
 #
-# See https://github.com/matrix-org/synapse/blob/master/docs/openid.md
+# See https://matrix-org.github.io/synapse/latest/openid.html
 # for information on how to configure these options.
 #
 # For backwards compatibility, it is also possible to configure a single OIDC
@@ -2143,6 +2083,9 @@ cas_config:
 # Additional settings to use with single-sign on systems such as OpenID Connect,
 # SAML2 and CAS.
 #
+# Server admins can configure custom templates for pages related to SSO. See
+# https://matrix-org.github.io/synapse/latest/templates.html for more information.
+#
 sso:
     # A list of client URLs which are whitelisted so that the user does not
     # have to confirm giving access to their account to the URL. Any client
@@ -2164,168 +2107,16 @@ sso:
     #  - https://riot.im/develop
     #  - https://my.custom.client/
 
-    # Directory in which Synapse will try to find the template files below.
-    # If not set, or the files named below are not found within the template
-    # directory, default templates from within the Synapse package will be used.
-    #
-    # Synapse will look for the following templates in this directory:
-    #
-    # * HTML page to prompt the user to choose an Identity Provider during
-    #   login: 'sso_login_idp_picker.html'.
-    #
-    #   This is only used if multiple SSO Identity Providers are configured.
-    #
-    #   When rendering, this template is given the following variables:
-    #     * redirect_url: the URL that the user will be redirected to after
-    #       login.
-    #
-    #     * server_name: the homeserver's name.
-    #
-    #     * providers: a list of available Identity Providers. Each element is
-    #       an object with the following attributes:
-    #
-    #         * idp_id: unique identifier for the IdP
-    #         * idp_name: user-facing name for the IdP
-    #         * idp_icon: if specified in the IdP config, an MXC URI for an icon
-    #              for the IdP
-    #         * idp_brand: if specified in the IdP config, a textual identifier
-    #              for the brand of the IdP
-    #
-    #   The rendered HTML page should contain a form which submits its results
-    #   back as a GET request, with the following query parameters:
-    #
-    #     * redirectUrl: the client redirect URI (ie, the `redirect_url` passed
-    #       to the template)
-    #
-    #     * idp: the 'idp_id' of the chosen IDP.
-    #
-    # * HTML page to prompt new users to enter a userid and confirm other
-    #   details: 'sso_auth_account_details.html'. This is only shown if the
-    #   SSO implementation (with any user_mapping_provider) does not return
-    #   a localpart.
-    #
-    #   When rendering, this template is given the following variables:
-    #
-    #     * server_name: the homeserver's name.
-    #
-    #     * idp: details of the SSO Identity Provider that the user logged in
-    #       with: an object with the following attributes:
-    #
-    #         * idp_id: unique identifier for the IdP
-    #         * idp_name: user-facing name for the IdP
-    #         * idp_icon: if specified in the IdP config, an MXC URI for an icon
-    #              for the IdP
-    #         * idp_brand: if specified in the IdP config, a textual identifier
-    #              for the brand of the IdP
-    #
-    #     * user_attributes: an object containing details about the user that
-    #       we received from the IdP. May have the following attributes:
-    #
-    #         * display_name: the user's display_name
-    #         * emails: a list of email addresses
-    #
-    #   The template should render a form which submits the following fields:
-    #
-    #     * username: the localpart of the user's chosen user id
-    #
-    # * HTML page allowing the user to consent to the server's terms and
-    #   conditions. This is only shown for new users, and only if
-    #   `user_consent.require_at_registration` is set.
-    #
-    #   When rendering, this template is given the following variables:
-    #
-    #     * server_name: the homeserver's name.
-    #
-    #     * user_id: the user's matrix proposed ID.
-    #
-    #     * user_profile.display_name: the user's proposed display name, if any.
-    #
-    #     * consent_version: the version of the terms that the user will be
-    #       shown
-    #
-    #     * terms_url: a link to the page showing the terms.
-    #
-    #   The template should render a form which submits the following fields:
-    #
-    #     * accepted_version: the version of the terms accepted by the user
-    #       (ie, 'consent_version' from the input variables).
-    #
-    # * HTML page for a confirmation step before redirecting back to the client
-    #   with the login token: 'sso_redirect_confirm.html'.
-    #
-    #   When rendering, this template is given the following variables:
-    #
-    #     * redirect_url: the URL the user is about to be redirected to.
-    #
-    #     * display_url: the same as `redirect_url`, but with the query
-    #                    parameters stripped. The intention is to have a
-    #                    human-readable URL to show to users, not to use it as
-    #                    the final address to redirect to.
-    #
-    #     * server_name: the homeserver's name.
-    #
-    #     * new_user: a boolean indicating whether this is the user's first time
-    #          logging in.
-    #
-    #     * user_id: the user's matrix ID.
-    #
-    #     * user_profile.avatar_url: an MXC URI for the user's avatar, if any.
-    #           None if the user has not set an avatar.
+    # Uncomment to keep a user's profile fields in sync with information from
+    # the identity provider. Currently only syncing the displayname is
+    # supported. Fields are checked on every SSO login, and are updated
+    # if necessary.
     #
-    #     * user_profile.display_name: the user's display name. None if the user
-    #           has not set a display name.
+    # Note that enabling this option will override user profile information,
+    # regardless of whether users have opted-out of syncing that
+    # information when first signing in. Defaults to false.
     #
-    # * HTML page which notifies the user that they are authenticating to confirm
-    #   an operation on their account during the user interactive authentication
-    #   process: 'sso_auth_confirm.html'.
-    #
-    #   When rendering, this template is given the following variables:
-    #     * redirect_url: the URL the user is about to be redirected to.
-    #
-    #     * description: the operation which the user is being asked to confirm
-    #
-    #     * idp: details of the Identity Provider that we will use to confirm
-    #       the user's identity: an object with the following attributes:
-    #
-    #         * idp_id: unique identifier for the IdP
-    #         * idp_name: user-facing name for the IdP
-    #         * idp_icon: if specified in the IdP config, an MXC URI for an icon
-    #              for the IdP
-    #         * idp_brand: if specified in the IdP config, a textual identifier
-    #              for the brand of the IdP
-    #
-    # * HTML page shown after a successful user interactive authentication session:
-    #   'sso_auth_success.html'.
-    #
-    #   Note that this page must include the JavaScript which notifies of a successful authentication
-    #   (see https://matrix.org/docs/spec/client_server/r0.6.0#fallback).
-    #
-    #   This template has no additional variables.
-    #
-    # * HTML page shown after a user-interactive authentication session which
-    #   does not map correctly onto the expected user: 'sso_auth_bad_user.html'.
-    #
-    #   When rendering, this template is given the following variables:
-    #     * server_name: the homeserver's name.
-    #     * user_id_to_verify: the MXID of the user that we are trying to
-    #       validate.
-    #
-    # * HTML page shown during single sign-on if a deactivated user (according to Synapse's database)
-    #   attempts to login: 'sso_account_deactivated.html'.
-    #
-    #   This template has no additional variables.
-    #
-    # * HTML page to display to users if something goes wrong during the
-    #   OpenID Connect authentication process: 'sso_error.html'.
-    #
-    #   When rendering, this template is given two variables:
-    #     * error: the technical name of the error
-    #     * error_description: a human-readable message for the error
-    #
-    # You can see the default templates at:
-    # https://github.com/matrix-org/synapse/tree/master/synapse/res/templates
-    #
-    #template_dir: "res/templates"
+    #update_profile_information: true
 
 
 # JSON web token integration. The following settings can be used to make
@@ -2341,7 +2132,7 @@ sso:
 # Note that this is a non-standard login type and client support is
 # expected to be non-existent.
 #
-# See https://github.com/matrix-org/synapse/blob/master/docs/jwt.md.
+# See https://matrix-org.github.io/synapse/latest/jwt.html.
 #
 #jwt_config:
     # Uncomment the following to enable authorization using JSON web
@@ -2445,6 +2236,10 @@ ui_auth:
     # the user-interactive authentication process, by allowing for multiple
     # (and potentially different) operations to use the same validation session.
     #
+    # This is ignored for potentially "dangerous" operations (including
+    # deactivating an account, modifying an account password, and
+    # adding a 3PID).
+    #
     # Uncomment below to allow for credential validation to last for 15
     # seconds.
     #
@@ -2453,6 +2248,9 @@ ui_auth:
 
 # Configuration for sending emails from Synapse.
 #
+# Server admins can configure custom templates for email content. See
+# https://matrix-org.github.io/synapse/latest/templates.html for more information.
+#
 email:
   # The hostname of the outgoing SMTP server to use. Defaults to 'localhost'.
   #
@@ -2475,6 +2273,14 @@ email:
   #
   #require_transport_security: true
 
+  # Uncomment the following to disable TLS for SMTP.
+  #
+  # By default, if the server supports TLS, it will be used, and the server
+  # must present a certificate that is valid for 'smtp_host'. If this option
+  # is set to false, TLS will not be used.
+  #
+  #enable_tls: false
+
   # notif_from defines the "From" address to use when sending emails.
   # It must be set if email sending is enabled.
   #
@@ -2521,49 +2327,6 @@ email:
   #
   #invite_client_location: https://app.element.io
 
-  # Directory in which Synapse will try to find the template files below.
-  # If not set, or the files named below are not found within the template
-  # directory, default templates from within the Synapse package will be used.
-  #
-  # Synapse will look for the following templates in this directory:
-  #
-  # * The contents of email notifications of missed events: 'notif_mail.html' and
-  #   'notif_mail.txt'.
-  #
-  # * The contents of account expiry notice emails: 'notice_expiry.html' and
-  #   'notice_expiry.txt'.
-  #
-  # * The contents of password reset emails sent by the homeserver:
-  #   'password_reset.html' and 'password_reset.txt'
-  #
-  # * An HTML page that a user will see when they follow the link in the password
-  #   reset email. The user will be asked to confirm the action before their
-  #   password is reset: 'password_reset_confirmation.html'
-  #
-  # * HTML pages for success and failure that a user will see when they confirm
-  #   the password reset flow using the page above: 'password_reset_success.html'
-  #   and 'password_reset_failure.html'
-  #
-  # * The contents of address verification emails sent during registration:
-  #   'registration.html' and 'registration.txt'
-  #
-  # * HTML pages for success and failure that a user will see when they follow
-  #   the link in an address verification email sent during registration:
-  #   'registration_success.html' and 'registration_failure.html'
-  #
-  # * The contents of address verification emails sent when an address is added
-  #   to a Matrix account: 'add_threepid.html' and 'add_threepid.txt'
-  #
-  # * HTML pages for success and failure that a user will see when they follow
-  #   the link in an address verification email sent when an address is added
-  #   to a Matrix account: 'add_threepid_success.html' and
-  #   'add_threepid_failure.html'
-  #
-  # You can see the default templates at:
-  # https://github.com/matrix-org/synapse/tree/master/synapse/res/templates
-  #
-  #template_dir: "res/templates"
-
   # Subjects to use when sending emails from Synapse.
   #
   # The placeholder '%(app)s' will be replaced with the value of the 'app_name'
@@ -2637,7 +2400,7 @@ email:
 # ex. LDAP, external tokens, etc.
 #
 # For more information and known implementations, please see
-# https://github.com/matrix-org/synapse/blob/master/docs/password_auth_providers.md
+# https://matrix-org.github.io/synapse/latest/password_auth_providers.html
 #
 # Note: instances wishing to use SAML or CAS authentication should
 # instead use the `saml2_config` or `cas_config` options,
@@ -2692,19 +2455,6 @@ push:
   #group_unread_count_by_room: false
 
 
-# Spam checkers are third-party modules that can block specific actions
-# of local users, such as creating rooms and registering undesirable
-# usernames, as well as remote users by redacting incoming events.
-#
-spam_checker:
-   #- module: "my_custom_project.SuperSpamChecker"
-   #  config:
-   #    example_option: 'things'
-   #- module: "some_other_project.BadEventStopper"
-   #  config:
-   #    example_stop_events_from: ['@bad:example.com']
-
-
 ## Rooms ##
 
 # Controls whether locally-created rooms should be end-to-end encrypted by
@@ -2752,7 +2502,7 @@ user_directory:
     #
     # If you set it true, you'll have to rebuild the user_directory search
     # indexes, see:
-    # https://github.com/matrix-org/synapse/blob/master/docs/user_directory.md
+    # https://matrix-org.github.io/synapse/latest/user_directory.html
     #
     # Uncomment to return search results containing all known users, even if that
     # user does not share a room with the requester.
@@ -2778,7 +2528,7 @@ user_directory:
 # User Consent configuration
 #
 # for detailed instructions, see
-# https://github.com/matrix-org/synapse/blob/master/docs/consent_tracking.md
+# https://matrix-org.github.io/synapse/latest/consent_tracking.html
 #
 # Parts of this section are required if enabling the 'consent' resource under
 # 'listeners', in particular 'template_dir' and 'version'.
@@ -2828,7 +2578,7 @@ user_directory:
 
 
 # Settings for local room and user statistics collection. See
-# docs/room_and_user_statistics.md.
+# https://matrix-org.github.io/synapse/latest/room_and_user_statistics.html.
 #
 stats:
   # Uncomment the following to disable room and user statistics. Note that doing
@@ -2837,11 +2587,6 @@ stats:
   #
   #enabled: false
 
-  # The size of each timeslice in the room_stats_historical and
-  # user_stats_historical tables, as a time period. Defaults to "1d".
-  #
-  #bucket_size: 1h
-
 
 # Server Notices room configuration
 #
@@ -2928,19 +2673,6 @@ stats:
 #    action: allow
 
 
-# Server admins can define a Python module that implements extra rules for
-# allowing or denying incoming events. In order to work, this module needs to
-# override the methods defined in synapse/events/third_party_rules.py.
-#
-# This feature is designed to be used in closed federations only, where each
-# participating server enforces the same rules.
-#
-#third_party_event_rules:
-#  module: "my_custom_project.SuperRulesSet"
-#  config:
-#    example_option: 'things'
-
-
 ## Opentracing ##
 
 # These settings enable opentracing, which implements distributed tracing.
@@ -2955,7 +2687,8 @@ opentracing:
     #enabled: true
 
     # The list of homeservers we wish to send and receive span contexts and span baggage.
-    # See docs/opentracing.rst
+    # See https://matrix-org.github.io/synapse/latest/opentracing.html.
+    #
     # This is a list of regexes which are matched against the server_name of the
     # homeserver.
     #
@@ -2964,19 +2697,26 @@ opentracing:
     #homeserver_whitelist:
     #  - ".*"
 
+    # A list of the matrix IDs of users whose requests will always be traced,
+    # even if the tracing system would otherwise drop the traces due to
+    # probabilistic sampling.
+    #
+    # By default, the list is empty.
+    #
+    #force_tracing_for_users:
+    #  - "@user1:server_name"
+    #  - "@user2:server_name"
+
     # Jaeger can be configured to sample traces at different rates.
     # All configuration options provided by Jaeger can be set here.
-    # Jaeger's configuration mostly related to trace sampling which
+    # Jaeger's configuration is mostly related to trace sampling which
     # is documented here:
-    # https://www.jaegertracing.io/docs/1.13/sampling/.
+    # https://www.jaegertracing.io/docs/latest/sampling/.
     #
     #jaeger_config:
     #  sampler:
     #    type: const
     #    param: 1
-
-    #  Logging whether spans were started and reported
-    #
     #  logging:
     #    false
 
diff --git a/docs/sample_log_config.yaml b/docs/sample_log_config.yaml
index ff3c747180..2485ad25ed 100644
--- a/docs/sample_log_config.yaml
+++ b/docs/sample_log_config.yaml
@@ -7,7 +7,7 @@
 # be ingested by ELK stacks. See [2] for details.
 #
 # [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
-# [2]: https://github.com/matrix-org/synapse/blob/master/docs/structured_logging.md
+# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
 
 version: 1
 
@@ -24,18 +24,31 @@ handlers:
         backupCount: 3  # Does not include the current log file.
         encoding: utf8
 
-    # Default to buffering writes to log file for efficiency. This means that
-    # will be a delay for INFO/DEBUG logs to get written, but WARNING/ERROR
-    # logs will still be flushed immediately.
+    # Default to buffering writes to log file for efficiency.
+    # WARNING/ERROR logs will still be flushed immediately, but there will be a
+    # delay (of up to `period` seconds, or until the buffer is full with
+    # `capacity` messages) before INFO/DEBUG logs get written.
     buffer:
-        class: logging.handlers.MemoryHandler
+        class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler
         target: file
-        # The capacity is the number of log lines that are buffered before
-        # being written to disk. Increasing this will lead to better
+
+        # The capacity is the maximum number of log lines that are buffered
+        # before being written to disk. Increasing this will lead to better
         # performance, at the expensive of it taking longer for log lines to
         # be written to disk.
+        # This parameter is required.
         capacity: 10
-        flushLevel: 30  # Flush for WARNING logs as well
+
+        # Logs with a level at or above the flush level will cause the buffer to
+        # be flushed immediately.
+        # Default value: 40 (ERROR)
+        # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG)
+        flushLevel: 30  # Flush immediately for WARNING logs and higher
+
+        # The period of time, in seconds, between forced flushes.
+        # Messages will not be delayed for longer than this time.
+        # Default value: 5 seconds
+        period: 5
 
     # A handler that writes logs to stderr. Unused by default, but can be used
     # instead of "buffer" and "file" in the logger handlers.
diff --git a/docs/server_notices.md b/docs/server_notices.md
index 950a6608e9..339d10a0ab 100644
--- a/docs/server_notices.md
+++ b/docs/server_notices.md
@@ -3,8 +3,8 @@
 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
 channel whereby server administrators can send messages to users on the server.
 
-They are used as part of communication of the server polices(see
-[consent_tracking.md](consent_tracking.md)), however the intention is that
+They are used as part of communication of the server polices (see
+[Consent Tracking](consent_tracking.md)), however the intention is that
 they may also find a use for features such as "Message of the day".
 
 This is a feature specific to Synapse, but it uses standard Matrix
diff --git a/docs/setup/forward_proxy.md b/docs/setup/forward_proxy.md
new file mode 100644
index 0000000000..494c14893b
--- /dev/null
+++ b/docs/setup/forward_proxy.md
@@ -0,0 +1,74 @@
+# Using a forward proxy with Synapse
+
+You can use Synapse with a forward or outbound proxy. An example of when
+this is necessary is in corporate environments behind a DMZ (demilitarized zone).
+Synapse supports routing outbound HTTP(S) requests via a proxy. Only HTTP(S)
+proxy is supported, not SOCKS proxy or anything else.
+
+## Configure
+
+The `http_proxy`, `https_proxy`, `no_proxy` environment variables are used to
+specify proxy settings. The environment variable is not case sensitive.
+- `http_proxy`: Proxy server to use for HTTP requests.
+- `https_proxy`: Proxy server to use for HTTPS requests.
+- `no_proxy`: Comma-separated list of hosts, IP addresses, or IP ranges in CIDR
+  format which should not use the proxy. Synapse will directly connect to these hosts.
+
+The `http_proxy` and `https_proxy` environment variables have the form: `[scheme://][<username>:<password>@]<host>[:<port>]`
+- Supported schemes are `http://` and `https://`. The default scheme is `http://`
+  for compatibility reasons; it is recommended to set a scheme. If scheme is set
+  to `https://` the connection uses TLS between Synapse and the proxy.
+
+  **NOTE**: Synapse validates the certificates. If the certificate is not
+  valid, then the connection is dropped.
+- Default port if not given is `1080`.
+- Username and password are optional and will be used to authenticate against
+  the proxy.
+
+**Examples**
+- HTTP_PROXY=http://USERNAME:PASSWORD@10.0.1.1:8080/
+- HTTPS_PROXY=http://USERNAME:PASSWORD@proxy.example.com:8080/
+- NO_PROXY=master.hostname.example.com,10.1.0.0/16,172.30.0.0/16
+
+**NOTE**:
+Synapse does not apply the IP blacklist to connections through the proxy (since
+the DNS resolution is done by the proxy). It is expected that the proxy or firewall
+will apply blacklisting of IP addresses.
+
+## Connection types
+
+The proxy will be **used** for:
+
+- push
+- url previews
+- phone-home stats
+- recaptcha validation
+- CAS auth validation
+- OpenID Connect
+- Outbound federation
+- Federation (checking public key revocation)
+- Fetching public keys of other servers
+- Downloading remote media
+
+It will **not be used** for:
+
+- Application Services
+- Identity servers
+- In worker configurations
+  - connections between workers
+  - connections from workers to Redis
+
+## Troubleshooting
+
+If a proxy server is used with TLS (HTTPS) and no connections are established,
+it is most likely due to the proxy's certificates. To test this, the validation
+in Synapse can be deactivated.
+
+**NOTE**: This has an impact on security and is for testing purposes only!
+
+To deactivate the certificate validation, the following setting must be made in
+[homserver.yaml](../usage/configuration/homeserver_sample_config.md).
+
+```yaml
+use_insecure_ssl_client_just_for_testing_do_not_use: true
+```
diff --git a/docs/setup/installation.md b/docs/setup/installation.md
new file mode 100644
index 0000000000..8540a7b0c1
--- /dev/null
+++ b/docs/setup/installation.md
@@ -0,0 +1,600 @@
+# Installation Instructions
+
+There are 3 steps to follow under **Installation Instructions**.
+
+- [Installation Instructions](#installation-instructions)
+  - [Choosing your server name](#choosing-your-server-name)
+  - [Installing Synapse](#installing-synapse)
+    - [Installing from source](#installing-from-source)
+      - [Platform-specific prerequisites](#platform-specific-prerequisites)
+        - [Debian/Ubuntu/Raspbian](#debianubunturaspbian)
+        - [ArchLinux](#archlinux)
+        - [CentOS/Fedora](#centosfedora)
+        - [macOS](#macos)
+        - [OpenSUSE](#opensuse)
+        - [OpenBSD](#openbsd)
+        - [Windows](#windows)
+    - [Prebuilt packages](#prebuilt-packages)
+      - [Docker images and Ansible playbooks](#docker-images-and-ansible-playbooks)
+      - [Debian/Ubuntu](#debianubuntu)
+        - [Matrix.org packages](#matrixorg-packages)
+        - [Downstream Debian packages](#downstream-debian-packages)
+        - [Downstream Ubuntu packages](#downstream-ubuntu-packages)
+      - [Fedora](#fedora)
+      - [OpenSUSE](#opensuse-1)
+      - [SUSE Linux Enterprise Server](#suse-linux-enterprise-server)
+      - [ArchLinux](#archlinux-1)
+      - [Void Linux](#void-linux)
+      - [FreeBSD](#freebsd)
+      - [OpenBSD](#openbsd-1)
+      - [NixOS](#nixos)
+  - [Setting up Synapse](#setting-up-synapse)
+    - [Using PostgreSQL](#using-postgresql)
+    - [TLS certificates](#tls-certificates)
+    - [Client Well-Known URI](#client-well-known-uri)
+    - [Email](#email)
+    - [Registering a user](#registering-a-user)
+    - [Setting up a TURN server](#setting-up-a-turn-server)
+    - [URL previews](#url-previews)
+    - [Troubleshooting Installation](#troubleshooting-installation)
+
+
+## Choosing your server name
+
+It is important to choose the name for your server before you install Synapse,
+because it cannot be changed later.
+
+The server name determines the "domain" part of user-ids for users on your
+server: these will all be of the format `@user:my.domain.name`. It also
+determines how other matrix servers will reach yours for federation.
+
+For a test configuration, set this to the hostname of your server. For a more
+production-ready setup, you will probably want to specify your domain
+(`example.com`) rather than a matrix-specific hostname here (in the same way
+that your email address is probably `user@example.com` rather than
+`user@email.example.com`) - but doing so may require more advanced setup: see
+[Setting up Federation](../federate.md).
+
+## Installing Synapse
+
+### Installing from source
+
+(Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
+
+When installing from source please make sure that the [Platform-specific prerequisites](#platform-specific-prerequisites) are already installed.
+
+System requirements:
+
+- POSIX-compliant system (tested on Linux & OS X)
+- Python 3.5.2 or later, up to Python 3.9.
+- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
+
+
+To install the Synapse homeserver run:
+
+```sh
+mkdir -p ~/synapse
+virtualenv -p python3 ~/synapse/env
+source ~/synapse/env/bin/activate
+pip install --upgrade pip
+pip install --upgrade setuptools
+pip install matrix-synapse
+```
+
+This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
+and install it, along with the python libraries it uses, into a virtual environment
+under `~/synapse/env`.  Feel free to pick a different directory if you
+prefer.
+
+This Synapse installation can then be later upgraded by using pip again with the
+update flag:
+
+```sh
+source ~/synapse/env/bin/activate
+pip install -U matrix-synapse
+```
+
+Before you can start Synapse, you will need to generate a configuration
+file. To do this, run (in your virtualenv, as before):
+
+```sh
+cd ~/synapse
+python -m synapse.app.homeserver \
+    --server-name my.domain.name \
+    --config-path homeserver.yaml \
+    --generate-config \
+    --report-stats=[yes|no]
+```
+
+... substituting an appropriate value for `--server-name`.
+
+This command will generate you a config file that you can then customise, but it will
+also generate a set of keys for you. These keys will allow your homeserver to
+identify itself to other homeserver, so don't lose or delete them. It would be
+wise to back them up somewhere safe. (If, for whatever reason, you do need to
+change your homeserver's keys, you may find that other homeserver have the
+old key cached. If you update the signing key, you should change the name of the
+key in the `<server name>.signing.key` file (the second word) to something
+different. See the [spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys) for more information on key management).
+
+To actually run your new homeserver, pick a working directory for Synapse to
+run (e.g. `~/synapse`), and:
+
+```sh
+cd ~/synapse
+source env/bin/activate
+synctl start
+```
+
+#### Platform-specific prerequisites
+
+Synapse is written in Python but some of the libraries it uses are written in
+C. So before we can install Synapse itself we need a working C compiler and the
+header files for Python C extensions.
+
+##### Debian/Ubuntu/Raspbian
+
+Installing prerequisites on Ubuntu or Debian:
+
+```sh
+sudo apt install build-essential python3-dev libffi-dev \
+                     python3-pip python3-setuptools sqlite3 \
+                     libssl-dev virtualenv libjpeg-dev libxslt1-dev
+```
+
+##### ArchLinux
+
+Installing prerequisites on ArchLinux:
+
+```sh
+sudo pacman -S base-devel python python-pip \
+               python-setuptools python-virtualenv sqlite3
+```
+
+##### CentOS/Fedora
+
+Installing prerequisites on CentOS or Fedora Linux:
+
+```sh
+sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+                 libwebp-devel libxml2-devel libxslt-devel libpq-devel \
+                 python3-virtualenv libffi-devel openssl-devel python3-devel
+sudo dnf groupinstall "Development Tools"
+```
+
+##### macOS
+
+Installing prerequisites on macOS:
+
+You may need to install the latest Xcode developer tools:
+```sh
+xcode-select --install
+```
+
+On ARM-based Macs you may need to explicitly install libjpeg which is a pillow dependency. You can use Homebrew (https://brew.sh):
+```sh
+ brew install jpeg
+ ```
+
+On macOS Catalina (10.15) you may need to explicitly install OpenSSL
+via brew and inform `pip` about it so that `psycopg2` builds:
+
+```sh
+brew install openssl@1.1
+export LDFLAGS="-L/usr/local/opt/openssl/lib"
+export CPPFLAGS="-I/usr/local/opt/openssl/include"
+```
+
+##### OpenSUSE
+
+Installing prerequisites on openSUSE:
+
+```sh
+sudo zypper in -t pattern devel_basis
+sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
+               python-devel libffi-devel libopenssl-devel libjpeg62-devel
+```
+
+##### OpenBSD
+
+A port of Synapse is available under `net/synapse`. The filesystem
+underlying the homeserver directory (defaults to `/var/synapse`) has to be
+mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
+and mounting it to `/var/synapse` should be taken into consideration.
+
+To be able to build Synapse's dependency on python the `WRKOBJDIR`
+(cf. `bsd.port.mk(5)`) for building python, too, needs to be on a filesystem
+mounted with `wxallowed` (cf. `mount(8)`).
+
+Creating a `WRKOBJDIR` for building python under `/usr/local` (which on a
+default OpenBSD installation is mounted with `wxallowed`):
+
+```sh
+doas mkdir /usr/local/pobj_wxallowed
+```
+
+Assuming `PORTS_PRIVSEP=Yes` (cf. `bsd.port.mk(5)`) and `SUDO=doas` are
+configured in `/etc/mk.conf`:
+
+```sh
+doas chown _pbuild:_pbuild /usr/local/pobj_wxallowed
+```
+
+Setting the `WRKOBJDIR` for building python:
+
+```sh
+echo WRKOBJDIR_lang/python/3.7=/usr/local/pobj_wxallowed  \\nWRKOBJDIR_lang/python/2.7=/usr/local/pobj_wxallowed >> /etc/mk.conf
+```
+
+Building Synapse:
+
+```sh
+cd /usr/ports/net/synapse
+make install
+```
+
+##### Windows
+
+If you wish to run or develop Synapse on Windows, the Windows Subsystem For
+Linux provides a Linux environment on Windows 10 which is capable of using the
+Debian, Fedora, or source installation methods. More information about WSL can
+be found at <https://docs.microsoft.com/en-us/windows/wsl/install-win10> for
+Windows 10 and <https://docs.microsoft.com/en-us/windows/wsl/install-on-server>
+for Windows Server.
+
+### Prebuilt packages
+
+As an alternative to installing from source, prebuilt packages are available
+for a number of platforms.
+
+#### Docker images and Ansible playbooks
+
+There is an official synapse image available at
+<https://hub.docker.com/r/matrixdotorg/synapse> which can be used with
+the docker-compose file available at
+[contrib/docker](https://github.com/matrix-org/synapse/tree/develop/contrib/docker).
+Further information on this including configuration options is available in the README
+on hub.docker.com.
+
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
+Dockerfile to automate a synapse server in a single Docker image, at
+<https://hub.docker.com/r/avhost/docker-matrix/tags/>
+
+Slavi Pantaleev has created an Ansible playbook,
+which installs the offical Docker image of Matrix Synapse
+along with many other Matrix-related services (Postgres database, Element, coturn,
+ma1sd, SSL support, etc.).
+For more details, see
+<https://github.com/spantaleev/matrix-docker-ansible-deploy>
+
+#### Debian/Ubuntu
+
+##### Matrix.org packages
+
+Matrix.org provides Debian/Ubuntu packages of Synapse via
+<https://packages.matrix.org/debian/>.  To install the latest release:
+
+```sh
+sudo apt install -y lsb-release wget apt-transport-https
+sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
+    sudo tee /etc/apt/sources.list.d/matrix-org.list
+sudo apt update
+sudo apt install matrix-synapse-py3
+```
+
+Packages are also published for release candidates. To enable the prerelease
+channel, add `prerelease` to the `sources.list` line. For example:
+
+```sh
+sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main prerelease" |
+    sudo tee /etc/apt/sources.list.d/matrix-org.list
+sudo apt update
+sudo apt install matrix-synapse-py3
+```
+
+The fingerprint of the repository signing key (as shown by `gpg
+/usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
+`AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
+
+##### Downstream Debian packages
+
+We do not recommend using the packages from the default Debian `buster`
+repository at this time, as they are old and suffer from known security
+vulnerabilities. You can install the latest version of Synapse from
+[our repository](#matrixorg-packages) or from `buster-backports`. Please
+see the [Debian documentation](https://backports.debian.org/Instructions/)
+for information on how to use backports.
+
+If you are using Debian `sid` or testing, Synapse is available in the default
+repositories and it should be possible to install it simply with:
+
+```sh
+sudo apt install matrix-synapse
+```
+
+##### Downstream Ubuntu packages
+
+We do not recommend using the packages in the default Ubuntu repository
+at this time, as they are old and suffer from known security vulnerabilities.
+The latest version of Synapse can be installed from [our repository](#matrixorg-packages).
+
+#### Fedora
+
+Synapse is in the Fedora repositories as `matrix-synapse`:
+
+```sh
+sudo dnf install matrix-synapse
+```
+
+Oleg Girko provides Fedora RPMs at
+<https://obs.infoserver.lv/project/monitor/matrix-synapse>
+
+#### OpenSUSE
+
+Synapse is in the OpenSUSE repositories as `matrix-synapse`:
+
+```sh
+sudo zypper install matrix-synapse
+```
+
+#### SUSE Linux Enterprise Server
+
+Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
+<https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/>
+
+#### ArchLinux
+
+The quickest way to get up and running with ArchLinux is probably with the community package
+<https://www.archlinux.org/packages/community/any/matrix-synapse/>, which should pull in most of
+the necessary dependencies.
+
+pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
+
+```sh
+sudo pip install --upgrade pip
+```
+
+If you encounter an error with lib bcrypt causing an Wrong ELF Class:
+ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
+compile it under the right architecture. (This should not be needed if
+installing under virtualenv):
+
+```sh
+sudo pip uninstall py-bcrypt
+sudo pip install py-bcrypt
+```
+
+#### Void Linux
+
+Synapse can be found in the void repositories as 'synapse':
+
+```sh
+xbps-install -Su
+xbps-install -S synapse
+```
+
+#### FreeBSD
+
+Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
+
+- Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
+- Packages: `pkg install py37-matrix-synapse`
+
+#### OpenBSD
+
+As of OpenBSD 6.7 Synapse is available as a pre-compiled binary. The filesystem
+underlying the homeserver directory (defaults to `/var/synapse`) has to be
+mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
+and mounting it to `/var/synapse` should be taken into consideration.
+
+Installing Synapse:
+
+```sh
+doas pkg_add synapse
+```
+
+#### NixOS
+
+Robin Lambertz has packaged Synapse for NixOS at:
+<https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix>
+
+## Setting up Synapse
+
+Once you have installed synapse as above, you will need to configure it.
+
+### Using PostgreSQL
+
+By default Synapse uses an [SQLite](https://sqlite.org/) database and in doing so trades
+performance for convenience. Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org)
+instead. Advantages include:
+
+- significant performance improvements due to the superior threading and
+  caching model, smarter query optimiser
+- allowing the DB to be run on separate hardware
+
+For information on how to install and use PostgreSQL in Synapse, please see
+[Using Postgres](../postgres.md)
+
+SQLite is only acceptable for testing purposes. SQLite should not be used in
+a production server. Synapse will perform poorly when using
+SQLite, especially when participating in large rooms.
+
+### TLS certificates
+
+The default configuration exposes a single HTTP port on the local
+interface: `http://localhost:8008`. It is suitable for local testing,
+but for any practical use, you will need Synapse's APIs to be served
+over HTTPS.
+
+The recommended way to do so is to set up a reverse proxy on port
+`8448`. You can find documentation on doing so in
+[the reverse proxy documentation](../reverse_proxy.md).
+
+Alternatively, you can configure Synapse to expose an HTTPS port. To do
+so, you will need to edit `homeserver.yaml`, as follows:
+
+- First, under the `listeners` section, uncomment the configuration for the
+  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
+  each line). The relevant lines are like this:
+
+```yaml
+  - port: 8448
+    type: http
+    tls: true
+    resources:
+      - names: [client, federation]
+  ```
+
+- You will also need to uncomment the `tls_certificate_path` and
+  `tls_private_key_path` lines under the `TLS` section. You will need to manage
+  provisioning of these certificates yourself.
+
+  If you are using your own certificate, be sure to use a `.pem` file that
+  includes the full certificate chain including any intermediate certificates
+  (for instance, if using certbot, use `fullchain.pem` as your certificate, not
+  `cert.pem`).
+
+For a more detailed guide to configuring your server for federation, see
+[Federation](../federate.md).
+
+### Client Well-Known URI
+
+Setting up the client Well-Known URI is optional but if you set it up, it will
+allow users to enter their full username (e.g. `@user:<server_name>`) into clients
+which support well-known lookup to automatically configure the homeserver and
+identity server URLs. This is useful so that users don't have to memorize or think
+about the actual homeserver URL you are using.
+
+The URL `https://<server_name>/.well-known/matrix/client` should return JSON in
+the following format.
+
+```json
+{
+  "m.homeserver": {
+    "base_url": "https://<matrix.example.com>"
+  }
+}
+```
+
+It can optionally contain identity server information as well.
+
+```json
+{
+  "m.homeserver": {
+    "base_url": "https://<matrix.example.com>"
+  },
+  "m.identity_server": {
+    "base_url": "https://<identity.example.com>"
+  }
+}
+```
+
+To work in browser based clients, the file must be served with the appropriate
+Cross-Origin Resource Sharing (CORS) headers. A recommended value would be
+`Access-Control-Allow-Origin: *` which would allow all browser based clients to
+view it.
+
+In nginx this would be something like:
+
+```nginx
+location /.well-known/matrix/client {
+    return 200 '{"m.homeserver": {"base_url": "https://<matrix.example.com>"}}';
+    default_type application/json;
+    add_header Access-Control-Allow-Origin *;
+}
+```
+
+You should also ensure the `public_baseurl` option in `homeserver.yaml` is set
+correctly. `public_baseurl` should be set to the URL that clients will use to
+connect to your server. This is the same URL you put for the `m.homeserver`
+`base_url` above.
+
+```yaml
+public_baseurl: "https://<matrix.example.com>"
+```
+
+### Email
+
+It is desirable for Synapse to have the capability to send email. This allows
+Synapse to send password reset emails, send verifications when an email address
+is added to a user's account, and send email notifications to users when they
+receive new messages.
+
+To configure an SMTP server for Synapse, modify the configuration section
+headed `email`, and be sure to have at least the `smtp_host`, `smtp_port`
+and `notif_from` fields filled out.  You may also need to set `smtp_user`,
+`smtp_pass`, and `require_transport_security`.
+
+If email is not configured, password reset, registration and notifications via
+email will be disabled.
+
+### Registering a user
+
+The easiest way to create a new user is to do so from a client like [Element](https://element.io/).
+
+Alternatively, you can do so from the command line. This can be done as follows:
+
+ 1. If synapse was installed via pip, activate the virtualenv as follows (if Synapse was
+    installed via a prebuilt package, `register_new_matrix_user` should already be
+    on the search path):
+    ```sh
+    cd ~/synapse
+    source env/bin/activate
+    synctl start # if not already running
+    ```
+ 2. Run the following command:
+    ```sh
+    register_new_matrix_user -c homeserver.yaml http://localhost:8008
+    ```
+
+This will prompt you to add details for the new user, and will then connect to
+the running Synapse to create the new user. For example:
+```
+New user localpart: erikj
+Password:
+Confirm password:
+Make admin [no]:
+Success!
+```
+
+This process uses a setting `registration_shared_secret` in
+`homeserver.yaml`, which is shared between Synapse itself and the
+`register_new_matrix_user` script. It doesn't matter what it is (a random
+value is generated by `--generate-config`), but it should be kept secret, as
+anyone with knowledge of it can register users, including admin accounts,
+on your server even if `enable_registration` is `false`.
+
+### Setting up a TURN server
+
+For reliable VoIP calls to be routed via this homeserver, you MUST configure
+a TURN server. See [TURN setup](../turn-howto.md) for details.
+
+### URL previews
+
+Synapse includes support for previewing URLs, which is disabled by default.  To
+turn it on you must enable the `url_preview_enabled: True` config parameter
+and explicitly specify the IP ranges that Synapse is not allowed to spider for
+previewing in the `url_preview_ip_range_blacklist` configuration parameter.
+This is critical from a security perspective to stop arbitrary Matrix users
+spidering 'internal' URLs on your network. At the very least we recommend that
+your loopback and RFC1918 IP addresses are blacklisted.
+
+This also requires the optional `lxml` python dependency to be  installed. This
+in turn requires the `libxml2` library to be available - on  Debian/Ubuntu this
+means `apt-get install libxml2-dev`, or equivalent for your OS.
+
+### Troubleshooting Installation
+
+`pip` seems to leak *lots* of memory during installation. For instance, a Linux
+host with 512MB of RAM may run out of memory whilst installing Twisted. If this
+happens, you will have to individually install the dependencies which are
+failing, e.g.:
+
+```sh
+pip install twisted
+```
+
+If you have any other problems, feel free to ask in
+[#synapse:matrix.org](https://matrix.to/#/#synapse:matrix.org).
diff --git a/docs/spam_checker.md b/docs/spam_checker.md
index 52947f605e..1b6d814937 100644
--- a/docs/spam_checker.md
+++ b/docs/spam_checker.md
@@ -1,3 +1,9 @@
+<h2 style="color:red">
+This page of the Synapse documentation is now deprecated. For up to date
+documentation on setting up or writing a spam checker module, please see
+<a href="modules.md">this page</a>.
+</h2>
+
 # Handling spam in Synapse
 
 Synapse has support to customize spam checking behavior. It can plug into a
diff --git a/docs/sso_mapping_providers.md b/docs/sso_mapping_providers.md
index e1d6ede7ba..7a407012e0 100644
--- a/docs/sso_mapping_providers.md
+++ b/docs/sso_mapping_providers.md
@@ -67,8 +67,8 @@ A custom mapping provider must specify the following methods:
     - Arguments:
       - `userinfo` - A `authlib.oidc.core.claims.UserInfo` object to extract user
                      information from.
-    - This method must return a string, which is the unique identifier for the
-      user. Commonly the ``sub`` claim of the response.
+    - This method must return a string, which is the unique, immutable identifier
+      for the user. Commonly the `sub` claim of the response.
 * `map_user_attributes(self, userinfo, token, failures)`
     - This method must be async.
     - Arguments:
@@ -87,7 +87,9 @@ A custom mapping provider must specify the following methods:
                      `localpart` value, such as `john.doe1`.
     - Returns a dictionary with two keys:
       - `localpart`: A string, used to generate the Matrix ID. If this is
-        `None`, the user is prompted to pick their own username.
+        `None`, the user is prompted to pick their own username. This is only used
+        during a user's first login. Once a localpart has been associated with a
+        remote user ID (see `get_remote_user_id`) it cannot be updated.
       - `displayname`: An optional string, the display name for the user.
 * `get_extra_attributes(self, userinfo, token)`
     - This method must be async.
@@ -106,7 +108,7 @@ A custom mapping provider must specify the following methods:
 
 Synapse has a built-in OpenID mapping provider if a custom provider isn't
 specified in the config. It is located at
-[`synapse.handlers.oidc_handler.JinjaOidcMappingProvider`](../synapse/handlers/oidc_handler.py).
+[`synapse.handlers.oidc.JinjaOidcMappingProvider`](https://github.com/matrix-org/synapse/blob/develop/synapse/handlers/oidc.py).
 
 ## SAML Mapping Providers
 
@@ -153,8 +155,8 @@ A custom mapping provider must specify the following methods:
                           information from.
       - `client_redirect_url` - A string, the URL that the client will be
                                 redirected to.
-    - This method must return a string, which is the unique identifier for the
-      user. Commonly the ``uid`` claim of the response.
+    - This method must return a string, which is the unique, immutable identifier
+      for the user. Commonly the `uid` claim of the response.
 * `saml_response_to_user_attributes(self, saml_response, failures, client_redirect_url)`
     - Arguments:
       - `saml_response` - A `saml2.response.AuthnResponse` object to extract user
@@ -172,8 +174,10 @@ A custom mapping provider must specify the following methods:
                                 redirected to.
     - This method must return a dictionary, which will then be used by Synapse
       to build a new user. The following keys are allowed:
-       * `mxid_localpart` - The mxid localpart of the new user.  If this is
-         `None`, the user is prompted to pick their own username.
+       * `mxid_localpart` - A string, the mxid localpart of the new user. If this is
+         `None`, the user is prompted to pick their own username. This is only used
+         during a user's first login. Once a localpart has been associated with a
+         remote user ID (see `get_remote_user_id`) it cannot be updated.
        * `displayname` - The displayname of the new user. If not provided, will default to
                          the value of `mxid_localpart`.
        * `emails` - A list of emails for the new user. If not provided, will
@@ -190,4 +194,4 @@ A custom mapping provider must specify the following methods:
 
 Synapse has a built-in SAML mapping provider if a custom provider isn't
 specified in the config. It is located at
-[`synapse.handlers.saml_handler.DefaultSamlMappingProvider`](../synapse/handlers/saml_handler.py).
+[`synapse.handlers.saml.DefaultSamlMappingProvider`](https://github.com/matrix-org/synapse/blob/develop/synapse/handlers/saml.py).
diff --git a/docs/systemd-with-workers/README.md b/docs/systemd-with-workers/README.md
index cfa36be7b4..b160d93528 100644
--- a/docs/systemd-with-workers/README.md
+++ b/docs/systemd-with-workers/README.md
@@ -6,16 +6,20 @@ well as a `matrix-synapse-worker@` service template for any workers you
 require. Additionally, to group the required services, it sets up a
 `matrix-synapse.target`.
 
-See the folder [system](system) for the systemd unit files.
+See the folder [system](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/system/)
+for the systemd unit files.
 
-The folder [workers](workers) contains an example configuration for the
-`federation_reader` worker.
+The folder [workers](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/workers/)
+contains an example configuration for the `federation_reader` worker.
 
 ## Synapse configuration files
 
-See [workers.md](../workers.md) for information on how to set up the
-configuration files and reverse-proxy correctly. You can find an example worker
-config in the [workers](workers) folder.
+See [the worker documentation](../workers.md) for information on how to set up the
+configuration files and reverse-proxy correctly.
+Below is a sample `federation_reader` worker configuration file.
+```yaml
+{{#include workers/federation_reader.yaml}}
+```
 
 Systemd manages daemonization itself, so ensure that none of the configuration
 files set either `daemonize` or `worker_daemonize`.
@@ -29,8 +33,8 @@ There is no need for a separate configuration file for the master process.
 ## Set up
 
 1. Adjust synapse configuration files as above.
-1. Copy the `*.service` and `*.target` files in [system](system) to
-`/etc/systemd/system`.
+1. Copy the `*.service` and `*.target` files in [system](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/system/)
+to `/etc/systemd/system`.
 1. Run `systemctl daemon-reload` to tell systemd to load the new unit files.
 1. Run `systemctl enable matrix-synapse.service`. This will configure the
 synapse master process to be started as part of the `matrix-synapse.target`
@@ -65,3 +69,33 @@ systemctl restart matrix-synapse-worker@federation_reader.service
 systemctl enable matrix-synapse-worker@federation_writer.service
 systemctl restart matrix-synapse.target
 ```
+
+## Hardening
+
+**Optional:** If further hardening is desired, the file
+`override-hardened.conf` may be copied from
+[contrib/systemd/override-hardened.conf](https://github.com/matrix-org/synapse/tree/develop/contrib/systemd/)
+in this repository to the location
+`/etc/systemd/system/matrix-synapse.service.d/override-hardened.conf` (the
+directory may have to be created). It enables certain sandboxing features in
+systemd to further secure the synapse service. You may read the comments to
+understand what the override file is doing. The same file will need to be copied to
+`/etc/systemd/system/matrix-synapse-worker@.service.d/override-hardened-worker.conf`
+(this directory may also have to be created) in order to apply the same
+hardening options to any worker processes.
+
+Once these files have been copied to their appropriate locations, simply reload
+systemd's manager config files and restart all Synapse services to apply the hardening options. They will automatically
+be applied at every restart as long as the override files are present at the
+specified locations.
+
+```sh
+systemctl daemon-reload
+
+# Restart services
+systemctl restart matrix-synapse.target
+```
+
+In order to see their effect, you may run `systemd-analyze security
+matrix-synapse.service` before and after applying the hardening options to see
+the changes being applied at a glance.
diff --git a/docs/templates.md b/docs/templates.md
new file mode 100644
index 0000000000..a240f58b54
--- /dev/null
+++ b/docs/templates.md
@@ -0,0 +1,239 @@
+# Templates
+
+Synapse uses parametrised templates to generate the content of emails it sends and
+webpages it shows to users.
+
+By default, Synapse will use the templates listed [here](https://github.com/matrix-org/synapse/tree/master/synapse/res/templates).
+Server admins can configure an additional directory for Synapse to look for templates
+in, allowing them to specify custom templates:
+
+```yaml
+templates:
+  custom_templates_directory: /path/to/custom/templates/
+```
+
+If this setting is not set, or the files named below are not found within the directory,
+default templates from within the Synapse package will be used.
+
+Templates that are given variables when being rendered are rendered using [Jinja 2](https://jinja.palletsprojects.com/en/2.11.x/).
+Templates rendered by Jinja 2 can also access two functions on top of the functions
+already available as part of Jinja 2:
+
+```python
+format_ts(value: int, format: str) -> str
+```
+
+Formats a timestamp in milliseconds.
+
+Example: `reason.last_sent_ts|format_ts("%c")`
+
+```python
+mxc_to_http(value: str, width: int, height: int, resize_method: str = "crop") -> str
+```
+
+Turns a `mxc://` URL for media content into an HTTP(S) one using the homeserver's
+`public_baseurl` configuration setting as the URL's base.
+
+Example: `message.sender_avatar_url|mxc_to_http(32,32)`
+
+
+## Email templates
+
+Below are the templates Synapse will look for when generating the content of an email:
+
+* `notif_mail.html` and `notif_mail.txt`: The contents of email notifications of missed
+  events.
+  When rendering, this template is given the following variables:
+    * `user_display_name`: the display name for the user receiving the notification
+    * `unsubscribe_link`: the link users can click to unsubscribe from email notifications
+    * `summary_text`: a summary of the notification(s). The text used can be customised
+      by configuring the various settings in the `email.subjects` section of the
+      configuration file.
+    * `rooms`: a list of rooms containing events to include in the email. Each element is
+      an object with the following attributes:
+        * `title`: a human-readable name for the room
+        * `hash`: a hash of the ID of the room
+        * `invite`: a boolean, which is `True` if the room is an invite the user hasn't
+          accepted yet, `False` otherwise
+        * `notifs`: a list of events, or an empty list if `invite` is `True`. Each element
+          is an object with the following attributes:
+            * `link`: a `matrix.to` link to the event
+            * `ts`: the time in milliseconds at which the event was received
+            * `messages`: a list of messages containing one message before the event, the
+              message in the event, and one message after the event. Each element is an
+              object with the following attributes:
+                * `event_type`: the type of the event
+                * `is_historical`: a boolean, which is `False` if the message is the one
+                  that triggered the notification, `True` otherwise
+                * `id`: the ID of the event
+                * `ts`: the time in milliseconds at which the event was sent
+                * `sender_name`: the display name for the event's sender
+                * `sender_avatar_url`: the avatar URL (as a `mxc://` URL) for the event's
+                  sender
+                * `sender_hash`: a hash of the user ID of the sender
+        * `link`: a `matrix.to` link to the room
+    * `reason`: information on the event that triggered the email to be sent. It's an
+      object with the following attributes:
+        * `room_id`: the ID of the room the event was sent in
+        * `room_name`: a human-readable name for the room the event was sent in
+        * `now`: the current time in milliseconds
+        * `received_at`: the time in milliseconds at which the event was received
+        * `delay_before_mail_ms`: the amount of time in milliseconds Synapse always waits
+          before ever emailing about a notification (to give the user a chance to respond
+          to other push or notice the window)
+        * `last_sent_ts`: the time in milliseconds at which a notification was last sent
+          for an event in this room
+        * `throttle_ms`: the minimum amount of time in milliseconds between two
+          notifications can be sent for this room
+* `password_reset.html` and `password_reset.txt`: The contents of password reset emails
+  sent by the homeserver.
+  When rendering, these templates are given a `link` variable which contains the link the
+  user must click in order to reset their password.
+* `registration.html` and `registration.txt`: The contents of address verification emails
+  sent during registration.
+  When rendering, these templates are given a `link` variable which contains the link the
+  user must click in order to validate their email address.
+* `add_threepid.html` and `add_threepid.txt`: The contents of address verification emails
+  sent when an address is added to a Matrix account.
+  When rendering, these templates are given a `link` variable which contains the link the
+  user must click in order to validate their email address.
+
+
+## HTML page templates for registration and password reset
+
+Below are the templates Synapse will look for when generating pages related to
+registration and password reset:
+
+* `password_reset_confirmation.html`: An HTML page that a user will see when they follow
+  the link in the password reset email. The user will be asked to confirm the action
+  before their password is reset.
+  When rendering, this template is given the following variables:
+    * `sid`: the session ID for the password reset
+    * `token`: the token for the password reset
+    * `client_secret`: the client secret for the password reset
+* `password_reset_success.html` and `password_reset_failure.html`: HTML pages for success
+  and failure that a user will see when they confirm the password reset flow using the
+  page above.
+  When rendering, `password_reset_success.html` is given no variable, and
+  `password_reset_failure.html` is given a `failure_reason`, which contains the reason
+  for the password reset failure. 
+* `registration_success.html` and `registration_failure.html`: HTML pages for success and
+  failure that a user will see when they follow the link in an address verification email
+  sent during registration.
+  When rendering, `registration_success.html` is given no variable, and
+  `registration_failure.html` is given a `failure_reason`, which contains the reason
+  for the registration failure.
+* `add_threepid_success.html` and `add_threepid_failure.html`: HTML pages for success and
+  failure that a user will see when they follow the link in an address verification email
+  sent when an address is added to a Matrix account.
+  When rendering, `add_threepid_success.html` is given no variable, and
+  `add_threepid_failure.html` is given a `failure_reason`, which contains the reason
+  for the registration failure.
+
+
+## HTML page templates for Single Sign-On (SSO)
+
+Below are the templates Synapse will look for when generating pages related to SSO:
+
+* `sso_login_idp_picker.html`: HTML page to prompt the user to choose an
+  Identity Provider during login.
+  This is only used if multiple SSO Identity Providers are configured.
+  When rendering, this template is given the following variables:
+    * `redirect_url`: the URL that the user will be redirected to after
+      login.
+    * `server_name`: the homeserver's name.
+    * `providers`: a list of available Identity Providers. Each element is
+      an object with the following attributes:
+        * `idp_id`: unique identifier for the IdP
+        * `idp_name`: user-facing name for the IdP
+        * `idp_icon`: if specified in the IdP config, an MXC URI for an icon
+             for the IdP
+        * `idp_brand`: if specified in the IdP config, a textual identifier
+             for the brand of the IdP
+  The rendered HTML page should contain a form which submits its results
+  back as a GET request, with the following query parameters:
+    * `redirectUrl`: the client redirect URI (ie, the `redirect_url` passed
+      to the template)
+    * `idp`: the 'idp_id' of the chosen IDP.
+* `sso_auth_account_details.html`: HTML page to prompt new users to enter a
+  userid and confirm other details. This is only shown if the
+  SSO implementation (with any `user_mapping_provider`) does not return
+  a localpart.
+  When rendering, this template is given the following variables:
+    * `server_name`: the homeserver's name.
+    * `idp`: details of the SSO Identity Provider that the user logged in
+      with: an object with the following attributes:
+        * `idp_id`: unique identifier for the IdP
+        * `idp_name`: user-facing name for the IdP
+        * `idp_icon`: if specified in the IdP config, an MXC URI for an icon
+             for the IdP
+        * `idp_brand`: if specified in the IdP config, a textual identifier
+             for the brand of the IdP
+    * `user_attributes`: an object containing details about the user that
+      we received from the IdP. May have the following attributes:
+        * display_name: the user's display_name
+        * emails: a list of email addresses
+  The template should render a form which submits the following fields:
+    * `username`: the localpart of the user's chosen user id
+* `sso_new_user_consent.html`: HTML page allowing the user to consent to the
+  server's terms and conditions. This is only shown for new users, and only if
+  `user_consent.require_at_registration` is set.
+  When rendering, this template is given the following variables:
+    * `server_name`: the homeserver's name.
+    * `user_id`: the user's matrix proposed ID.
+    * `user_profile.display_name`: the user's proposed display name, if any.
+    * consent_version: the version of the terms that the user will be
+      shown
+    * `terms_url`: a link to the page showing the terms.
+  The template should render a form which submits the following fields:
+    * `accepted_version`: the version of the terms accepted by the user
+      (ie, 'consent_version' from the input variables).
+* `sso_redirect_confirm.html`: HTML page for a confirmation step before redirecting back
+  to the client with the login token.
+  When rendering, this template is given the following variables:
+    * `redirect_url`: the URL the user is about to be redirected to.
+    * `display_url`: the same as `redirect_url`, but with the query
+                   parameters stripped. The intention is to have a
+                   human-readable URL to show to users, not to use it as
+                   the final address to redirect to.
+    * `server_name`: the homeserver's name.
+    * `new_user`: a boolean indicating whether this is the user's first time
+         logging in.
+    * `user_id`: the user's matrix ID.
+    * `user_profile.avatar_url`: an MXC URI for the user's avatar, if any.
+          `None` if the user has not set an avatar.
+    * `user_profile.display_name`: the user's display name. `None` if the user
+          has not set a display name.
+* `sso_auth_confirm.html`: HTML page which notifies the user that they are authenticating
+  to confirm an operation on their account during the user interactive authentication
+  process.
+  When rendering, this template is given the following variables:
+    * `redirect_url`: the URL the user is about to be redirected to.
+    * `description`: the operation which the user is being asked to confirm
+    * `idp`: details of the Identity Provider that we will use to confirm
+      the user's identity: an object with the following attributes:
+        * `idp_id`: unique identifier for the IdP
+        * `idp_name`: user-facing name for the IdP
+        * `idp_icon`: if specified in the IdP config, an MXC URI for an icon
+             for the IdP
+        * `idp_brand`: if specified in the IdP config, a textual identifier
+             for the brand of the IdP
+* `sso_auth_success.html`: HTML page shown after a successful user interactive
+  authentication session.
+  Note that this page must include the JavaScript which notifies of a successful
+  authentication (see https://matrix.org/docs/spec/client_server/r0.6.0#fallback).
+  This template has no additional variables.
+* `sso_auth_bad_user.html`: HTML page shown after a user-interactive authentication
+  session which does not map correctly onto the expected user.
+  When rendering, this template is given the following variables:
+    * `server_name`: the homeserver's name.
+    * `user_id_to_verify`: the MXID of the user that we are trying to
+      validate.
+* `sso_account_deactivated.html`: HTML page shown during single sign-on if a deactivated
+  user (according to Synapse's database) attempts to login.
+  This template has no additional variables.
+* `sso_error.html`: HTML page to display to users if something goes wrong during the
+  OpenID Connect authentication process.
+  When rendering, this template is given two variables:
+    * `error`: the technical name of the error
+    * `error_description`: a human-readable message for the error
diff --git a/docs/turn-howto.md b/docs/turn-howto.md
index 41738bbe69..6433446c2a 100644
--- a/docs/turn-howto.md
+++ b/docs/turn-howto.md
@@ -4,7 +4,7 @@ This document explains how to enable VoIP relaying on your Home Server with
 TURN.
 
 The synapse Matrix Home Server supports integration with TURN server via the
-[TURN server REST API](<http://tools.ietf.org/html/draft-uberti-behave-turn-rest-00>). This
+[TURN server REST API](<https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00>). This
 allows the Home Server to generate credentials that are valid for use on the
 TURN server through the use of a secret shared between the Home Server and the
 TURN server.
diff --git a/docs/upgrade.md b/docs/upgrade.md
new file mode 100644
index 0000000000..e5d386b02f
--- /dev/null
+++ b/docs/upgrade.md
@@ -0,0 +1,1443 @@
+# Upgrading Synapse
+
+Before upgrading check if any special steps are required to upgrade from
+the version you currently have installed to the current version of
+Synapse. The extra instructions that may be required are listed later in
+this document.
+
+-   Check that your versions of Python and PostgreSQL are still
+    supported.
+
+    Synapse follows upstream lifecycles for [Python](https://endoflife.date/python) and
+    [PostgreSQL](https://endoflife.date/postgresql), and removes support for versions
+    which are no longer maintained.
+
+    The website <https://endoflife.date> also offers convenient
+    summaries.
+
+-   If Synapse was installed using [prebuilt
+    packages](setup/installation.md#prebuilt-packages), you will need to follow the
+    normal process for upgrading those packages.
+
+-   If Synapse was installed from source, then:
+
+    1.  Activate the virtualenv before upgrading. For example, if
+        Synapse is installed in a virtualenv in `~/synapse/env` then
+        run:
+
+        ```bash
+        source ~/synapse/env/bin/activate
+        ```
+
+    2.  If Synapse was installed using pip then upgrade to the latest
+        version by running:
+
+        ```bash
+        pip install --upgrade matrix-synapse
+        ```
+
+        If Synapse was installed using git then upgrade to the latest
+        version by running:
+
+        ```bash
+        git pull
+        pip install --upgrade .
+        ```
+
+    3.  Restart Synapse:
+
+        ```bash
+        ./synctl restart
+        ```
+
+To check whether your update was successful, you can check the running
+server version with:
+
+```bash
+# you may need to replace 'localhost:8008' if synapse is not configured
+# to listen on port 8008.
+
+curl http://localhost:8008/_synapse/admin/v1/server_version
+```
+
+## Rolling back to older versions
+
+Rolling back to previous releases can be difficult, due to database
+schema changes between releases. Where we have been able to test the
+rollback process, this will be noted below.
+
+In general, you will need to undo any changes made during the upgrade
+process, for example:
+
+-   pip:
+
+    ```bash
+    source env/bin/activate
+    # replace `1.3.0` accordingly:
+    pip install matrix-synapse==1.3.0
+    ```
+
+-   Debian:
+
+    ```bash
+    # replace `1.3.0` and `stretch` accordingly:
+    wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
+    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
+    ```
+
+
+# Upgrading to v1.41.0
+
+## Add support for routing outbound HTTP requests via a proxy for federation
+
+Since Synapse 1.6.0 (2019-11-26) you can set a proxy for outbound HTTP requests via
+http_proxy/https_proxy environment variables. This proxy was set for:
+- push
+- url previews
+- phone-home stats
+- recaptcha validation
+- CAS auth validation
+- OpenID Connect
+- Federation (checking public key revocation)
+
+In this version we have added support for outbound requests for:
+- Outbound federation
+- Downloading remote media
+- Fetching public keys of other servers
+
+These requests use the same proxy configuration. If you have a proxy configuration we
+recommend to verify the configuration. It may be necessary to adjust the `no_proxy`
+environment variable.
+
+See [using a forward proxy with Synapse documentation](setup/forward_proxy.md) for
+details.
+
+## Deprecation of `template_dir`
+
+The `template_dir` settings in the `sso`, `account_validity` and `email` sections of the
+configuration file are now deprecated. Server admins should use the new
+`templates.custom_template_directory` setting in the configuration file and use one single
+custom template directory for all aforementioned features. Template file names remain
+unchanged. See [the related documentation](https://matrix-org.github.io/synapse/latest/templates.html)
+for more information and examples.
+
+We plan to remove support for these settings in October 2021.
+
+## `/_synapse/admin/v1/users/{userId}/media` must be handled by media workers
+
+The [media repository worker documentation](https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository)
+has been updated to reflect that calls to `/_synapse/admin/v1/users/{userId}/media`
+must now be handled by media repository workers. This is due to the new `DELETE` method
+of this endpoint modifying the media store.
+
+# Upgrading to v1.39.0
+
+## Deprecation of the current third-party rules module interface
+
+The current third-party rules module interface is deprecated in favour of the new generic
+modules system introduced in Synapse v1.37.0. Authors of third-party rules modules can refer
+to [this documentation](modules.md#porting-an-existing-module-that-uses-the-old-interface)
+to update their modules. Synapse administrators can refer to [this documentation](modules.md#using-modules)
+to update their configuration once the modules they are using have been updated.
+
+We plan to remove support for the current third-party rules interface in September 2021.
+
+
+# Upgrading to v1.38.0
+
+## Re-indexing of `events` table on Postgres databases
+
+This release includes a database schema update which requires re-indexing one of
+the larger tables in the database, `events`. This could result in increased
+disk I/O for several hours or days after upgrading while the migration
+completes. Furthermore, because we have to keep the old indexes until the new
+indexes are ready, it could result in a significant, temporary, increase in
+disk space.
+
+To get a rough idea of the disk space required, check the current size of one
+of the indexes. For example, from a `psql` shell, run the following sql:
+
+```sql
+SELECT pg_size_pretty(pg_relation_size('events_order_room'));
+```
+
+We need to rebuild **four** indexes, so you will need to multiply this result
+by four to give an estimate of the disk space required. For example, on one
+particular server:
+
+```
+synapse=# select pg_size_pretty(pg_relation_size('events_order_room'));
+ pg_size_pretty
+----------------
+ 288 MB
+(1 row)
+```
+
+On this server, it would be wise to ensure that at least 1152MB are free.
+
+The additional disk space will be freed once the migration completes.
+
+SQLite databases are unaffected by this change.
+
+
+# Upgrading to v1.37.0
+
+## Deprecation of the current spam checker interface
+
+The current spam checker interface is deprecated in favour of a new generic modules system.
+Authors of spam checker modules can refer to [this
+documentation](modules.md#porting-an-existing-module-that-uses-the-old-interface)
+to update their modules. Synapse administrators can refer to [this
+documentation](modules.md#using-modules)
+to update their configuration once the modules they are using have been updated.
+
+We plan to remove support for the current spam checker interface in August 2021.
+
+More module interfaces will be ported over to this new generic system in future versions
+of Synapse.
+
+
+# Upgrading to v1.34.0
+
+## `room_invite_state_types` configuration setting
+
+The `room_invite_state_types` configuration setting has been deprecated
+and replaced with `room_prejoin_state`. See the [sample configuration
+file](https://github.com/matrix-org/synapse/blob/v1.34.0/docs/sample_config.yaml#L1515).
+
+If you have set `room_invite_state_types` to the default value you
+should simply remove it from your configuration file. The default value
+used to be:
+
+```yaml
+room_invite_state_types:
+   - "m.room.join_rules"
+   - "m.room.canonical_alias"
+   - "m.room.avatar"
+   - "m.room.encryption"
+   - "m.room.name"
+```
+
+If you have customised this value, you should remove
+`room_invite_state_types` and configure `room_prejoin_state` instead.
+
+# Upgrading to v1.33.0
+
+## Account Validity HTML templates can now display a user's expiration date
+
+This may affect you if you have enabled the account validity feature,
+and have made use of a custom HTML template specified by the
+`account_validity.template_dir` or
+`account_validity.account_renewed_html_path` Synapse config options.
+
+The template can now accept an `expiration_ts` variable, which
+represents the unix timestamp in milliseconds for the future date of
+which their account has been renewed until. See the [default
+template](https://github.com/matrix-org/synapse/blob/release-v1.33.0/synapse/res/templates/account_renewed.html)
+for an example of usage.
+
+ALso note that a new HTML template, `account_previously_renewed.html`,
+has been added. This is is shown to users when they attempt to renew
+their account with a valid renewal token that has already been used
+before. The default template contents can been found
+[here](https://github.com/matrix-org/synapse/blob/release-v1.33.0/synapse/res/templates/account_previously_renewed.html),
+and can also accept an `expiration_ts` variable. This template replaces
+the error message users would previously see upon attempting to use a
+valid renewal token more than once.
+
+# Upgrading to v1.32.0
+
+## Regression causing connected Prometheus instances to become overwhelmed
+
+This release introduces [a
+regression](https://github.com/matrix-org/synapse/issues/9853) that can
+overwhelm connected Prometheus instances. This issue is not present in
+Synapse v1.32.0rc1.
+
+If you have been affected, please downgrade to 1.31.0. You then may need
+to remove excess writeahead logs in order for Prometheus to recover.
+Instructions for doing so are provided
+[here](https://github.com/matrix-org/synapse/pull/9854#issuecomment-823472183).
+
+## Dropping support for old Python, Postgres and SQLite versions
+
+In line with our [deprecation policy](deprecation_policy.md),
+we've dropped support for Python 3.5 and PostgreSQL 9.5, as they are no
+longer supported upstream.
+
+This release of Synapse requires Python 3.6+ and PostgresSQL 9.6+ or
+SQLite 3.22+.
+
+## Removal of old List Accounts Admin API
+
+The deprecated v1 "list accounts" admin API
+(`GET /_synapse/admin/v1/users/<user_id>`) has been removed in this
+version.
+
+The [v2 list accounts API](admin_api/user_admin_api.md#list-accounts)
+has been available since Synapse 1.7.0 (2019-12-13), and is accessible
+under `GET /_synapse/admin/v2/users`.
+
+The deprecation of the old endpoint was announced with Synapse 1.28.0
+(released on 2021-02-25).
+
+## Application Services must use type `m.login.application_service` when registering users
+
+In compliance with the [Application Service
+spec](https://matrix.org/docs/spec/application_service/r0.1.2#server-admin-style-permissions),
+Application Services are now required to use the
+`m.login.application_service` type when registering users via the
+`/_matrix/client/r0/register` endpoint. This behaviour was deprecated in
+Synapse v1.30.0.
+
+Please ensure your Application Services are up to date.
+
+# Upgrading to v1.29.0
+
+## Requirement for X-Forwarded-Proto header
+
+When using Synapse with a reverse proxy (in particular, when using the
+[x_forwarded]{.title-ref} option on an HTTP listener), Synapse now
+expects to receive an [X-Forwarded-Proto]{.title-ref} header on incoming
+HTTP requests. If it is not set, Synapse will log a warning on each
+received request.
+
+To avoid the warning, administrators using a reverse proxy should ensure
+that the reverse proxy sets [X-Forwarded-Proto]{.title-ref} header to
+[https]{.title-ref} or [http]{.title-ref} to indicate the protocol used
+by the client.
+
+Synapse also requires the [Host]{.title-ref} header to be preserved.
+
+See the [reverse proxy documentation](reverse_proxy.md), where the
+example configurations have been updated to show how to set these
+headers.
+
+(Users of [Caddy](https://caddyserver.com/) are unaffected, since we
+believe it sets [X-Forwarded-Proto]{.title-ref} by default.)
+
+# Upgrading to v1.27.0
+
+## Changes to callback URI for OAuth2 / OpenID Connect and SAML2
+
+This version changes the URI used for callbacks from OAuth2 and SAML2
+identity providers:
+
+-   If your server is configured for single sign-on via an OpenID
+    Connect or OAuth2 identity provider, you will need to add
+    `[synapse public baseurl]/_synapse/client/oidc/callback` to the list
+    of permitted "redirect URIs" at the identity provider.
+
+    See the [OpenID docs](openid.md) for more information on setting
+    up OpenID Connect.
+
+-   If your server is configured for single sign-on via a SAML2 identity
+    provider, you will need to add
+    `[synapse public baseurl]/_synapse/client/saml2/authn_response` as a
+    permitted "ACS location" (also known as "allowed callback URLs")
+    at the identity provider.
+
+    The "Issuer" in the "AuthnRequest" to the SAML2 identity
+    provider is also updated to
+    `[synapse public baseurl]/_synapse/client/saml2/metadata.xml`. If
+    your SAML2 identity provider uses this property to validate or
+    otherwise identify Synapse, its configuration will need to be
+    updated to use the new URL. Alternatively you could create a new,
+    separate "EntityDescriptor" in your SAML2 identity provider with
+    the new URLs and leave the URLs in the existing "EntityDescriptor"
+    as they were.
+
+## Changes to HTML templates
+
+The HTML templates for SSO and email notifications now have [Jinja2's
+autoescape](https://jinja.palletsprojects.com/en/2.11.x/api/#autoescaping)
+enabled for files ending in `.html`, `.htm`, and `.xml`. If you have
+customised these templates and see issues when viewing them you might
+need to update them. It is expected that most configurations will need
+no changes.
+
+If you have customised the templates *names* for these templates, it is
+recommended to verify they end in `.html` to ensure autoescape is
+enabled.
+
+The above applies to the following templates:
+
+-   `add_threepid.html`
+-   `add_threepid_failure.html`
+-   `add_threepid_success.html`
+-   `notice_expiry.html`
+-   `notice_expiry.html`
+-   `notif_mail.html` (which, by default, includes `room.html` and
+    `notif.html`)
+-   `password_reset.html`
+-   `password_reset_confirmation.html`
+-   `password_reset_failure.html`
+-   `password_reset_success.html`
+-   `registration.html`
+-   `registration_failure.html`
+-   `registration_success.html`
+-   `sso_account_deactivated.html`
+-   `sso_auth_bad_user.html`
+-   `sso_auth_confirm.html`
+-   `sso_auth_success.html`
+-   `sso_error.html`
+-   `sso_login_idp_picker.html`
+-   `sso_redirect_confirm.html`
+
+# Upgrading to v1.26.0
+
+## Rolling back to v1.25.0 after a failed upgrade
+
+v1.26.0 includes a lot of large changes. If something problematic
+occurs, you may want to roll-back to a previous version of Synapse.
+Because v1.26.0 also includes a new database schema version, reverting
+that version is also required alongside the generic rollback
+instructions mentioned above. In short, to roll back to v1.25.0 you need
+to:
+
+1.  Stop the server
+
+2.  Decrease the schema version in the database:
+
+    ```sql
+    UPDATE schema_version SET version = 58;
+    ```
+
+3.  Delete the ignored users & chain cover data:
+
+    ```sql
+    DROP TABLE IF EXISTS ignored_users;
+    UPDATE rooms SET has_auth_chain_index = false;
+    ```
+
+    For PostgreSQL run:
+
+    ```sql
+    TRUNCATE event_auth_chain_links;
+    TRUNCATE event_auth_chains;
+    ```
+
+    For SQLite run:
+
+    ```sql
+    DELETE FROM event_auth_chain_links;
+    DELETE FROM event_auth_chains;
+    ```
+
+4.  Mark the deltas as not run (so they will re-run on upgrade).
+
+    ```sql
+    DELETE FROM applied_schema_deltas WHERE version = 59 AND file = "59/01ignored_user.py";
+    DELETE FROM applied_schema_deltas WHERE version = 59 AND file = "59/06chain_cover_index.sql";
+    ```
+
+5.  Downgrade Synapse by following the instructions for your
+    installation method in the "Rolling back to older versions"
+    section above.
+
+# Upgrading to v1.25.0
+
+## Last release supporting Python 3.5
+
+This is the last release of Synapse which guarantees support with Python
+3.5, which passed its upstream End of Life date several months ago.
+
+We will attempt to maintain support through March 2021, but without
+guarantees.
+
+In the future, Synapse will follow upstream schedules for ending support
+of older versions of Python and PostgreSQL. Please upgrade to at least
+Python 3.6 and PostgreSQL 9.6 as soon as possible.
+
+## Blacklisting IP ranges
+
+Synapse v1.25.0 includes new settings, `ip_range_blacklist` and
+`ip_range_whitelist`, for controlling outgoing requests from Synapse for
+federation, identity servers, push, and for checking key validity for
+third-party invite events. The previous setting,
+`federation_ip_range_blacklist`, is deprecated. The new
+`ip_range_blacklist` defaults to private IP ranges if it is not defined.
+
+If you have never customised `federation_ip_range_blacklist` it is
+recommended that you remove that setting.
+
+If you have customised `federation_ip_range_blacklist` you should update
+the setting name to `ip_range_blacklist`.
+
+If you have a custom push server that is reached via private IP space
+you may need to customise `ip_range_blacklist` or `ip_range_whitelist`.
+
+# Upgrading to v1.24.0
+
+## Custom OpenID Connect mapping provider breaking change
+
+This release allows the OpenID Connect mapping provider to perform
+normalisation of the localpart of the Matrix ID. This allows for the
+mapping provider to specify different algorithms, instead of the
+[default
+way](<https://matrix.org/docs/spec/appendices#mapping-from-other-character-sets>).
+
+If your Synapse configuration uses a custom mapping provider
+([oidc_config.user_mapping_provider.module]{.title-ref} is specified and
+not equal to
+[synapse.handlers.oidc_handler.JinjaOidcMappingProvider]{.title-ref})
+then you *must* ensure that [map_user_attributes]{.title-ref} of the
+mapping provider performs some normalisation of the
+[localpart]{.title-ref} returned. To match previous behaviour you can
+use the [map_username_to_mxid_localpart]{.title-ref} function provided
+by Synapse. An example is shown below:
+
+```python
+from synapse.types import map_username_to_mxid_localpart
+
+class MyMappingProvider:
+    def map_user_attributes(self, userinfo, token):
+        # ... your custom logic ...
+        sso_user_id = ...
+        localpart = map_username_to_mxid_localpart(sso_user_id)
+
+        return {"localpart": localpart}
+```
+
+## Removal historical Synapse Admin API
+
+Historically, the Synapse Admin API has been accessible under:
+
+-   `/_matrix/client/api/v1/admin`
+-   `/_matrix/client/unstable/admin`
+-   `/_matrix/client/r0/admin`
+-   `/_synapse/admin/v1`
+
+The endpoints with `/_matrix/client/*` prefixes have been removed as of
+v1.24.0. The Admin API is now only accessible under:
+
+-   `/_synapse/admin/v1`
+
+The only exception is the [/admin/whois]{.title-ref} endpoint, which is
+[also available via the client-server
+API](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid).
+
+The deprecation of the old endpoints was announced with Synapse 1.20.0
+(released on 2020-09-22) and makes it easier for homeserver admins to
+lock down external access to the Admin API endpoints.
+
+# Upgrading to v1.23.0
+
+## Structured logging configuration breaking changes
+
+This release deprecates use of the `structured: true` logging
+configuration for structured logging. If your logging configuration
+contains `structured: true` then it should be modified based on the
+[structured logging documentation](structured_logging.md).
+
+The `structured` and `drains` logging options are now deprecated and
+should be replaced by standard logging configuration of `handlers` and
+`formatters`.
+
+A future will release of Synapse will make using `structured: true` an
+error.
+
+# Upgrading to v1.22.0
+
+## ThirdPartyEventRules breaking changes
+
+This release introduces a backwards-incompatible change to modules
+making use of `ThirdPartyEventRules` in Synapse. If you make use of a
+module defined under the `third_party_event_rules` config option, please
+make sure it is updated to handle the below change:
+
+The `http_client` argument is no longer passed to modules as they are
+initialised. Instead, modules are expected to make use of the
+`http_client` property on the `ModuleApi` class. Modules are now passed
+a `module_api` argument during initialisation, which is an instance of
+`ModuleApi`. `ModuleApi` instances have a `http_client` property which
+acts the same as the `http_client` argument previously passed to
+`ThirdPartyEventRules` modules.
+
+# Upgrading to v1.21.0
+
+## Forwarding `/_synapse/client` through your reverse proxy
+
+The [reverse proxy documentation](reverse_proxy.md)
+has been updated to include reverse proxy directives for
+`/_synapse/client/*` endpoints. As the user password reset flow now uses
+endpoints under this prefix, **you must update your reverse proxy
+configurations for user password reset to work**.
+
+Additionally, note that the [Synapse worker documentation](workers.md) has been updated to
+
+:   state that the `/_synapse/client/password_reset/email/submit_token`
+    endpoint can be handled
+
+by all workers. If you make use of Synapse's worker feature, please
+update your reverse proxy configuration to reflect this change.
+
+## New HTML templates
+
+A new HTML template,
+[password_reset_confirmation.html](https://github.com/matrix-org/synapse/blob/develop/synapse/res/templates/password_reset_confirmation.html),
+has been added to the `synapse/res/templates` directory. If you are
+using a custom template directory, you may want to copy the template
+over and modify it.
+
+Note that as of v1.20.0, templates do not need to be included in custom
+template directories for Synapse to start. The default templates will be
+used if a custom template cannot be found.
+
+This page will appear to the user after clicking a password reset link
+that has been emailed to them.
+
+To complete password reset, the page must include a way to make a
+[POST]{.title-ref} request to
+`/_synapse/client/password_reset/{medium}/submit_token` with the query
+parameters from the original link, presented as a URL-encoded form. See
+the file itself for more details.
+
+## Updated Single Sign-on HTML Templates
+
+The `saml_error.html` template was removed from Synapse and replaced
+with the `sso_error.html` template. If your Synapse is configured to use
+SAML and a custom `sso_redirect_confirm_template_dir` configuration then
+any customisations of the `saml_error.html` template will need to be
+merged into the `sso_error.html` template. These templates are similar,
+but the parameters are slightly different:
+
+-   The `msg` parameter should be renamed to `error_description`.
+-   There is no longer a `code` parameter for the response code.
+-   A string `error` parameter is available that includes a short hint
+    of why a user is seeing the error page.
+
+# Upgrading to v1.18.0
+
+## Docker [-py3]{.title-ref} suffix will be removed in future versions
+
+From 10th August 2020, we will no longer publish Docker images with the
+[-py3]{.title-ref} tag suffix. The images tagged with the
+[-py3]{.title-ref} suffix have been identical to the non-suffixed tags
+since release 0.99.0, and the suffix is obsolete.
+
+On 10th August, we will remove the [latest-py3]{.title-ref} tag.
+Existing per-release tags (such as [v1.18.0-py3]{.title-ref}) will not
+be removed, but no new [-py3]{.title-ref} tags will be added.
+
+Scripts relying on the [-py3]{.title-ref} suffix will need to be
+updated.
+
+## Redis replication is now recommended in lieu of TCP replication
+
+When setting up worker processes, we now recommend the use of a Redis
+server for replication. **The old direct TCP connection method is
+deprecated and will be removed in a future release.** See
+[workers](workers.md) for more details.
+
+# Upgrading to v1.14.0
+
+This version includes a database update which is run as part of the
+upgrade, and which may take a couple of minutes in the case of a large
+server. Synapse will not respond to HTTP requests while this update is
+taking place.
+
+# Upgrading to v1.13.0
+
+## Incorrect database migration in old synapse versions
+
+A bug was introduced in Synapse 1.4.0 which could cause the room
+directory to be incomplete or empty if Synapse was upgraded directly
+from v1.2.1 or earlier, to versions between v1.4.0 and v1.12.x.
+
+This will *not* be a problem for Synapse installations which were:
+
+:   -   created at v1.4.0 or later,
+    -   upgraded via v1.3.x, or
+    -   upgraded straight from v1.2.1 or earlier to v1.13.0 or later.
+
+If completeness of the room directory is a concern, installations which
+are affected can be repaired as follows:
+
+1.  Run the following sql from a [psql]{.title-ref} or
+    [sqlite3]{.title-ref} console:
+
+    ```sql
+    INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
+       ('populate_stats_process_rooms', '{}', 'current_state_events_membership');
+
+    INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
+       ('populate_stats_process_users', '{}', 'populate_stats_process_rooms');
+    ```
+
+2.  Restart synapse.
+
+## New Single Sign-on HTML Templates
+
+New templates (`sso_auth_confirm.html`, `sso_auth_success.html`, and
+`sso_account_deactivated.html`) were added to Synapse. If your Synapse
+is configured to use SSO and a custom
+`sso_redirect_confirm_template_dir` configuration then these templates
+will need to be copied from
+[synapse/res/templates](synapse/res/templates) into that directory.
+
+## Synapse SSO Plugins Method Deprecation
+
+Plugins using the `complete_sso_login` method of
+`synapse.module_api.ModuleApi` should update to using the async/await
+version `complete_sso_login_async` which includes additional checks. The
+non-async version is considered deprecated.
+
+## Rolling back to v1.12.4 after a failed upgrade
+
+v1.13.0 includes a lot of large changes. If something problematic
+occurs, you may want to roll-back to a previous version of Synapse.
+Because v1.13.0 also includes a new database schema version, reverting
+that version is also required alongside the generic rollback
+instructions mentioned above. In short, to roll back to v1.12.4 you need
+to:
+
+1.  Stop the server
+
+2.  Decrease the schema version in the database:
+
+    ```sql
+    UPDATE schema_version SET version = 57;
+    ```
+
+3.  Downgrade Synapse by following the instructions for your
+    installation method in the "Rolling back to older versions"
+    section above.
+
+# Upgrading to v1.12.0
+
+This version includes a database update which is run as part of the
+upgrade, and which may take some time (several hours in the case of a
+large server). Synapse will not respond to HTTP requests while this
+update is taking place.
+
+This is only likely to be a problem in the case of a server which is
+participating in many rooms.
+
+0.  As with all upgrades, it is recommended that you have a recent
+    backup of your database which can be used for recovery in the event
+    of any problems.
+
+1.  As an initial check to see if you will be affected, you can try
+    running the following query from the [psql]{.title-ref} or
+    [sqlite3]{.title-ref} console. It is safe to run it while Synapse is
+    still running.
+
+    ```sql
+    SELECT MAX(q.v) FROM (
+      SELECT (
+        SELECT ej.json AS v
+        FROM state_events se INNER JOIN event_json ej USING (event_id)
+        WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key=''
+        LIMIT 1
+      ) FROM rooms WHERE rooms.room_version IS NULL
+    ) q;
+    ```
+
+    This query will take about the same amount of time as the upgrade
+    process: ie, if it takes 5 minutes, then it is likely that Synapse
+    will be unresponsive for 5 minutes during the upgrade.
+
+    If you consider an outage of this duration to be acceptable, no
+    further action is necessary and you can simply start Synapse 1.12.0.
+
+    If you would prefer to reduce the downtime, continue with the steps
+    below.
+
+2.  The easiest workaround for this issue is to manually create a new
+    index before upgrading. On PostgreSQL, his can be done as follows:
+
+    ```sql
+    CREATE INDEX CONCURRENTLY tmp_upgrade_1_12_0_index
+    ON state_events(room_id) WHERE type = 'm.room.create';
+    ```
+
+    The above query may take some time, but is also safe to run while
+    Synapse is running.
+
+    We assume that no SQLite users have databases large enough to be
+    affected. If you *are* affected, you can run a similar query,
+    omitting the `CONCURRENTLY` keyword. Note however that this
+    operation may in itself cause Synapse to stop running for some time.
+    Synapse admins are reminded that [SQLite is not recommended for use
+    outside a test environment](postgres.md).
+
+3.  Once the index has been created, the `SELECT` query in step 1 above
+    should complete quickly. It is therefore safe to upgrade to Synapse
+    1.12.0.
+
+4.  Once Synapse 1.12.0 has successfully started and is responding to
+    HTTP requests, the temporary index can be removed:
+
+    ```sql
+    DROP INDEX tmp_upgrade_1_12_0_index;
+    ```
+
+# Upgrading to v1.10.0
+
+Synapse will now log a warning on start up if used with a PostgreSQL
+database that has a non-recommended locale set.
+
+See [Postgres](postgres.md) for details.
+
+# Upgrading to v1.8.0
+
+Specifying a `log_file` config option will now cause Synapse to refuse
+to start, and should be replaced by with the `log_config` option.
+Support for the `log_file` option was removed in v1.3.0 and has since
+had no effect.
+
+# Upgrading to v1.7.0
+
+In an attempt to configure Synapse in a privacy preserving way, the
+default behaviours of `allow_public_rooms_without_auth` and
+`allow_public_rooms_over_federation` have been inverted. This means that
+by default, only authenticated users querying the Client/Server API will
+be able to query the room directory, and relatedly that the server will
+not share room directory information with other servers over federation.
+
+If your installation does not explicitly set these settings one way or
+the other and you want either setting to be `true` then it will
+necessary to update your homeserver configuration file accordingly.
+
+For more details on the surrounding context see our
+[explainer](https://matrix.org/blog/2019/11/09/avoiding-unwelcome-visitors-on-private-matrix-servers).
+
+# Upgrading to v1.5.0
+
+This release includes a database migration which may take several
+minutes to complete if there are a large number (more than a million or
+so) of entries in the `devices` table. This is only likely to a be a
+problem on very large installations.
+
+# Upgrading to v1.4.0
+
+## New custom templates
+
+If you have configured a custom template directory with the
+`email.template_dir` option, be aware that there are new templates
+regarding registration and threepid management (see below) that must be
+included.
+
+-   `registration.html` and `registration.txt`
+-   `registration_success.html` and `registration_failure.html`
+-   `add_threepid.html` and `add_threepid.txt`
+-   `add_threepid_failure.html` and `add_threepid_success.html`
+
+Synapse will expect these files to exist inside the configured template
+directory, and **will fail to start** if they are absent. To view the
+default templates, see
+[synapse/res/templates](https://github.com/matrix-org/synapse/tree/master/synapse/res/templates).
+
+## 3pid verification changes
+
+**Note: As of this release, users will be unable to add phone numbers or
+email addresses to their accounts, without changes to the Synapse
+configuration. This includes adding an email address during
+registration.**
+
+It is possible for a user to associate an email address or phone number
+with their account, for a number of reasons:
+
+-   for use when logging in, as an alternative to the user id.
+-   in the case of email, as an alternative contact to help with account
+    recovery.
+-   in the case of email, to receive notifications of missed messages.
+
+Before an email address or phone number can be added to a user's
+account, or before such an address is used to carry out a
+password-reset, Synapse must confirm the operation with the owner of the
+email address or phone number. It does this by sending an email or text
+giving the user a link or token to confirm receipt. This process is
+known as '3pid verification'. ('3pid', or 'threepid', stands for
+third-party identifier, and we use it to refer to external identifiers
+such as email addresses and phone numbers.)
+
+Previous versions of Synapse delegated the task of 3pid verification to
+an identity server by default. In most cases this server is `vector.im`
+or `matrix.org`.
+
+In Synapse 1.4.0, for security and privacy reasons, the homeserver will
+no longer delegate this task to an identity server by default. Instead,
+the server administrator will need to explicitly decide how they would
+like the verification messages to be sent.
+
+In the medium term, the `vector.im` and `matrix.org` identity servers
+will disable support for delegated 3pid verification entirely. However,
+in order to ease the transition, they will retain the capability for a
+limited period. Delegated email verification will be disabled on Monday
+2nd December 2019 (giving roughly 2 months notice). Disabling delegated
+SMS verification will follow some time after that once SMS verification
+support lands in Synapse.
+
+Once delegated 3pid verification support has been disabled in the
+`vector.im` and `matrix.org` identity servers, all Synapse versions that
+depend on those instances will be unable to verify email and phone
+numbers through them. There are no imminent plans to remove delegated
+3pid verification from Sydent generally. (Sydent is the identity server
+project that backs the `vector.im` and `matrix.org` instances).
+
+### Email
+
+Following upgrade, to continue verifying email (e.g. as part of the
+registration process), admins can either:-
+
+-   Configure Synapse to use an email server.
+-   Run or choose an identity server which allows delegated email
+    verification and delegate to it.
+
+#### Configure SMTP in Synapse
+
+To configure an SMTP server for Synapse, modify the configuration
+section headed `email`, and be sure to have at least the
+`smtp_host, smtp_port` and `notif_from` fields filled out.
+
+You may also need to set `smtp_user`, `smtp_pass`, and
+`require_transport_security`.
+
+See the [sample configuration file](usage/configuration/homeserver_sample_config.md)
+for more details on these settings.
+
+#### Delegate email to an identity server
+
+Some admins will wish to continue using email verification as part of
+the registration process, but will not immediately have an appropriate
+SMTP server at hand.
+
+To this end, we will continue to support email verification delegation
+via the `vector.im` and `matrix.org` identity servers for two months.
+Support for delegated email verification will be disabled on Monday 2nd
+December.
+
+The `account_threepid_delegates` dictionary defines whether the
+homeserver should delegate an external server (typically an [identity
+server](https://matrix.org/docs/spec/identity_service/r0.2.1)) to handle
+sending confirmation messages via email and SMS.
+
+So to delegate email verification, in `homeserver.yaml`, set
+`account_threepid_delegates.email` to the base URL of an identity
+server. For example:
+
+```yaml
+account_threepid_delegates:
+    email: https://example.com     # Delegate email sending to example.com
+```
+
+Note that `account_threepid_delegates.email` replaces the deprecated
+`email.trust_identity_server_for_password_resets`: if
+`email.trust_identity_server_for_password_resets` is set to `true`, and
+`account_threepid_delegates.email` is not set, then the first entry in
+`trusted_third_party_id_servers` will be used as the
+`account_threepid_delegate` for email. This is to ensure compatibility
+with existing Synapse installs that set up external server handling for
+these tasks before v1.4.0. If
+`email.trust_identity_server_for_password_resets` is `true` and no
+trusted identity server domains are configured, Synapse will report an
+error and refuse to start.
+
+If `email.trust_identity_server_for_password_resets` is `false` or
+absent and no `email` delegate is configured in
+`account_threepid_delegates`, then Synapse will send email verification
+messages itself, using the configured SMTP server (see above). that
+type.
+
+### Phone numbers
+
+Synapse does not support phone-number verification itself, so the only
+way to maintain the ability for users to add phone numbers to their
+accounts will be by continuing to delegate phone number verification to
+the `matrix.org` and `vector.im` identity servers (or another identity
+server that supports SMS sending).
+
+The `account_threepid_delegates` dictionary defines whether the
+homeserver should delegate an external server (typically an [identity
+server](https://matrix.org/docs/spec/identity_service/r0.2.1)) to handle
+sending confirmation messages via email and SMS.
+
+So to delegate phone number verification, in `homeserver.yaml`, set
+`account_threepid_delegates.msisdn` to the base URL of an identity
+server. For example:
+
+```yaml
+account_threepid_delegates:
+    msisdn: https://example.com     # Delegate sms sending to example.com
+```
+
+The `matrix.org` and `vector.im` identity servers will continue to
+support delegated phone number verification via SMS until such time as
+it is possible for admins to configure their servers to perform phone
+number verification directly. More details will follow in a future
+release.
+
+## Rolling back to v1.3.1
+
+If you encounter problems with v1.4.0, it should be possible to roll
+back to v1.3.1, subject to the following:
+
+-   The 'room statistics' engine was heavily reworked in this release
+    (see [#5971](https://github.com/matrix-org/synapse/pull/5971)),
+    including significant changes to the database schema, which are not
+    easily reverted. This will cause the room statistics engine to stop
+    updating when you downgrade.
+
+    The room statistics are essentially unused in v1.3.1 (in future
+    versions of Synapse, they will be used to populate the room
+    directory), so there should be no loss of functionality. However,
+    the statistics engine will write errors to the logs, which can be
+    avoided by setting the following in `homeserver.yaml`:
+
+    ```yaml
+    stats:
+      enabled: false
+    ```
+
+    Don't forget to re-enable it when you upgrade again, in preparation
+    for its use in the room directory!
+
+# Upgrading to v1.2.0
+
+Some counter metrics have been renamed, with the old names deprecated.
+See [the metrics
+documentation](metrics-howto.md#renaming-of-metrics--deprecation-of-old-names-in-12)
+for details.
+
+# Upgrading to v1.1.0
+
+Synapse v1.1.0 removes support for older Python and PostgreSQL versions,
+as outlined in [our deprecation
+notice](https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x).
+
+## Minimum Python Version
+
+Synapse v1.1.0 has a minimum Python requirement of Python 3.5. Python
+3.6 or Python 3.7 are recommended as they have improved internal string
+handling, significantly reducing memory usage.
+
+If you use current versions of the Matrix.org-distributed Debian
+packages or Docker images, action is not required.
+
+If you install Synapse in a Python virtual environment, please see
+"Upgrading to v0.34.0" for notes on setting up a new virtualenv under
+Python 3.
+
+## Minimum PostgreSQL Version
+
+If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5
+or above. Please see the [PostgreSQL
+documentation](https://www.postgresql.org/docs/11/upgrading.html) for
+more details on upgrading your database.
+
+# Upgrading to v1.0
+
+## Validation of TLS certificates
+
+Synapse v1.0 is the first release to enforce validation of TLS
+certificates for the federation API. It is therefore essential that your
+certificates are correctly configured. See the
+[FAQ](MSC1711_certificates_FAQ.md) for more information.
+
+Note, v1.0 installations will also no longer be able to federate with
+servers that have not correctly configured their certificates.
+
+In rare cases, it may be desirable to disable certificate checking: for
+example, it might be essential to be able to federate with a given
+legacy server in a closed federation. This can be done in one of two
+ways:-
+
+-   Configure the global switch `federation_verify_certificates` to
+    `false`.
+-   Configure a whitelist of server domains to trust via
+    `federation_certificate_verification_whitelist`.
+
+See the [sample configuration file](usage/configuration/homeserver_sample_config.md)
+for more details on these settings.
+
+## Email
+
+When a user requests a password reset, Synapse will send an email to the
+user to confirm the request.
+
+Previous versions of Synapse delegated the job of sending this email to
+an identity server. If the identity server was somehow malicious or
+became compromised, it would be theoretically possible to hijack an
+account through this means.
+
+Therefore, by default, Synapse v1.0 will send the confirmation email
+itself. If Synapse is not configured with an SMTP server, password reset
+via email will be disabled.
+
+To configure an SMTP server for Synapse, modify the configuration
+section headed `email`, and be sure to have at least the `smtp_host`,
+`smtp_port` and `notif_from` fields filled out. You may also need to set
+`smtp_user`, `smtp_pass`, and `require_transport_security`.
+
+If you are absolutely certain that you wish to continue using an
+identity server for password resets, set
+`trust_identity_server_for_password_resets` to `true`.
+
+See the [sample configuration file](usage/configuration/homeserver_sample_config.md)
+for more details on these settings.
+
+## New email templates
+
+Some new templates have been added to the default template directory for the purpose of
+the homeserver sending its own password reset emails. If you have configured a
+custom `template_dir` in your Synapse config, these files will need to be added.
+
+`password_reset.html` and `password_reset.txt` are HTML and plain text
+templates respectively that contain the contents of what will be emailed
+to the user upon attempting to reset their password via email.
+`password_reset_success.html` and `password_reset_failure.html` are HTML
+files that the content of which (assuming no redirect URL is set) will
+be shown to the user after they attempt to click the link in the email
+sent to them.
+
+# Upgrading to v0.99.0
+
+Please be aware that, before Synapse v1.0 is released around March 2019,
+you will need to replace any self-signed certificates with those
+verified by a root CA. Information on how to do so can be found at the
+ACME docs.
+
+For more information on configuring TLS certificates see the
+[FAQ](MSC1711_certificates_FAQ.md).
+
+# Upgrading to v0.34.0
+
+1.  This release is the first to fully support Python 3. Synapse will
+    now run on Python versions 3.5, or 3.6 (as well as 2.7). We
+    recommend switching to Python 3, as it has been shown to give
+    performance improvements.
+
+    For users who have installed Synapse into a virtualenv, we recommend
+    doing this by creating a new virtualenv. For example:
+
+        virtualenv -p python3 ~/synapse/env3
+        source ~/synapse/env3/bin/activate
+        pip install matrix-synapse
+
+    You can then start synapse as normal, having activated the new
+    virtualenv:
+
+        cd ~/synapse
+        source env3/bin/activate
+        synctl start
+
+    Users who have installed from distribution packages should see the
+    relevant package documentation. See below for notes on Debian
+    packages.
+
+    -   When upgrading to Python 3, you **must** make sure that your log
+        files are configured as UTF-8, by adding `encoding: utf8` to the
+        `RotatingFileHandler` configuration (if you have one) in your
+        `<server>.log.config` file. For example, if your `log.config`
+        file contains:
+
+            handlers:
+              file:
+                class: logging.handlers.RotatingFileHandler
+                formatter: precise
+                filename: homeserver.log
+                maxBytes: 104857600
+                backupCount: 10
+                filters: [context]
+              console:
+                class: logging.StreamHandler
+                formatter: precise
+                filters: [context]
+
+        Then you should update this to be:
+
+            handlers:
+              file:
+                class: logging.handlers.RotatingFileHandler
+                formatter: precise
+                filename: homeserver.log
+                maxBytes: 104857600
+                backupCount: 10
+                filters: [context]
+                encoding: utf8
+              console:
+                class: logging.StreamHandler
+                formatter: precise
+                filters: [context]
+
+        There is no need to revert this change if downgrading to
+        Python 2.
+
+    We are also making available Debian packages which will run Synapse
+    on Python 3. You can switch to these packages with
+    `apt-get install matrix-synapse-py3`, however, please read
+    [debian/NEWS](https://github.com/matrix-org/synapse/blob/release-v0.34.0/debian/NEWS)
+    before doing so. The existing `matrix-synapse` packages will
+    continue to use Python 2 for the time being.
+
+2.  This release removes the `riot.im` from the default list of trusted
+    identity servers.
+
+    If `riot.im` is in your homeserver's list of
+    `trusted_third_party_id_servers`, you should remove it. It was added
+    in case a hypothetical future identity server was put there. If you
+    don't remove it, users may be unable to deactivate their accounts.
+
+3.  This release no longer installs the (unmaintained) Matrix Console
+    web client as part of the default installation. It is possible to
+    re-enable it by installing it separately and setting the
+    `web_client_location` config option, but please consider switching
+    to another client.
+
+# Upgrading to v0.33.7
+
+This release removes the example email notification templates from
+`res/templates` (they are now internal to the python package). This
+should only affect you if you (a) deploy your Synapse instance from a
+git checkout or a github snapshot URL, and (b) have email notifications
+enabled.
+
+If you have email notifications enabled, you should ensure that
+`email.template_dir` is either configured to point at a directory where
+you have installed customised templates, or leave it unset to use the
+default templates.
+
+# Upgrading to v0.27.3
+
+This release expands the anonymous usage stats sent if the opt-in
+`report_stats` configuration is set to `true`. We now capture RSS memory
+and cpu use at a very coarse level. This requires administrators to
+install the optional `psutil` python module.
+
+We would appreciate it if you could assist by ensuring this module is
+available and `report_stats` is enabled. This will let us see if
+performance changes to synapse are having an impact to the general
+community.
+
+# Upgrading to v0.15.0
+
+If you want to use the new URL previewing API
+(`/_matrix/media/r0/preview_url`) then you have to explicitly enable it
+in the config and update your dependencies dependencies. See README.rst
+for details.
+
+# Upgrading to v0.11.0
+
+This release includes the option to send anonymous usage stats to
+matrix.org, and requires that administrators explictly opt in or out by
+setting the `report_stats` option to either `true` or `false`.
+
+We would really appreciate it if you could help our project out by
+reporting anonymized usage statistics from your homeserver. Only very
+basic aggregate data (e.g. number of users) will be reported, but it
+helps us to track the growth of the Matrix community, and helps us to
+make Matrix a success, as well as to convince other networks that they
+should peer with us.
+
+# Upgrading to v0.9.0
+
+Application services have had a breaking API change in this version.
+
+They can no longer register themselves with a home server using the AS
+HTTP API. This decision was made because a compromised application
+service with free reign to register any regex in effect grants full
+read/write access to the home server if a regex of `.*` is used. An
+attack where a compromised AS re-registers itself with `.*` was deemed
+too big of a security risk to ignore, and so the ability to register
+with the HS remotely has been removed.
+
+It has been replaced by specifying a list of application service
+registrations in `homeserver.yaml`:
+
+    app_service_config_files: ["registration-01.yaml", "registration-02.yaml"]
+
+Where `registration-01.yaml` looks like:
+
+    url: <String>  # e.g. "https://my.application.service.com"
+    as_token: <String>
+    hs_token: <String>
+    sender_localpart: <String>  # This is a new field which denotes the user_id localpart when using the AS token
+    namespaces:
+      users:
+        - exclusive: <Boolean>
+          regex: <String>  # e.g. "@prefix_.*"
+      aliases:
+        - exclusive: <Boolean>
+          regex: <String>
+      rooms:
+        - exclusive: <Boolean>
+          regex: <String>
+
+# Upgrading to v0.8.0
+
+Servers which use captchas will need to add their public key to:
+
+    static/client/register/register_config.js
+
+      window.matrixRegistrationConfig = {
+          recaptcha_public_key: "YOUR_PUBLIC_KEY"
+      };
+
+This is required in order to support registration fallback (typically
+used on mobile devices).
+
+# Upgrading to v0.7.0
+
+New dependencies are:
+
+-   pydenticon
+-   simplejson
+-   syutil
+-   matrix-angular-sdk
+
+To pull in these dependencies in a virtual env, run:
+
+    python synapse/python_dependencies.py | xargs -n 1 pip install
+
+# Upgrading to v0.6.0
+
+To pull in new dependencies, run:
+
+    python setup.py develop --user
+
+This update includes a change to the database schema. To upgrade you
+first need to upgrade the database by running:
+
+    python scripts/upgrade_db_to_v0.6.0.py <db> <server_name> <signing_key>
+
+Where [<db>]{.title-ref} is the location of the database,
+[<server_name>]{.title-ref} is the server name as specified in the
+synapse configuration, and [<signing_key>]{.title-ref} is the location
+of the signing key as specified in the synapse configuration.
+
+This may take some time to complete. Failures of signatures and content
+hashes can safely be ignored.
+
+# Upgrading to v0.5.1
+
+Depending on precisely when you installed v0.5.0 you may have ended up
+with a stale release of the reference matrix webclient installed as a
+python module. To uninstall it and ensure you are depending on the
+latest module, please run:
+
+    $ pip uninstall syweb
+
+# Upgrading to v0.5.0
+
+The webclient has been split out into a seperate repository/pacakage in
+this release. Before you restart your homeserver you will need to pull
+in the webclient package by running:
+
+    python setup.py develop --user
+
+This release completely changes the database schema and so requires
+upgrading it before starting the new version of the homeserver.
+
+The script "database-prepare-for-0.5.0.sh" should be used to upgrade
+the database. This will save all user information, such as logins and
+profiles, but will otherwise purge the database. This includes messages,
+which rooms the home server was a member of and room alias mappings.
+
+If you would like to keep your history, please take a copy of your
+database file and ask for help in #matrix:matrix.org. The upgrade
+process is, unfortunately, non trivial and requires human intervention
+to resolve any resulting conflicts during the upgrade process.
+
+Before running the command the homeserver should be first completely
+shutdown. To run it, simply specify the location of the database, e.g.:
+
+> ./scripts/database-prepare-for-0.5.0.sh "homeserver.db"
+
+Once this has successfully completed it will be safe to restart the
+homeserver. You may notice that the homeserver takes a few seconds
+longer to restart than usual as it reinitializes the database.
+
+On startup of the new version, users can either rejoin remote rooms
+using room aliases or by being reinvited. Alternatively, if any other
+homeserver sends a message to a room that the homeserver was previously
+in the local HS will automatically rejoin the room.
+
+# Upgrading to v0.4.0
+
+This release needs an updated syutil version. Run:
+
+    python setup.py develop
+
+You will also need to upgrade your configuration as the signing key
+format has changed. Run:
+
+    python -m synapse.app.homeserver --config-path <CONFIG> --generate-config
+
+# Upgrading to v0.3.0
+
+This registration API now closely matches the login API. This introduces
+a bit more backwards and forwards between the HS and the client, but
+this improves the overall flexibility of the API. You can now GET on
+/register to retrieve a list of valid registration flows. Upon choosing
+one, they are submitted in the same way as login, e.g:
+
+    {
+      type: m.login.password,
+      user: foo,
+      password: bar
+    }
+
+The default HS supports 2 flows, with and without Identity Server email
+authentication. Enabling captcha on the HS will add in an extra step to
+all flows: `m.login.recaptcha` which must be completed before you can
+transition to the next stage. There is a new login type:
+`m.login.email.identity` which contains the `threepidCreds` key which
+were previously sent in the original register request. For more
+information on this, see the specification.
+
+## Web Client
+
+The VoIP specification has changed between v0.2.0 and v0.3.0. Users
+should refresh any browser tabs to get the latest web client code. Users
+on v0.2.0 of the web client will not be able to call those on v0.3.0 and
+vice versa.
+
+# Upgrading to v0.2.0
+
+The home server now requires setting up of SSL config before it can run.
+To automatically generate default config use:
+
+    $ python synapse/app/homeserver.py \
+        --server-name machine.my.domain.name \
+        --bind-port 8448 \
+        --config-path homeserver.config \
+        --generate-config
+
+This config can be edited if desired, for example to specify a different
+SSL certificate to use. Once done you can run the home server using:
+
+    $ python synapse/app/homeserver.py --config-path homeserver.config
+
+See the README.rst for more information.
+
+Also note that some config options have been renamed, including:
+
+-   "host" to "server-name"
+-   "database" to "database-path"
+-   "port" to "bind-port" and "unsecure-port"
+
+# Upgrading to v0.0.1
+
+This release completely changes the database schema and so requires
+upgrading it before starting the new version of the homeserver.
+
+The script "database-prepare-for-0.0.1.sh" should be used to upgrade
+the database. This will save all user information, such as logins and
+profiles, but will otherwise purge the database. This includes messages,
+which rooms the home server was a member of and room alias mappings.
+
+Before running the command the homeserver should be first completely
+shutdown. To run it, simply specify the location of the database, e.g.:
+
+> ./scripts/database-prepare-for-0.0.1.sh "homeserver.db"
+
+Once this has successfully completed it will be safe to restart the
+homeserver. You may notice that the homeserver takes a few seconds
+longer to restart than usual as it reinitializes the database.
+
+On startup of the new version, users can either rejoin remote rooms
+using room aliases or by being reinvited. Alternatively, if any other
+homeserver sends a message to a room that the homeserver was previously
+in the local HS will automatically rejoin the room.
diff --git a/docs/usage/administration/README.md b/docs/usage/administration/README.md
new file mode 100644
index 0000000000..e1e57546ab
--- /dev/null
+++ b/docs/usage/administration/README.md
@@ -0,0 +1,7 @@
+# Administration
+
+This section contains information on managing your Synapse homeserver. This includes:
+
+* Managing users, rooms and media via the Admin API.
+* Setting up metrics and monitoring to give you insight into your homeserver's health.
+* Configuring structured logging.
\ No newline at end of file
diff --git a/docs/usage/administration/admin_api/README.md b/docs/usage/administration/admin_api/README.md
new file mode 100644
index 0000000000..2fca96f8be
--- /dev/null
+++ b/docs/usage/administration/admin_api/README.md
@@ -0,0 +1,29 @@
+# The Admin API
+
+## Authenticate as a server admin
+
+Many of the API calls in the admin api will require an `access_token` for a
+server admin. (Note that a server admin is distinct from a room admin.)
+
+A user can be marked as a server admin by updating the database directly, e.g.:
+
+```sql
+UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
+```
+
+A new server admin user can also be created using the `register_new_matrix_user`
+command. This is a script that is located in the `scripts/` directory, or possibly
+already on your `$PATH` depending on how Synapse was installed.
+
+Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.
+
+## Making an Admin API request
+Once you have your `access_token`, you will need to authenticate each request to an Admin API endpoint by
+providing the token as either a query parameter or a request header. To add it as a request header in cURL:
+
+```sh
+curl --header "Authorization: Bearer <access_token>" <the_rest_of_your_API_request>
+```
+
+For more details on access tokens in Matrix, please refer to the complete
+[matrix spec documentation](https://matrix.org/docs/spec/client_server/r0.6.1#using-access-tokens).
diff --git a/docs/usage/administration/request_log.md b/docs/usage/administration/request_log.md
new file mode 100644
index 0000000000..316304c734
--- /dev/null
+++ b/docs/usage/administration/request_log.md
@@ -0,0 +1,44 @@
+# Request log format
+
+HTTP request logs are written by synapse (see [`site.py`](../synapse/http/site.py) for details).
+
+See the following for how to decode the dense data available from the default logging configuration.
+
+```
+2020-10-01 12:00:00,000 - synapse.access.http.8008 - 311 - INFO - PUT-1000- 192.168.0.1 - 8008 - {another-matrix-server.com} Processed request: 0.100sec/-0.000sec (0.000sec, 0.000sec) (0.001sec/0.090sec/3) 11B !200 "PUT /_matrix/federation/v1/send/1600000000000 HTTP/1.1" "Synapse/1.20.1" [0 dbevts]
+-AAAAAAAAAAAAAAAAAAAAA-   -BBBBBBBBBBBBBBBBBBBBBB-   -C-   -DD-   -EEEEEE-  -FFFFFFFFF-   -GG-    -HHHHHHHHHHHHHHHHHHHHHHH-                     -IIIIII- -JJJJJJJ-  -KKKKKK-, -LLLLLL-  -MMMMMMM- -NNNNNN- O  -P- -QQ-  -RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR-   -SSSSSSSSSSSS-   -TTTTTT-
+```
+
+
+| Part  | Explanation | 
+| ----- | ------------ |
+| AAAA  | Timestamp request was logged (not recieved) |
+| BBBB  | Logger name (`synapse.access.(http\|https).<tag>`, where 'tag' is defined in the `listeners` config section, normally the port) |
+| CCCC  | Line number in code |
+| DDDD  | Log Level |
+| EEEE  | Request Identifier (This identifier is shared by related log lines)|
+| FFFF  | Source IP (Or X-Forwarded-For if enabled) |
+| GGGG  | Server Port |
+| HHHH  | Federated Server or Local User making request (blank if unauthenticated or not supplied) |
+| IIII  | Total Time to process the request |
+| JJJJ  | Time to send response over network once generated (this may be negative if the socket is closed before the response is generated)|
+| KKKK  | Userland CPU time |
+| LLLL  | System CPU time |
+| MMMM  | Total time waiting for a free DB connection from the pool across all parallel DB work from this request |
+| NNNN  | Total time waiting for response to DB queries across all parallel DB work from this request |
+| OOOO  | Count of DB transactions performed |
+| PPPP  | Response body size |
+| QQQQ  | Response status code (prefixed with ! if the socket was closed before the response was generated) |
+| RRRR  | Request |
+| SSSS  | User-agent |
+| TTTT  | Events fetched from DB to service this request (note that this does not include events fetched from the cache) |
+
+
+MMMM / NNNN can be greater than IIII if there are multiple slow database queries
+running in parallel.
+
+Some actions can result in multiple identical http requests, which will return
+the same data, but only the first request will report time/transactions in
+`KKKK`/`LLLL`/`MMMM`/`NNNN`/`OOOO` - the others will be awaiting the first query to return a
+response and will simultaneously return with the first request, but with very
+small processing times.
diff --git a/docs/usage/configuration/README.md b/docs/usage/configuration/README.md
new file mode 100644
index 0000000000..41d41167c6
--- /dev/null
+++ b/docs/usage/configuration/README.md
@@ -0,0 +1,4 @@
+# Configuration
+
+This section contains information on tweaking Synapse via the various options in the configuration file. A configuration
+file should have been generated when you [installed Synapse](../../setup/installation.html).
diff --git a/docs/usage/configuration/homeserver_sample_config.md b/docs/usage/configuration/homeserver_sample_config.md
new file mode 100644
index 0000000000..11e806998d
--- /dev/null
+++ b/docs/usage/configuration/homeserver_sample_config.md
@@ -0,0 +1,14 @@
+# Homeserver Sample Configuration File
+
+Below is a sample homeserver configuration file. The homeserver configuration file 
+can be tweaked to change the behaviour of your homeserver. A restart of the server is 
+generally required to apply any changes made to this file.
+
+Note that the contents below are *not* intended to be copied and used as the basis for
+a real homeserver.yaml. Instead, if you are starting from scratch, please generate
+a fresh config using Synapse by following the instructions in
+[Installation](../../setup/installation.md).
+
+```yaml
+{{#include ../../sample_config.yaml}}
+```
diff --git a/docs/usage/configuration/logging_sample_config.md b/docs/usage/configuration/logging_sample_config.md
new file mode 100644
index 0000000000..a673d487b8
--- /dev/null
+++ b/docs/usage/configuration/logging_sample_config.md
@@ -0,0 +1,14 @@
+# Logging Sample Configuration File
+
+Below is a sample logging configuration file. This file can be tweaked to control how your
+homeserver will output logs. A restart of the server is generally required to apply any
+changes made to this file.
+
+Note that the contents below are *not* intended to be copied and used as the basis for
+a real homeserver.yaml. Instead, if you are starting from scratch, please generate
+a fresh config using Synapse by following the instructions in
+[Installation](../../setup/installation.md).
+
+```yaml
+{{#include ../../sample_log_config.yaml}}
+```
\ No newline at end of file
diff --git a/docs/usage/configuration/user_authentication/README.md b/docs/usage/configuration/user_authentication/README.md
new file mode 100644
index 0000000000..087ae053cf
--- /dev/null
+++ b/docs/usage/configuration/user_authentication/README.md
@@ -0,0 +1,15 @@
+# User Authentication
+
+Synapse supports multiple methods of authenticating users, either out-of-the-box or through custom pluggable
+authentication modules.
+
+Included in Synapse is support for authenticating users via:
+
+* A username and password.
+* An email address and password.
+* Single Sign-On through the SAML, Open ID Connect or CAS protocols.
+* JSON Web Tokens.
+* An administrator's shared secret.
+
+Synapse can additionally be extended to support custom authentication schemes through optional "password auth provider"
+modules.
\ No newline at end of file
diff --git a/docs/user_directory.md b/docs/user_directory.md
index 872fc21979..d4f38d2cf1 100644
--- a/docs/user_directory.md
+++ b/docs/user_directory.md
@@ -7,6 +7,6 @@ who are present in a publicly viewable room present on the server.
 
 The directory info is stored in various tables, which can (typically after
 DB corruption) get stale or out of sync.  If this happens, for now the
-solution to fix it is to execute the SQL [here](../synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql)
+solution to fix it is to execute the SQL [here](https://github.com/matrix-org/synapse/blob/master/synapse/storage/schema/main/delta/53/user_dir_populate.sql)
 and then restart synapse. This should then start a background task to
 flush the current tables and regenerate the directory.
diff --git a/docs/website_files/README.md b/docs/website_files/README.md
new file mode 100644
index 0000000000..04d191479b
--- /dev/null
+++ b/docs/website_files/README.md
@@ -0,0 +1,30 @@
+# Documentation Website Files and Assets
+
+This directory contains extra files for modifying the look and functionality of 
+[mdbook](https://github.com/rust-lang/mdBook), the documentation software that's
+used to generate Synapse's documentation website.
+
+The configuration options in the `output.html` section of [book.toml](../../book.toml)
+point to additional JS/CSS in this directory that are added on each page load. In
+addition, the `theme` directory contains files that overwrite their counterparts in
+each of the default themes included with mdbook.
+
+Currently we use these files to generate a floating Table of Contents panel. The code for
+which was partially taken from
+[JorelAli/mdBook-pagetoc](https://github.com/JorelAli/mdBook-pagetoc/)
+before being modified such that it scrolls with the content of the page. This is handled
+by the `table-of-contents.js/css` files. The table of contents panel only appears on pages
+that have more than one header, as well as only appearing on desktop-sized monitors.
+
+We remove the navigation arrows which typically appear on the left and right side of the
+screen on desktop as they interfere with the table of contents. This is handled by
+the `remove-nav-buttons.css` file.
+
+Finally, we also stylise the chapter titles in the left sidebar by indenting them
+slightly so that they are more visually distinguishable from the section headers
+(the bold titles). This is done through the `indent-section-headers.css` file.
+
+More information can be found in mdbook's official documentation for
+[injecting page JS/CSS](https://rust-lang.github.io/mdBook/format/config.html)
+and
+[customising the default themes](https://rust-lang.github.io/mdBook/format/theme/index.html).
\ No newline at end of file
diff --git a/docs/website_files/indent-section-headers.css b/docs/website_files/indent-section-headers.css
new file mode 100644
index 0000000000..f9b3c82ca6
--- /dev/null
+++ b/docs/website_files/indent-section-headers.css
@@ -0,0 +1,7 @@
+/*
+ * Indents each chapter title in the left sidebar so that they aren't
+ * at the same level as the section headers.
+ */
+.chapter-item {
+    margin-left: 1em;
+}
\ No newline at end of file
diff --git a/docs/website_files/remove-nav-buttons.css b/docs/website_files/remove-nav-buttons.css
new file mode 100644
index 0000000000..4b280794ea
--- /dev/null
+++ b/docs/website_files/remove-nav-buttons.css
@@ -0,0 +1,8 @@
+/* Remove the prev, next chapter buttons as they interfere with the
+ * table of contents.
+ * Note that the table of contents only appears on desktop, thus we
+ * only remove the desktop (wide) chapter buttons.
+ */
+.nav-wide-wrapper {
+    display: none
+}
\ No newline at end of file
diff --git a/docs/website_files/table-of-contents.css b/docs/website_files/table-of-contents.css
new file mode 100644
index 0000000000..d16bb3b988
--- /dev/null
+++ b/docs/website_files/table-of-contents.css
@@ -0,0 +1,42 @@
+@media only screen and (max-width:1439px) {
+    .sidetoc {
+        display: none;
+    }
+}
+
+@media only screen and (min-width:1440px) {
+    main {
+        position: relative;
+        margin-left: 100px !important;
+    }
+    .sidetoc {
+        margin-left: auto;
+        margin-right: auto;
+        left: calc(100% + (var(--content-max-width))/4 - 140px);
+        position: absolute;
+        text-align: right;
+    }
+    .pagetoc {
+        position: fixed;
+        width: 250px;
+        overflow: auto;
+        right: 20px;
+        height: calc(100% - var(--menu-bar-height));
+    }
+    .pagetoc a {
+        color: var(--fg) !important;
+        display: block;
+        padding: 5px 15px 5px 10px;
+        text-align: left;
+        text-decoration: none;
+    }
+    .pagetoc a:hover,
+    .pagetoc a.active {
+        background: var(--sidebar-bg) !important;
+        color: var(--sidebar-fg) !important;
+    }
+    .pagetoc .active {
+        background: var(--sidebar-bg);
+        color: var(--sidebar-fg);
+    }
+}
diff --git a/docs/website_files/table-of-contents.js b/docs/website_files/table-of-contents.js
new file mode 100644
index 0000000000..0de5960b22
--- /dev/null
+++ b/docs/website_files/table-of-contents.js
@@ -0,0 +1,134 @@
+const getPageToc = () => document.getElementsByClassName('pagetoc')[0];
+
+const pageToc = getPageToc();
+const pageTocChildren = [...pageToc.children];
+const headers = [...document.getElementsByClassName('header')];
+
+
+// Select highlighted item in ToC when clicking an item
+pageTocChildren.forEach(child => {
+    child.addEventHandler('click', () => {
+        pageTocChildren.forEach(child => {
+            child.classList.remove('active');
+        });
+        child.classList.add('active');
+    });
+});
+
+
+/**
+ * Test whether a node is in the viewport
+ */
+function isInViewport(node) {
+    const rect = node.getBoundingClientRect();
+    return rect.top >= 0 && rect.left >= 0 && rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) && rect.right <= (window.innerWidth || document.documentElement.clientWidth);
+}
+
+
+/**
+ * Set a new ToC entry.
+ * Clear any previously highlighted ToC items, set the new one,
+ * and adjust the ToC scroll position.
+ */
+function setTocEntry() {
+    let activeEntry;
+    const pageTocChildren = [...getPageToc().children];
+
+    // Calculate which header is the current one at the top of screen
+    headers.forEach(header => {
+        if (window.pageYOffset >= header.offsetTop) {
+            activeEntry = header;
+        }
+    });
+
+    // Update selected item in ToC when scrolling
+    pageTocChildren.forEach(child => {
+        if (activeEntry.href.localeCompare(child.href) === 0) {
+            child.classList.add('active');
+        } else {
+            child.classList.remove('active');
+        }
+    });
+
+    let tocEntryForLocation = document.querySelector(`nav a[href="${activeEntry.href}"]`);
+    if (tocEntryForLocation) {
+        const headingForLocation = document.querySelector(activeEntry.hash);
+        if (headingForLocation && isInViewport(headingForLocation)) {
+            // Update ToC scroll
+            const nav = getPageToc();
+            const content = document.querySelector('html');
+            if (content.scrollTop !== 0) {
+                nav.scrollTo({
+                    top: tocEntryForLocation.offsetTop - 100,
+                    left: 0,
+                    behavior: 'smooth',
+                });
+            } else {
+                nav.scrollTop = 0;
+            }
+        }
+    }
+}
+
+
+/**
+ * Populate sidebar on load
+ */
+window.addEventListener('load', () => {
+    // Only create table of contents if there is more than one header on the page
+    if (headers.length <= 1) {
+        return;
+    }
+
+    // Create an entry in the page table of contents for each header in the document
+    headers.forEach((header, index) => {
+        const link = document.createElement('a');
+
+        // Indent shows hierarchy
+        let indent = '0px';
+        switch (header.parentElement.tagName) {
+            case 'H1':
+                indent = '5px';
+                break;
+            case 'H2':
+                indent = '20px';
+                break;
+            case 'H3':
+                indent = '30px';
+                break;
+            case 'H4':
+                indent = '40px';
+                break;
+            case 'H5':
+                indent = '50px';
+                break;
+            case 'H6':
+                indent = '60px';
+                break;
+            default:
+                break;
+        }
+
+        let tocEntry;
+        if (index == 0) {
+            // Create a bolded title for the first element
+            tocEntry = document.createElement("strong");
+            tocEntry.innerHTML = header.text;
+        } else {
+            // All other elements are non-bold
+            tocEntry = document.createTextNode(header.text);
+        }
+        link.appendChild(tocEntry);
+
+        link.style.paddingLeft = indent;
+        link.href = header.href;
+        pageToc.appendChild(link);
+    });
+    setTocEntry.call();
+});
+
+
+// Handle active headers on scroll, if there is more than one header on the page
+if (headers.length > 1) {
+    window.addEventListener('scroll', setTocEntry);
+}
diff --git a/docs/website_files/theme/index.hbs b/docs/website_files/theme/index.hbs
new file mode 100644
index 0000000000..3b7a5b6163
--- /dev/null
+++ b/docs/website_files/theme/index.hbs
@@ -0,0 +1,312 @@
+<!DOCTYPE HTML>
+<html lang="{{ language }}" class="sidebar-visible no-js {{ default_theme }}">
+    <head>
+        <!-- Book generated using mdBook -->
+        <meta charset="UTF-8">
+        <title>{{ title }}</title>
+        {{#if is_print }}
+        <meta name="robots" content="noindex" />
+        {{/if}}
+        {{#if base_url}}
+        <base href="{{ base_url }}">
+        {{/if}}
+
+
+        <!-- Custom HTML head -->
+        {{> head}}
+
+        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
+        <meta name="description" content="{{ description }}">
+        <meta name="viewport" content="width=device-width, initial-scale=1">
+        <meta name="theme-color" content="#ffffff" />
+
+        {{#if favicon_svg}}
+        <link rel="icon" href="{{ path_to_root }}favicon.svg">
+        {{/if}}
+        {{#if favicon_png}}
+        <link rel="shortcut icon" href="{{ path_to_root }}favicon.png">
+        {{/if}}
+        <link rel="stylesheet" href="{{ path_to_root }}css/variables.css">
+        <link rel="stylesheet" href="{{ path_to_root }}css/general.css">
+        <link rel="stylesheet" href="{{ path_to_root }}css/chrome.css">
+        {{#if print_enable}}
+        <link rel="stylesheet" href="{{ path_to_root }}css/print.css" media="print">
+        {{/if}}
+
+        <!-- Fonts -->
+        <link rel="stylesheet" href="{{ path_to_root }}FontAwesome/css/font-awesome.css">
+        {{#if copy_fonts}}
+        <link rel="stylesheet" href="{{ path_to_root }}fonts/fonts.css">
+        {{/if}}
+
+        <!-- Highlight.js Stylesheets -->
+        <link rel="stylesheet" href="{{ path_to_root }}highlight.css">
+        <link rel="stylesheet" href="{{ path_to_root }}tomorrow-night.css">
+        <link rel="stylesheet" href="{{ path_to_root }}ayu-highlight.css">
+
+        <!-- Custom theme stylesheets -->
+        {{#each additional_css}}
+        <link rel="stylesheet" href="{{ ../path_to_root }}{{ this }}">
+        {{/each}}
+
+        {{#if mathjax_support}}
+        <!-- MathJax -->
+        <script async type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
+        {{/if}}
+    </head>
+    <body>
+        <!-- Provide site root to javascript -->
+        <script type="text/javascript">
+            var path_to_root = "{{ path_to_root }}";
+            var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "{{ preferred_dark_theme }}" : "{{ default_theme }}";
+        </script>
+
+        <!-- Work around some values being stored in localStorage wrapped in quotes -->
+        <script type="text/javascript">
+            try {
+                var theme = localStorage.getItem('mdbook-theme');
+                var sidebar = localStorage.getItem('mdbook-sidebar');
+                if (theme.startsWith('"') && theme.endsWith('"')) {
+                    localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
+                }
+                if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
+                    localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
+                }
+            } catch (e) { }
+        </script>
+
+        <!-- Set the theme before any content is loaded, prevents flash -->
+        <script type="text/javascript">
+            var theme;
+            try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
+            if (theme === null || theme === undefined) { theme = default_theme; }
+            var html = document.querySelector('html');
+            html.classList.remove('no-js')
+            html.classList.remove('{{ default_theme }}')
+            html.classList.add(theme);
+            html.classList.add('js');
+        </script>
+
+        <!-- Hide / unhide sidebar before it is displayed -->
+        <script type="text/javascript">
+            var html = document.querySelector('html');
+            var sidebar = 'hidden';
+            if (document.body.clientWidth >= 1080) {
+                try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
+                sidebar = sidebar || 'visible';
+            }
+            html.classList.remove('sidebar-visible');
+            html.classList.add("sidebar-" + sidebar);
+        </script>
+
+        <nav id="sidebar" class="sidebar" aria-label="Table of contents">
+            <div class="sidebar-scrollbox">
+                {{#toc}}{{/toc}}
+            </div>
+            <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
+        </nav>
+
+        <div id="page-wrapper" class="page-wrapper">
+
+            <div class="page">
+                {{> header}}
+                <div id="menu-bar-hover-placeholder"></div>
+                <div id="menu-bar" class="menu-bar sticky bordered">
+                    <div class="left-buttons">
+                        <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
+                            <i class="fa fa-bars"></i>
+                        </button>
+                        <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
+                            <i class="fa fa-paint-brush"></i>
+                        </button>
+                        <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
+                            <li role="none"><button role="menuitem" class="theme" id="light">{{ theme_option "Light" }}</button></li>
+                            <li role="none"><button role="menuitem" class="theme" id="rust">{{ theme_option "Rust" }}</button></li>
+                            <li role="none"><button role="menuitem" class="theme" id="coal">{{ theme_option "Coal" }}</button></li>
+                            <li role="none"><button role="menuitem" class="theme" id="navy">{{ theme_option "Navy" }}</button></li>
+                            <li role="none"><button role="menuitem" class="theme" id="ayu">{{ theme_option "Ayu" }}</button></li>
+                        </ul>
+                        {{#if search_enabled}}
+                        <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
+                            <i class="fa fa-search"></i>
+                        </button>
+                        {{/if}}
+                    </div>
+
+                    <h1 class="menu-title">{{ book_title }}</h1>
+
+                    <div class="right-buttons">
+                        {{#if print_enable}}
+                        <a href="{{ path_to_root }}print.html" title="Print this book" aria-label="Print this book">
+                            <i id="print-button" class="fa fa-print"></i>
+                        </a>
+                        {{/if}}
+                        {{#if git_repository_url}}
+                        <a href="{{git_repository_url}}" title="Git repository" aria-label="Git repository">
+                            <i id="git-repository-button" class="fa {{git_repository_icon}}"></i>
+                        </a>
+                        {{/if}}
+                        {{#if git_repository_edit_url}}
+                        <a href="{{git_repository_edit_url}}" title="Suggest an edit" aria-label="Suggest an edit">
+                            <i id="git-edit-button" class="fa fa-edit"></i>
+                        </a>
+                        {{/if}}
+
+                    </div>
+                </div>
+
+                {{#if search_enabled}}
+                <div id="search-wrapper" class="hidden">
+                    <form id="searchbar-outer" class="searchbar-outer">
+                        <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
+                    </form>
+                    <div id="searchresults-outer" class="searchresults-outer hidden">
+                        <div id="searchresults-header" class="searchresults-header"></div>
+                        <ul id="searchresults">
+                        </ul>
+                    </div>
+                </div>
+                {{/if}}
+
+                <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
+                <script type="text/javascript">
+                    document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
+                    document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
+                    Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
+                        link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
+                    });
+                </script>
+
+                <div id="content" class="content">
+                    <main>
+                        <!-- Page table of contents -->
+                        <div class="sidetoc">
+                            <nav class="pagetoc"></nav>
+                        </div>
+
+                        {{{ content }}}
+                    </main>
+
+                    <nav class="nav-wrapper" aria-label="Page navigation">
+                        <!-- Mobile navigation buttons -->
+                        {{#previous}}
+                            <a rel="prev" href="{{ path_to_root }}{{link}}" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
+                                <i class="fa fa-angle-left"></i>
+                            </a>
+                        {{/previous}}
+
+                        {{#next}}
+                            <a rel="next" href="{{ path_to_root }}{{link}}" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
+                                <i class="fa fa-angle-right"></i>
+                            </a>
+                        {{/next}}
+
+                        <div style="clear: both"></div>
+                    </nav>
+                </div>
+            </div>
+
+            <nav class="nav-wide-wrapper" aria-label="Page navigation">
+                {{#previous}}
+                    <a rel="prev" href="{{ path_to_root }}{{link}}" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
+                        <i class="fa fa-angle-left"></i>
+                    </a>
+                {{/previous}}
+
+                {{#next}}
+                    <a rel="next" href="{{ path_to_root }}{{link}}" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
+                        <i class="fa fa-angle-right"></i>
+                    </a>
+                {{/next}}
+            </nav>
+
+        </div>
+
+        {{#if livereload}}
+        <!-- Livereload script (if served using the cli tool) -->
+        <script type="text/javascript">
+            var socket = new WebSocket("{{{livereload}}}");
+            socket.onmessage = function (event) {
+                if (event.data === "reload") {
+                    socket.close();
+                    location.reload();
+                }
+            };
+            window.onbeforeunload = function() {
+                socket.close();
+            }
+        </script>
+        {{/if}}
+
+        {{#if google_analytics}}
+        <!-- Google Analytics Tag -->
+        <script type="text/javascript">
+            var localAddrs = ["localhost", "127.0.0.1", ""];
+            // make sure we don't activate google analytics if the developer is
+            // inspecting the book locally...
+            if (localAddrs.indexOf(document.location.hostname) === -1) {
+                (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+                (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+                m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+                })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
+                ga('create', '{{google_analytics}}', 'auto');
+                ga('send', 'pageview');
+            }
+        </script>
+        {{/if}}
+
+        {{#if playground_line_numbers}}
+        <script type="text/javascript">
+            window.playground_line_numbers = true;
+        </script>
+        {{/if}}
+
+        {{#if playground_copyable}}
+        <script type="text/javascript">
+            window.playground_copyable = true;
+        </script>
+        {{/if}}
+
+        {{#if playground_js}}
+        <script src="{{ path_to_root }}ace.js" type="text/javascript" charset="utf-8"></script>
+        <script src="{{ path_to_root }}editor.js" type="text/javascript" charset="utf-8"></script>
+        <script src="{{ path_to_root }}mode-rust.js" type="text/javascript" charset="utf-8"></script>
+        <script src="{{ path_to_root }}theme-dawn.js" type="text/javascript" charset="utf-8"></script>
+        <script src="{{ path_to_root }}theme-tomorrow_night.js" type="text/javascript" charset="utf-8"></script>
+        {{/if}}
+
+        {{#if search_js}}
+        <script src="{{ path_to_root }}elasticlunr.min.js" type="text/javascript" charset="utf-8"></script>
+        <script src="{{ path_to_root }}mark.min.js" type="text/javascript" charset="utf-8"></script>
+        <script src="{{ path_to_root }}searcher.js" type="text/javascript" charset="utf-8"></script>
+        {{/if}}
+
+        <script src="{{ path_to_root }}clipboard.min.js" type="text/javascript" charset="utf-8"></script>
+        <script src="{{ path_to_root }}highlight.js" type="text/javascript" charset="utf-8"></script>
+        <script src="{{ path_to_root }}book.js" type="text/javascript" charset="utf-8"></script>
+
+        <!-- Custom JS scripts -->
+        {{#each additional_js}}
+        <script type="text/javascript" src="{{ ../path_to_root }}{{this}}"></script>
+        {{/each}}
+
+        {{#if is_print}}
+        {{#if mathjax_support}}
+        <script type="text/javascript">
+        window.addEventListener('load', function() {
+            MathJax.Hub.Register.StartupHook('End', function() {
+                window.setTimeout(window.print, 100);
+            });
+        });
+        </script>
+        {{else}}
+        <script type="text/javascript">
+        window.addEventListener('load', function() {
+            window.setTimeout(window.print, 100);
+        });
+        </script>
+        {{/if}}
+        {{/if}}
+
+    </body>
+</html>
\ No newline at end of file
diff --git a/docs/welcome_and_overview.md b/docs/welcome_and_overview.md
new file mode 100644
index 0000000000..30e75984d1
--- /dev/null
+++ b/docs/welcome_and_overview.md
@@ -0,0 +1,4 @@
+# Introduction
+
+Welcome to the documentation repository for Synapse, the reference
+[Matrix](https://matrix.org) homeserver implementation.
\ No newline at end of file
diff --git a/docs/workers.md b/docs/workers.md
index c6282165b0..2e63f03452 100644
--- a/docs/workers.md
+++ b/docs/workers.md
@@ -16,7 +16,7 @@ workers only work with PostgreSQL-based Synapse deployments. SQLite should only
 be used for demo purposes and any admin considering workers should already be
 running PostgreSQL.
 
-See also https://matrix.org/blog/2020/11/03/how-we-fixed-synapses-scalability
+See also [Matrix.org blog post](https://matrix.org/blog/2020/11/03/how-we-fixed-synapses-scalability)
 for a higher level overview.
 
 ## Main process/worker communication
@@ -73,7 +73,7 @@ https://hub.docker.com/r/matrixdotorg/synapse/.
 To make effective use of the workers, you will need to configure an HTTP
 reverse-proxy such as nginx or haproxy, which will direct incoming requests to
 the correct worker, or to the main synapse instance. See
-[reverse_proxy.md](reverse_proxy.md) for information on setting up a reverse
+[the reverse proxy documentation](reverse_proxy.md) for information on setting up a reverse
 proxy.
 
 When using workers, each worker process has its own configuration file which
@@ -170,8 +170,8 @@ Finally, you need to start your worker processes. This can be done with either
 `synctl` or your distribution's preferred service manager such as `systemd`. We
 recommend the use of `systemd` where available: for information on setting up
 `systemd` to start synapse workers, see
-[systemd-with-workers](systemd-with-workers). To use `synctl`, see
-[synctl_workers.md](synctl_workers.md).
+[Systemd with Workers](systemd-with-workers). To use `synctl`, see
+[Using synctl with Workers](synctl_workers.md).
 
 
 ## Available worker applications
@@ -214,6 +214,7 @@ expressions:
     ^/_matrix/federation/v1/send/
 
     # Client API requests
+    ^/_matrix/client/(api/v1|r0|unstable)/createRoom$
     ^/_matrix/client/(api/v1|r0|unstable)/publicRooms$
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$
@@ -228,6 +229,9 @@ expressions:
     ^/_matrix/client/(api/v1|r0|unstable)/joined_groups$
     ^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$
     ^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/
+    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/event/
+    ^/_matrix/client/(api/v1|r0|unstable)/joined_rooms$
+    ^/_matrix/client/(api/v1|r0|unstable)/search$
 
     # Registration/login requests
     ^/_matrix/client/(api/v1|r0|unstable)/login$
@@ -422,10 +426,12 @@ Handles the media repository. It can handle all endpoints starting with:
     ^/_synapse/admin/v1/user/.*/media.*$
     ^/_synapse/admin/v1/media/.*$
     ^/_synapse/admin/v1/quarantine_media/.*$
+    ^/_synapse/admin/v1/users/.*/media$
 
 You should also set `enable_media_repo: False` in the shared configuration
 file to stop the main synapse running background jobs related to managing the
-media repository.
+media repository. Note that doing so will prevent the main process from being
+able to handle the above endpoints.
 
 In the `media_repository` worker configuration file, configure the http listener to
 expose the `media` resource. For example:
diff --git a/mypy.ini b/mypy.ini
index 3ae5d45787..349efe37bb 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -8,6 +8,7 @@ show_traceback = True
 mypy_path = stubs
 warn_unreachable = True
 local_partial_types = True
+no_implicit_optional = True
 
 # To find all folders that pass mypy you run:
 #
@@ -31,6 +32,7 @@ files =
   synapse/http/federation/matrix_federation_agent.py,
   synapse/http/federation/well_known_resolver.py,
   synapse/http/matrixfederationclient.py,
+  synapse/http/servlet.py,
   synapse/http/server.py,
   synapse/http/site.py,
   synapse/logging,
@@ -40,7 +42,6 @@ files =
   synapse/push,
   synapse/replication,
   synapse/rest,
-  synapse/secrets.py,
   synapse/server.py,
   synapse/server_notices,
   synapse/spam_checker_api,
@@ -71,15 +72,25 @@ files =
   synapse/types.py,
   synapse/util/async_helpers.py,
   synapse/util/caches,
+  synapse/util/daemonize.py,
+  synapse/util/hash.py,
+  synapse/util/iterutils.py,
+  synapse/util/linked_list.py,
   synapse/util/metrics.py,
   synapse/util/macaroons.py,
+  synapse/util/module_loader.py,
+  synapse/util/msisdn.py,
   synapse/util/stringutils.py,
   synapse/visibility.py,
   tests/replication,
+  tests/test_event_auth.py,
   tests/test_utils,
   tests/handlers/test_password_providers.py,
+  tests/handlers/test_room_summary.py,
+  tests/handlers/test_send_email.py,
   tests/rest/client/v1/test_login.py,
   tests/rest/client/v2_alpha/test_auth.py,
+  tests/util/test_itertools.py,
   tests/util/test_stream_change_cache.py
 
 [mypy-pymacaroons.*]
@@ -124,7 +135,7 @@ ignore_missing_imports = True
 [mypy-canonicaljson]
 ignore_missing_imports = True
 
-[mypy-jaeger_client]
+[mypy-jaeger_client.*]
 ignore_missing_imports = True
 
 [mypy-jsonschema]
@@ -169,5 +180,11 @@ ignore_missing_imports = True
 [mypy-josepy.*]
 ignore_missing_imports = True
 
-[mypy-txacme.*]
+[mypy-pympler.*]
+ignore_missing_imports = True
+
+[mypy-phonenumbers.*]
+ignore_missing_imports = True
+
+[mypy-ijson.*]
 ignore_missing_imports = True
diff --git a/pyproject.toml b/pyproject.toml
index cd880d4e39..8bca1fa4ef 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,7 +35,7 @@
         showcontent = true
 
 [tool.black]
-target-version = ['py35']
+target-version = ['py36']
 exclude = '''
 
 (
diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages
index d0685c8b35..e9f89e38ef 100755
--- a/scripts-dev/build_debian_packages
+++ b/scripts-dev/build_debian_packages
@@ -10,57 +10,60 @@
 # can be passed on the commandline for debugging.
 
 import argparse
+import json
 import os
 import signal
 import subprocess
 import sys
 import threading
 from concurrent.futures import ThreadPoolExecutor
+from typing import Optional, Sequence
 
 DISTS = (
-    "debian:stretch",
-    "debian:buster",
+    "debian:buster",  # oldstable: EOL 2022-08
     "debian:bullseye",
+    "debian:bookworm",
     "debian:sid",
-    "ubuntu:xenial",
-    "ubuntu:bionic",
-    "ubuntu:focal",
-    "ubuntu:groovy",
+    "ubuntu:bionic",  # 18.04 LTS (our EOL forced by Py36 on 2021-12-23)
+    "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
+    "ubuntu:hirsute",  # 21.04 (EOL 2022-01-05)
 )
 
-DESC = '''\
+DESC = """\
 Builds .debs for synapse, using a Docker image for the build environment.
 
 By default, builds for all known distributions, but a list of distributions
 can be passed on the commandline for debugging.
-'''
+"""
+
+projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 
 class Builder(object):
-    def __init__(self, redirect_stdout=False):
+    def __init__(
+        self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None
+    ):
         self.redirect_stdout = redirect_stdout
+        self._docker_build_args = tuple(docker_build_args or ())
         self.active_containers = set()
         self._lock = threading.Lock()
         self._failed = False
 
-    def run_build(self, dist):
+    def run_build(self, dist, skip_tests=False):
         """Build deb for a single distribution"""
 
         if self._failed:
-            print("not building %s due to earlier failure" % (dist, ))
+            print("not building %s due to earlier failure" % (dist,))
             raise Exception("failed")
 
         try:
-            self._inner_build(dist)
+            self._inner_build(dist, skip_tests)
         except Exception as e:
             print("build of %s failed: %s" % (dist, e), file=sys.stderr)
             self._failed = True
             raise
 
-    def _inner_build(self, dist):
-        projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-        os.chdir(projdir)
-
+    def _inner_build(self, dist, skip_tests=False):
         tag = dist.split(":", 1)[1]
 
         # Make the dir where the debs will live.
@@ -69,47 +72,71 @@ class Builder(object):
         # we tend to get source packages which are full of debs. (We could hack
         # around that with more magic in the build_debian.sh script, but that
         # doesn't solve the problem for natively-run dpkg-buildpakage).
-        debsdir = os.path.join(projdir, '../debs')
+        debsdir = os.path.join(projdir, "../debs")
         os.makedirs(debsdir, exist_ok=True)
 
         if self.redirect_stdout:
-            logfile = os.path.join(debsdir, "%s.buildlog" % (tag, ))
+            logfile = os.path.join(debsdir, "%s.buildlog" % (tag,))
             print("building %s: directing output to %s" % (dist, logfile))
             stdout = open(logfile, "w")
         else:
             stdout = None
 
         # first build a docker image for the build environment
-        subprocess.check_call([
-            "docker", "build",
-            "--tag", "dh-venv-builder:" + tag,
-            "--build-arg", "distro=" + dist,
-            "-f", "docker/Dockerfile-dhvirtualenv",
-            "docker",
-        ], stdout=stdout, stderr=subprocess.STDOUT)
+        build_args = (
+            (
+                "docker",
+                "build",
+                "--tag",
+                "dh-venv-builder:" + tag,
+                "--build-arg",
+                "distro=" + dist,
+                "-f",
+                "docker/Dockerfile-dhvirtualenv",
+            )
+            + self._docker_build_args
+            + ("docker",)
+        )
+
+        subprocess.check_call(
+            build_args,
+            stdout=stdout,
+            stderr=subprocess.STDOUT,
+            cwd=projdir,
+        )
 
         container_name = "synapse_build_" + tag
         with self._lock:
             self.active_containers.add(container_name)
 
         # then run the build itself
-        subprocess.check_call([
-            "docker", "run",
-            "--rm",
-            "--name", container_name,
-            "--volume=" + projdir + ":/synapse/source:ro",
-            "--volume=" + debsdir + ":/debs",
-            "-e", "TARGET_USERID=%i" % (os.getuid(), ),
-            "-e", "TARGET_GROUPID=%i" % (os.getgid(), ),
-            "dh-venv-builder:" + tag,
-        ], stdout=stdout, stderr=subprocess.STDOUT)
+        subprocess.check_call(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "--name",
+                container_name,
+                "--volume=" + projdir + ":/synapse/source:ro",
+                "--volume=" + debsdir + ":/debs",
+                "-e",
+                "TARGET_USERID=%i" % (os.getuid(),),
+                "-e",
+                "TARGET_GROUPID=%i" % (os.getgid(),),
+                "-e",
+                "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""),
+                "dh-venv-builder:" + tag,
+            ],
+            stdout=stdout,
+            stderr=subprocess.STDOUT,
+        )
 
         with self._lock:
             self.active_containers.remove(container_name)
 
         if stdout is not None:
             stdout.close()
-            print("Completed build of %s" % (dist, ))
+            print("Completed build of %s" % (dist,))
 
     def kill_containers(self):
         with self._lock:
@@ -117,40 +144,75 @@ class Builder(object):
 
         for c in active:
             print("killing container %s" % (c,))
-            subprocess.run([
-                "docker", "kill", c,
-            ], stdout=subprocess.DEVNULL)
+            subprocess.run(
+                [
+                    "docker",
+                    "kill",
+                    c,
+                ],
+                stdout=subprocess.DEVNULL,
+            )
             with self._lock:
                 self.active_containers.remove(c)
 
 
-def run_builds(dists, jobs=1):
-    builder = Builder(redirect_stdout=(jobs > 1))
-
+def run_builds(builder, dists, jobs=1, skip_tests=False):
     def sig(signum, _frame):
         print("Caught SIGINT")
         builder.kill_containers()
+
     signal.signal(signal.SIGINT, sig)
 
     with ThreadPoolExecutor(max_workers=jobs) as e:
-        res = e.map(builder.run_build, dists)
+        res = e.map(lambda dist: builder.run_build(dist, skip_tests), dists)
 
     # make sure we consume the iterable so that exceptions are raised.
-    for r in res:
+    for _ in res:
         pass
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description=DESC,
     )
     parser.add_argument(
-        '-j', '--jobs', type=int, default=1,
-        help='specify the number of builds to run in parallel',
+        "-j",
+        "--jobs",
+        type=int,
+        default=1,
+        help="specify the number of builds to run in parallel",
+    )
+    parser.add_argument(
+        "--no-check",
+        action="store_true",
+        help="skip running tests after building",
+    )
+    parser.add_argument(
+        "--docker-build-arg",
+        action="append",
+        help="specify an argument to pass to docker build",
+    )
+    parser.add_argument(
+        "--show-dists-json",
+        action="store_true",
+        help="instead of building the packages, just list the dists to build for, as a json array",
     )
     parser.add_argument(
-        'dist', nargs='*', default=DISTS,
-        help='a list of distributions to build for. Default: %(default)s',
+        "dist",
+        nargs="*",
+        default=DISTS,
+        help="a list of distributions to build for. Default: %(default)s",
     )
     args = parser.parse_args()
-    run_builds(dists=args.dist, jobs=args.jobs)
+    if args.show_dists_json:
+        print(json.dumps(DISTS))
+    else:
+        builder = Builder(
+            redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg
+        )
+        run_builds(
+            builder,
+            dists=args.dist,
+            jobs=args.jobs,
+            skip_tests=args.no_check,
+        )
diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment
index 47fc99efcf..cad3ecffe4 100755
--- a/scripts-dev/check-newsfragment
+++ b/scripts-dev/check-newsfragment
@@ -11,7 +11,7 @@ set -e
 git remote set-branches --add origin dinsic
 git fetch -q origin dinsic
 
-pr="$BUILDKITE_PULL_REQUEST"
+pr="$PULL_REQUEST_NUMBER"
 
 # if there are changes in the debian directory, check that the debian changelog
 # has been updated
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index 3cde53f5c0..5d0ef8dd3a 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -1,22 +1,68 @@
-#! /bin/bash -eu
+#!/usr/bin/env bash
 # This script is designed for developers who want to test their code
 # against Complement.
 #
 # It makes a Synapse image which represents the current checkout,
-# then downloads Complement and runs it with that image.
+# builds a synapse-complement image on top, then runs tests with it.
+#
+# By default the script will fetch the latest Complement master branch and
+# run tests with that. This can be overridden to use a custom Complement
+# checkout by setting the COMPLEMENT_DIR environment variable to the
+# filepath of a local Complement checkout.
+#
+# By default Synapse is run in monolith mode. This can be overridden by
+# setting the WORKERS environment variable.
+#
+# A regular expression of test method names can be supplied as the first
+# argument to the script. Complement will then only run those tests. If
+# no regex is supplied, all tests are run. For example;
+#
+# ./complement.sh "TestOutboundFederation(Profile|Send)"
+#
+
+# Exit if a line returns a non-zero exit code
+set -e
 
+# Change to the repository root
 cd "$(dirname $0)/.."
 
+# Check for a user-specified Complement checkout
+if [[ -z "$COMPLEMENT_DIR" ]]; then
+  echo "COMPLEMENT_DIR not set. Fetching the latest Complement checkout..."
+  wget -Nq https://github.com/matrix-org/complement/archive/master.tar.gz
+  tar -xzf master.tar.gz
+  COMPLEMENT_DIR=complement-master
+  echo "Checkout available at 'complement-master'"
+fi
+
+# If we're using workers, modify the docker files slightly.
+if [[ -n "$WORKERS" ]]; then
+  BASE_IMAGE=matrixdotorg/synapse-workers
+  BASE_DOCKERFILE=docker/Dockerfile-workers
+  export COMPLEMENT_BASE_IMAGE=complement-synapse-workers
+  COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile
+  # And provide some more configuration to complement.
+  export COMPLEMENT_CA=true
+  export COMPLEMENT_VERSION_CHECK_ITERATIONS=500
+else
+  BASE_IMAGE=matrixdotorg/synapse
+  BASE_DOCKERFILE=docker/Dockerfile
+  export COMPLEMENT_BASE_IMAGE=complement-synapse
+  COMPLEMENT_DOCKERFILE=Synapse.Dockerfile
+fi
+
 # Build the base Synapse image from the local checkout
-docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
+docker build -t $BASE_IMAGE -f "$BASE_DOCKERFILE" .
+# Build the Synapse monolith image from Complement, based on the above image we just built
+docker build -t $COMPLEMENT_BASE_IMAGE -f "$COMPLEMENT_DIR/dockerfiles/$COMPLEMENT_DOCKERFILE" "$COMPLEMENT_DIR/dockerfiles"
 
-# Download Complement
-wget -N https://github.com/matrix-org/complement/archive/master.tar.gz
-tar -xzf master.tar.gz
-cd complement-master
+cd "$COMPLEMENT_DIR"
 
-# Build the Synapse image from Complement, based on the above image we just built
-docker build -t complement-synapse -f dockerfiles/Synapse.Dockerfile ./dockerfiles
+EXTRA_COMPLEMENT_ARGS=""
+if [[ -n "$1" ]]; then
+  # A test name regex has been set, supply it to Complement
+  EXTRA_COMPLEMENT_ARGS+="-run $1 "
+fi
 
-# Run the tests on the resulting image!
-COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -count=1 ./tests
+# Run the tests!
+go test -v -tags synapse_blacklist,msc2946,msc3083,msc2403,msc2716 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/...
diff --git a/scripts-dev/convert_server_keys.py b/scripts-dev/convert_server_keys.py
deleted file mode 100644
index 961dc59f11..0000000000
--- a/scripts-dev/convert_server_keys.py
+++ /dev/null
@@ -1,115 +0,0 @@
-import hashlib
-import json
-import sys
-import time
-
-import psycopg2
-import yaml
-from canonicaljson import encode_canonical_json
-from signedjson.key import read_signing_keys
-from signedjson.sign import sign_json
-from unpaddedbase64 import encode_base64
-
-db_binary_type = memoryview
-
-
-def select_v1_keys(connection):
-    cursor = connection.cursor()
-    cursor.execute("SELECT server_name, key_id, verify_key FROM server_signature_keys")
-    rows = cursor.fetchall()
-    cursor.close()
-    results = {}
-    for server_name, key_id, verify_key in rows:
-        results.setdefault(server_name, {})[key_id] = encode_base64(verify_key)
-    return results
-
-
-def select_v1_certs(connection):
-    cursor = connection.cursor()
-    cursor.execute("SELECT server_name, tls_certificate FROM server_tls_certificates")
-    rows = cursor.fetchall()
-    cursor.close()
-    results = {}
-    for server_name, tls_certificate in rows:
-        results[server_name] = tls_certificate
-    return results
-
-
-def select_v2_json(connection):
-    cursor = connection.cursor()
-    cursor.execute("SELECT server_name, key_id, key_json FROM server_keys_json")
-    rows = cursor.fetchall()
-    cursor.close()
-    results = {}
-    for server_name, key_id, key_json in rows:
-        results.setdefault(server_name, {})[key_id] = json.loads(
-            str(key_json).decode("utf-8")
-        )
-    return results
-
-
-def convert_v1_to_v2(server_name, valid_until, keys, certificate):
-    return {
-        "old_verify_keys": {},
-        "server_name": server_name,
-        "verify_keys": {key_id: {"key": key} for key_id, key in keys.items()},
-        "valid_until_ts": valid_until,
-        "tls_fingerprints": [fingerprint(certificate)],
-    }
-
-
-def fingerprint(certificate):
-    finger = hashlib.sha256(certificate)
-    return {"sha256": encode_base64(finger.digest())}
-
-
-def rows_v2(server, json):
-    valid_until = json["valid_until_ts"]
-    key_json = encode_canonical_json(json)
-    for key_id in json["verify_keys"]:
-        yield (server, key_id, "-", valid_until, valid_until, db_binary_type(key_json))
-
-
-def main():
-    config = yaml.safe_load(open(sys.argv[1]))
-    valid_until = int(time.time() / (3600 * 24)) * 1000 * 3600 * 24
-
-    server_name = config["server_name"]
-    signing_key = read_signing_keys(open(config["signing_key_path"]))[0]
-
-    database = config["database"]
-    assert database["name"] == "psycopg2", "Can only convert for postgresql"
-    args = database["args"]
-    args.pop("cp_max")
-    args.pop("cp_min")
-    connection = psycopg2.connect(**args)
-    keys = select_v1_keys(connection)
-    certificates = select_v1_certs(connection)
-    json = select_v2_json(connection)
-
-    result = {}
-    for server in keys:
-        if server not in json:
-            v2_json = convert_v1_to_v2(
-                server, valid_until, keys[server], certificates[server]
-            )
-            v2_json = sign_json(v2_json, server_name, signing_key)
-            result[server] = v2_json
-
-    yaml.safe_dump(result, sys.stdout, default_flow_style=False)
-
-    rows = [row for server, json in result.items() for row in rows_v2(server, json)]
-
-    cursor = connection.cursor()
-    cursor.executemany(
-        "INSERT INTO server_keys_json ("
-        " server_name, key_id, from_server,"
-        " ts_added_ms, ts_valid_until_ms, key_json"
-        ") VALUES (%s, %s, %s, %s, %s, %s)",
-        rows,
-    )
-    connection.commit()
-
-
-if __name__ == "__main__":
-    main()
diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py
index 313860df13..c82ddd9677 100755
--- a/scripts-dev/definitions.py
+++ b/scripts-dev/definitions.py
@@ -140,7 +140,7 @@ def used_names(prefix, item, defs, names):
 
     definitions = {}
     for directory in args.directories:
-        for root, dirs, files in os.walk(directory):
+        for root, _, files in os.walk(directory):
             for filename in files:
                 if filename.endswith(".py"):
                     filepath = os.path.join(root, filename)
diff --git a/scripts-dev/dump_macaroon.py b/scripts-dev/dump_macaroon.py
index 980b5e709f..0ca75d3fe1 100755
--- a/scripts-dev/dump_macaroon.py
+++ b/scripts-dev/dump_macaroon.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 
 import sys
 
diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh
index 9761e97594..809eff166a 100755
--- a/scripts-dev/lint.sh
+++ b/scripts-dev/lint.sh
@@ -80,8 +80,22 @@ else
   # then lint everything!
   if [[ -z ${files+x} ]]; then
     # Lint all source code files and directories
-    # Note: this list aims the mirror the one in tox.ini
-    files=("synapse" "docker" "tests" "scripts-dev" "scripts" "contrib" "synctl" "setup.py" "synmark" "stubs" ".buildkite")
+    # Note: this list aims to mirror the one in tox.ini
+      files=(
+          "synapse" "docker" "tests"
+          # annoyingly, black doesn't find these so we have to list them
+          "scripts/export_signing_key"
+          "scripts/generate_config"
+          "scripts/generate_log_config"
+          "scripts/hash_password"
+          "scripts/register_new_matrix_user"
+          "scripts/synapse_port_db"
+          "scripts-dev"
+          "scripts-dev/build_debian_packages"
+          "scripts-dev/sign_json"
+          "scripts-dev/update_database"
+          "contrib" "synctl" "setup.py" "synmark" "stubs" ".ci"
+      )
   fi
 fi
 
diff --git a/scripts-dev/list_url_patterns.py b/scripts-dev/list_url_patterns.py
index 26ad7c67f4..e85420dea8 100755
--- a/scripts-dev/list_url_patterns.py
+++ b/scripts-dev/list_url_patterns.py
@@ -48,7 +48,7 @@ def find_patterns_in_file(filepath):
 
 
 for directory in args.directories:
-    for root, dirs, files in os.walk(directory):
+    for root, _, files in os.walk(directory):
         for filename in files:
             if filename.endswith(".py"):
                 filepath = os.path.join(root, filename)
diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh
index bc8f978660..39bf30d258 100755
--- a/scripts-dev/make_full_schema.sh
+++ b/scripts-dev/make_full_schema.sh
@@ -6,7 +6,7 @@
 # It does so by having Synapse generate an up-to-date SQLite DB, then running
 # synapse_port_db to convert it to Postgres. It then dumps the contents of both.
 
-POSTGRES_HOST="localhost"
+export PGHOST="localhost"
 POSTGRES_DB_NAME="synapse_full_schema.$$"
 
 SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite"
@@ -32,7 +32,7 @@ usage() {
 while getopts "p:co:h" opt; do
   case $opt in
     p)
-      POSTGRES_USERNAME=$OPTARG
+      export PGUSER=$OPTARG
       ;;
     c)
       # Print all commands that are being executed
@@ -69,7 +69,7 @@ if [ ${#unsatisfied_requirements} -ne 0 ]; then
   exit 1
 fi
 
-if [ -z "$POSTGRES_USERNAME" ]; then
+if [ -z "$PGUSER" ]; then
   echo "No postgres username supplied"
   usage
   exit 1
@@ -84,8 +84,9 @@ fi
 # Create the output directory if it doesn't exist
 mkdir -p "$OUTPUT_DIR"
 
-read -rsp "Postgres password for '$POSTGRES_USERNAME': " POSTGRES_PASSWORD
+read -rsp "Postgres password for '$PGUSER': " PGPASSWORD
 echo ""
+export PGPASSWORD
 
 # Exit immediately if a command fails
 set -e
@@ -131,9 +132,9 @@ report_stats: false
 database:
   name: "psycopg2"
   args:
-    user: "$POSTGRES_USERNAME"
-    host: "$POSTGRES_HOST"
-    password: "$POSTGRES_PASSWORD"
+    user: "$PGUSER"
+    host: "$PGHOST"
+    password: "$PGPASSWORD"
     database: "$POSTGRES_DB_NAME"
 
 # Suppress the key server warning.
@@ -150,7 +151,7 @@ scripts-dev/update_database --database-config "$SQLITE_CONFIG"
 
 # Create the PostgreSQL database.
 echo "Creating postgres database..."
-createdb $POSTGRES_DB_NAME
+createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
 
 echo "Copying data from SQLite3 to Postgres with synapse_port_db..."
 if [ -z "$COVERAGE" ]; then
@@ -181,7 +182,7 @@ DROP TABLE user_directory_search_docsize;
 DROP TABLE user_directory_search_stat;
 "
 sqlite3 "$SQLITE_DB" <<< "$SQL"
-psql $POSTGRES_DB_NAME -U "$POSTGRES_USERNAME" -w <<< "$SQL"
+psql "$POSTGRES_DB_NAME" -w <<< "$SQL"
 
 echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE'..."
 sqlite3 "$SQLITE_DB" ".dump" > "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE"
diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py
index 18df68305b..1217e14874 100644
--- a/scripts-dev/mypy_synapse_plugin.py
+++ b/scripts-dev/mypy_synapse_plugin.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/scripts-dev/release.py b/scripts-dev/release.py
new file mode 100755
index 0000000000..a339260c43
--- /dev/null
+++ b/scripts-dev/release.py
@@ -0,0 +1,499 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An interactive script for doing a release. See `cli()` below.
+"""
+
+import re
+import subprocess
+import sys
+import urllib.request
+from os import path
+from tempfile import TemporaryDirectory
+from typing import List, Optional, Tuple
+
+import attr
+import click
+import commonmark
+import git
+import redbaron
+from click.exceptions import ClickException
+from github import Github
+from packaging import version
+
+
+@click.group()
+def cli():
+    """An interactive script to walk through the parts of creating a release.
+
+    Requires the dev dependencies be installed, which can be done via:
+
+        pip install -e .[dev]
+
+    Then to use:
+
+        ./scripts-dev/release.py prepare
+
+        # ... ask others to look at the changelog ...
+
+        ./scripts-dev/release.py tag
+
+        # ... wait for asssets to build ...
+
+        ./scripts-dev/release.py publish
+        ./scripts-dev/release.py upload
+
+    If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the
+    `tag`/`publish` command, then a new draft release will be created/published.
+    """
+
+
+@cli.command()
+def prepare():
+    """Do the initial stages of creating a release, including creating release
+    branch, updating changelog and pushing to GitHub.
+    """
+
+    # Make sure we're in a git repo.
+    try:
+        repo = git.Repo()
+    except git.InvalidGitRepositoryError:
+        raise click.ClickException("Not in Synapse repo.")
+
+    if repo.is_dirty():
+        raise click.ClickException("Uncommitted changes exist.")
+
+    click.secho("Updating git repo...")
+    repo.remote().fetch()
+
+    # Get the current version and AST from root Synapse module.
+    current_version, parsed_synapse_ast, version_node = parse_version_from_module()
+
+    # Figure out what sort of release we're doing and calcuate the new version.
+    rc = click.confirm("RC", default=True)
+    if current_version.pre:
+        # If the current version is an RC we don't need to bump any of the
+        # version numbers (other than the RC number).
+        if rc:
+            new_version = "{}.{}.{}rc{}".format(
+                current_version.major,
+                current_version.minor,
+                current_version.micro,
+                current_version.pre[1] + 1,
+            )
+        else:
+            new_version = "{}.{}.{}".format(
+                current_version.major,
+                current_version.minor,
+                current_version.micro,
+            )
+    else:
+        # If this is a new release cycle then we need to know if it's a minor
+        # or a patch version bump.
+        release_type = click.prompt(
+            "Release type",
+            type=click.Choice(("minor", "patch")),
+            show_choices=True,
+            default="minor",
+        )
+
+        if release_type == "minor":
+            if rc:
+                new_version = "{}.{}.{}rc1".format(
+                    current_version.major,
+                    current_version.minor + 1,
+                    0,
+                )
+            else:
+                new_version = "{}.{}.{}".format(
+                    current_version.major,
+                    current_version.minor + 1,
+                    0,
+                )
+        else:
+            if rc:
+                new_version = "{}.{}.{}rc1".format(
+                    current_version.major,
+                    current_version.minor,
+                    current_version.micro + 1,
+                )
+            else:
+                new_version = "{}.{}.{}".format(
+                    current_version.major,
+                    current_version.minor,
+                    current_version.micro + 1,
+                )
+
+    # Confirm the calculated version is OK.
+    if not click.confirm(f"Create new version: {new_version}?", default=True):
+        click.get_current_context().abort()
+
+    # Switch to the release branch.
+    parsed_new_version = version.parse(new_version)
+
+    # We assume for debian changelogs that we only do RCs or full releases.
+    assert not parsed_new_version.is_devrelease
+    assert not parsed_new_version.is_postrelease
+
+    release_branch_name = (
+        f"release-v{parsed_new_version.major}.{parsed_new_version.minor}"
+    )
+    release_branch = find_ref(repo, release_branch_name)
+    if release_branch:
+        if release_branch.is_remote():
+            # If the release branch only exists on the remote we check it out
+            # locally.
+            repo.git.checkout(release_branch_name)
+            release_branch = repo.active_branch
+    else:
+        # If a branch doesn't exist we create one. We ask which one branch it
+        # should be based off, defaulting to sensible values depending on the
+        # release type.
+        if current_version.is_prerelease:
+            default = release_branch_name
+        elif release_type == "minor":
+            default = "develop"
+        else:
+            default = "master"
+
+        branch_name = click.prompt(
+            "Which branch should the release be based on?", default=default
+        )
+
+        base_branch = find_ref(repo, branch_name)
+        if not base_branch:
+            print(f"Could not find base branch {branch_name}!")
+            click.get_current_context().abort()
+
+        # Check out the base branch and ensure it's up to date
+        repo.head.reference = base_branch
+        repo.head.reset(index=True, working_tree=True)
+        if not base_branch.is_remote():
+            update_branch(repo)
+
+        # Create the new release branch
+        release_branch = repo.create_head(release_branch_name, commit=base_branch)
+
+    # Switch to the release branch and ensure its up to date.
+    repo.git.checkout(release_branch_name)
+    update_branch(repo)
+
+    # Update the `__version__` variable and write it back to the file.
+    version_node.value = '"' + new_version + '"'
+    with open("synapse/__init__.py", "w") as f:
+        f.write(parsed_synapse_ast.dumps())
+
+    # Generate changelogs
+    subprocess.run("python3 -m towncrier", shell=True)
+
+    # Generate debian changelogs
+    if parsed_new_version.pre is not None:
+        # If this is an RC then we need to coerce the version string to match
+        # Debian norms, e.g. 1.39.0rc2 gets converted to 1.39.0~rc2.
+        base_ver = parsed_new_version.base_version
+        pre_type, pre_num = parsed_new_version.pre
+        debian_version = f"{base_ver}~{pre_type}{pre_num}"
+    else:
+        debian_version = new_version
+
+    subprocess.run(
+        f'dch -M -v {debian_version} "New synapse release {debian_version}."',
+        shell=True,
+    )
+    subprocess.run('dch -M -r -D stable ""', shell=True)
+
+    # Show the user the changes and ask if they want to edit the change log.
+    repo.git.add("-u")
+    subprocess.run("git diff --cached", shell=True)
+
+    if click.confirm("Edit changelog?", default=False):
+        click.edit(filename="CHANGES.md")
+
+    # Commit the changes.
+    repo.git.add("-u")
+    repo.git.commit(f"-m {new_version}")
+
+    # We give the option to bail here in case the user wants to make sure things
+    # are OK before pushing.
+    if not click.confirm("Push branch to github?", default=True):
+        print("")
+        print("Run when ready to push:")
+        print("")
+        print(f"\tgit push -u {repo.remote().name} {repo.active_branch.name}")
+        print("")
+        sys.exit(0)
+
+    # Otherwise, push and open the changelog in the browser.
+    repo.git.push("-u", repo.remote().name, repo.active_branch.name)
+
+    click.launch(
+        f"https://github.com/matrix-org/synapse/blob/{repo.active_branch.name}/CHANGES.md"
+    )
+
+
+@cli.command()
+@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"])
+def tag(gh_token: Optional[str]):
+    """Tags the release and generates a draft GitHub release"""
+
+    # Make sure we're in a git repo.
+    try:
+        repo = git.Repo()
+    except git.InvalidGitRepositoryError:
+        raise click.ClickException("Not in Synapse repo.")
+
+    if repo.is_dirty():
+        raise click.ClickException("Uncommitted changes exist.")
+
+    click.secho("Updating git repo...")
+    repo.remote().fetch()
+
+    # Find out the version and tag name.
+    current_version, _, _ = parse_version_from_module()
+    tag_name = f"v{current_version}"
+
+    # Check we haven't released this version.
+    if tag_name in repo.tags:
+        raise click.ClickException(f"Tag {tag_name} already exists!\n")
+
+    # Get the appropriate changelogs and tag.
+    changes = get_changes_for_version(current_version)
+
+    click.echo_via_pager(changes)
+    if click.confirm("Edit text?", default=False):
+        changes = click.edit(changes, require_save=False)
+
+    repo.create_tag(tag_name, message=changes)
+
+    if not click.confirm("Push tag to GitHub?", default=True):
+        print("")
+        print("Run when ready to push:")
+        print("")
+        print(f"\tgit push {repo.remote().name} tag {current_version}")
+        print("")
+        return
+
+    repo.git.push(repo.remote().name, "tag", tag_name)
+
+    # If no token was given, we bail here
+    if not gh_token:
+        click.launch(f"https://github.com/matrix-org/synapse/releases/edit/{tag_name}")
+        return
+
+    # Create a new draft release
+    gh = Github(gh_token)
+    gh_repo = gh.get_repo("matrix-org/synapse")
+    release = gh_repo.create_git_release(
+        tag=tag_name,
+        name=tag_name,
+        message=changes,
+        draft=True,
+        prerelease=current_version.is_prerelease,
+    )
+
+    # Open the release and the actions where we are building the assets.
+    click.launch(release.html_url)
+    click.launch(
+        f"https://github.com/matrix-org/synapse/actions?query=branch%3A{tag_name}"
+    )
+
+    click.echo("Wait for release assets to be built")
+
+
+@cli.command()
+@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True)
+def publish(gh_token: str):
+    """Publish release."""
+
+    # Make sure we're in a git repo.
+    try:
+        repo = git.Repo()
+    except git.InvalidGitRepositoryError:
+        raise click.ClickException("Not in Synapse repo.")
+
+    if repo.is_dirty():
+        raise click.ClickException("Uncommitted changes exist.")
+
+    current_version, _, _ = parse_version_from_module()
+    tag_name = f"v{current_version}"
+
+    if not click.confirm(f"Publish {tag_name}?", default=True):
+        return
+
+    # Publish the draft release
+    gh = Github(gh_token)
+    gh_repo = gh.get_repo("matrix-org/synapse")
+    for release in gh_repo.get_releases():
+        if release.title == tag_name:
+            break
+    else:
+        raise ClickException(f"Failed to find GitHub release for {tag_name}")
+
+    assert release.title == tag_name
+
+    if not release.draft:
+        click.echo("Release already published.")
+        return
+
+    release = release.update_release(
+        name=release.title,
+        message=release.body,
+        tag_name=release.tag_name,
+        prerelease=release.prerelease,
+        draft=False,
+    )
+
+
+@cli.command()
+def upload():
+    """Upload release to pypi."""
+
+    current_version, _, _ = parse_version_from_module()
+    tag_name = f"v{current_version}"
+
+    pypi_asset_names = [
+        f"matrix_synapse-{current_version}-py3-none-any.whl",
+        f"matrix-synapse-{current_version}.tar.gz",
+    ]
+
+    with TemporaryDirectory(prefix=f"synapse_upload_{tag_name}_") as tmpdir:
+        for name in pypi_asset_names:
+            filename = path.join(tmpdir, name)
+            url = f"https://github.com/matrix-org/synapse/releases/download/{tag_name}/{name}"
+
+            click.echo(f"Downloading {name} into {filename}")
+            urllib.request.urlretrieve(url, filename=filename)
+
+        if click.confirm("Upload to PyPI?", default=True):
+            subprocess.run("twine upload *", shell=True, cwd=tmpdir)
+
+    click.echo(
+        f"Done! Remember to merge the tag {tag_name} into the appropriate branches"
+    )
+
+
+def parse_version_from_module() -> Tuple[
+    version.Version, redbaron.RedBaron, redbaron.Node
+]:
+    # Parse the AST and load the `__version__` node so that we can edit it
+    # later.
+    with open("synapse/__init__.py") as f:
+        red = redbaron.RedBaron(f.read())
+
+    version_node = None
+    for node in red:
+        if node.type != "assignment":
+            continue
+
+        if node.target.type != "name":
+            continue
+
+        if node.target.value != "__version__":
+            continue
+
+        version_node = node
+        break
+
+    if not version_node:
+        print("Failed to find '__version__' definition in synapse/__init__.py")
+        sys.exit(1)
+
+    # Parse the current version.
+    current_version = version.parse(version_node.value.value.strip('"'))
+    assert isinstance(current_version, version.Version)
+
+    return current_version, red, version_node
+
+
+def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]:
+    """Find the branch/ref, looking first locally then in the remote."""
+    if ref_name in repo.refs:
+        return repo.refs[ref_name]
+    elif ref_name in repo.remote().refs:
+        return repo.remote().refs[ref_name]
+    else:
+        return None
+
+
+def update_branch(repo: git.Repo):
+    """Ensure branch is up to date if it has a remote"""
+    if repo.active_branch.tracking_branch():
+        repo.git.merge(repo.active_branch.tracking_branch().name)
+
+
+def get_changes_for_version(wanted_version: version.Version) -> str:
+    """Get the changelogs for the given version.
+
+    If an RC then will only get the changelog for that RC version, otherwise if
+    its a full release will get the changelog for the release and all its RCs.
+    """
+
+    with open("CHANGES.md") as f:
+        changes = f.read()
+
+    # First we parse the changelog so that we can split it into sections based
+    # on the release headings.
+    ast = commonmark.Parser().parse(changes)
+
+    @attr.s(auto_attribs=True)
+    class VersionSection:
+        title: str
+
+        # These are 0-based.
+        start_line: int
+        end_line: Optional[int] = None  # Is none if its the last entry
+
+    headings: List[VersionSection] = []
+    for node, _ in ast.walker():
+        # We look for all text nodes that are in a level 1 heading.
+        if node.t != "text":
+            continue
+
+        if node.parent.t != "heading" or node.parent.level != 1:
+            continue
+
+        # If we have a previous heading then we update its `end_line`.
+        if headings:
+            headings[-1].end_line = node.parent.sourcepos[0][0] - 1
+
+        headings.append(VersionSection(node.literal, node.parent.sourcepos[0][0] - 1))
+
+    changes_by_line = changes.split("\n")
+
+    version_changelog = []  # The lines we want to include in the changelog
+
+    # Go through each section and find any that match the requested version.
+    regex = re.compile(r"^Synapse v?(\S+)")
+    for section in headings:
+        groups = regex.match(section.title)
+        if not groups:
+            continue
+
+        heading_version = version.parse(groups.group(1))
+        heading_base_version = version.parse(heading_version.base_version)
+
+        # Check if heading version matches the requested version, or if its an
+        # RC of the requested version.
+        if wanted_version not in (heading_version, heading_base_version):
+            continue
+
+        version_changelog.extend(changes_by_line[section.start_line : section.end_line])
+
+    return "\n".join(version_changelog)
+
+
+if __name__ == "__main__":
+    cli()
diff --git a/scripts-dev/sign_json b/scripts-dev/sign_json
index 44553fb79a..4a43d3f2b0 100755
--- a/scripts-dev/sign_json
+++ b/scripts-dev/sign_json
@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 #
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/scripts-dev/update_database b/scripts-dev/update_database
index 56365e2b58..87f709b6ed 100755
--- a/scripts-dev/update_database
+++ b/scripts-dev/update_database
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/scripts/export_signing_key b/scripts/export_signing_key
index 8aec9d802b..bf0139bd64 100755
--- a/scripts/export_signing_key
+++ b/scripts/export_signing_key
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -31,7 +30,11 @@ def exit(status: int = 0, message: Optional[str] = None):
 def format_plain(public_key: nacl.signing.VerifyKey):
     print(
         "%s:%s %s"
-        % (public_key.alg, public_key.version, encode_verify_key_base64(public_key),)
+        % (
+            public_key.alg,
+            public_key.version,
+            encode_verify_key_base64(public_key),
+        )
     )
 
 
@@ -51,7 +54,10 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 
     parser.add_argument(
-        "key_file", nargs="+", type=argparse.FileType("r"), help="The key file to read",
+        "key_file",
+        nargs="+",
+        type=argparse.FileType("r"),
+        help="The key file to read",
     )
 
     parser.add_argument(
@@ -64,7 +70,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--expiry-ts",
         type=int,
-        default=int(time.time() * 1000) + 6*3600000,
+        default=int(time.time() * 1000) + 6 * 3600000,
         help=(
             "The expiry time to use for -x, in milliseconds since 1970. The default "
             "is (now+6h)."
diff --git a/scripts/generate_config b/scripts/generate_config
index 771cbf8d95..931b40c045 100755
--- a/scripts/generate_config
+++ b/scripts/generate_config
@@ -11,23 +11,22 @@ if __name__ == "__main__":
     parser.add_argument(
         "--config-dir",
         default="CONFDIR",
-
         help="The path where the config files are kept. Used to create filenames for "
-             "things like the log config and the signing key. Default: %(default)s",
+        "things like the log config and the signing key. Default: %(default)s",
     )
 
     parser.add_argument(
         "--data-dir",
         default="DATADIR",
         help="The path where the data files are kept. Used to create filenames for "
-             "things like the database and media store. Default: %(default)s",
+        "things like the database and media store. Default: %(default)s",
     )
 
     parser.add_argument(
         "--server-name",
         default="SERVERNAME",
         help="The server name. Used to initialise the server_name config param, but also "
-             "used in the names of some of the config files. Default: %(default)s",
+        "used in the names of some of the config files. Default: %(default)s",
     )
 
     parser.add_argument(
@@ -41,21 +40,22 @@ if __name__ == "__main__":
         "--generate-secrets",
         action="store_true",
         help="Enable generation of new secrets for things like the macaroon_secret_key."
-             "By default, these parameters will be left unset."
+        "By default, these parameters will be left unset.",
     )
 
     parser.add_argument(
-        "-o", "--output-file",
-        type=argparse.FileType('w'),
+        "-o",
+        "--output-file",
+        type=argparse.FileType("w"),
         default=sys.stdout,
         help="File to write the configuration to. Default: stdout",
     )
 
     parser.add_argument(
         "--header-file",
-        type=argparse.FileType('r'),
+        type=argparse.FileType("r"),
         help="File from which to read a header, which will be printed before the "
-             "generated config.",
+        "generated config.",
     )
 
     args = parser.parse_args()
diff --git a/scripts/generate_log_config b/scripts/generate_log_config
index a13a5634a3..e72a0dafb7 100755
--- a/scripts/generate_log_config
+++ b/scripts/generate_log_config
@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/scripts/generate_signing_key.py b/scripts/generate_signing_key.py
index 16d7c4f382..07df25a809 100755
--- a/scripts/generate_signing_key.py
+++ b/scripts/generate_signing_key.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/scripts/hash_password b/scripts/hash_password
index a30767f758..1d6fb0d700 100755
--- a/scripts/hash_password
+++ b/scripts/hash_password
@@ -41,7 +41,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "-c",
         "--config",
-        type=argparse.FileType('r'),
+        type=argparse.FileType("r"),
         help=(
             "Path to server config file. "
             "Used to read in bcrypt_rounds and password_pepper."
@@ -72,8 +72,8 @@ if __name__ == "__main__":
     pw = unicodedata.normalize("NFKC", password)
 
     hashed = bcrypt.hashpw(
-        pw.encode('utf8') + password_pepper.encode("utf8"),
+        pw.encode("utf8") + password_pepper.encode("utf8"),
         bcrypt.gensalt(bcrypt_rounds),
-    ).decode('ascii')
+    ).decode("ascii")
 
     print(hashed)
diff --git a/scripts/move_remote_media_to_new_store.py b/scripts/move_remote_media_to_new_store.py
index 8477955a90..875aa4781f 100755
--- a/scripts/move_remote_media_to_new_store.py
+++ b/scripts/move_remote_media_to_new_store.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/scripts/register_new_matrix_user b/scripts/register_new_matrix_user
index 8b9d30877d..00104b9d62 100755
--- a/scripts/register_new_matrix_user
+++ b/scripts/register_new_matrix_user
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index bc1a989fc1..2c9a6460ec 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -95,6 +94,7 @@ BOOLEAN_COLUMNS = {
     "local_media_repository": ["safe_from_quarantine"],
     "users": ["shadow_banned"],
     "e2e_fallback_keys_json": ["used"],
+    "access_tokens": ["used"],
 }
 
 
@@ -297,8 +297,7 @@ class Porter(object):
         return table, already_ported, total_to_port, forward_chunk, backward_chunk
 
     async def get_table_constraints(self) -> Dict[str, Set[str]]:
-        """Returns a map of tables that have foreign key constraints to tables they depend on.
-        """
+        """Returns a map of tables that have foreign key constraints to tables they depend on."""
 
         def _get_constraints(txn):
             # We can pull the information about foreign key constraints out from
@@ -311,7 +310,8 @@ class Porter(object):
                     information_schema.table_constraints AS tc
                     INNER JOIN information_schema.constraint_column_usage AS ccu
                     USING (table_schema, constraint_name)
-                WHERE tc.constraint_type = 'FOREIGN KEY';
+                WHERE tc.constraint_type = 'FOREIGN KEY'
+                  AND tc.table_name != ccu.table_name;
             """
             txn.execute(sql)
 
@@ -507,7 +507,9 @@ class Porter(object):
                 return
 
     def build_db_store(
-        self, db_config: DatabaseConnectionConfig, allow_outdated_version: bool = False,
+        self,
+        db_config: DatabaseConnectionConfig,
+        allow_outdated_version: bool = False,
     ):
         """Builds and returns a database store using the provided configuration.
 
@@ -637,8 +639,11 @@ class Porter(object):
                 "device_inbox_sequence", ("device_inbox", "device_federation_outbox")
             )
             await self._setup_sequence(
-                "account_data_sequence", ("room_account_data", "room_tags_revisions", "account_data"))
-            await self._setup_sequence("receipts_sequence", ("receipts_linearized", ))
+                "account_data_sequence",
+                ("room_account_data", "room_tags_revisions", "account_data"),
+            )
+            await self._setup_sequence("receipts_sequence", ("receipts_linearized",))
+            await self._setup_sequence("presence_stream_sequence", ("presence_stream",))
             await self._setup_auth_chain_sequence()
 
             # Step 3. Get tables.
@@ -740,7 +745,7 @@ class Porter(object):
             return col
 
         outrows = []
-        for i, row in enumerate(rows):
+        for row in rows:
             try:
                 outrows.append(
                     tuple(conv(j, col) for j, col in enumerate(row) if j > 0)
@@ -890,8 +895,7 @@ class Porter(object):
         await self.postgres_store.db_pool.runInteraction("setup_user_id_seq", r)
 
     async def _setup_events_stream_seqs(self) -> None:
-        """Set the event stream sequences to the correct values.
-        """
+        """Set the event stream sequences to the correct values."""
 
         # We get called before we've ported the events table, so we need to
         # fetch the current positions from the SQLite store.
@@ -913,18 +917,21 @@ class Porter(object):
                     (curr_forward_id + 1,),
                 )
 
-            txn.execute(
-                "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
-                (curr_backward_id + 1,),
-            )
+            if curr_backward_id:
+                txn.execute(
+                    "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
+                    (curr_backward_id + 1,),
+                )
 
         await self.postgres_store.db_pool.runInteraction(
-            "_setup_events_stream_seqs", _setup_events_stream_seqs_set_pos,
+            "_setup_events_stream_seqs",
+            _setup_events_stream_seqs_set_pos,
         )
 
-    async def _setup_sequence(self, sequence_name: str, stream_id_tables: Iterable[str]) -> None:
-        """Set a sequence to the correct value.
-        """
+    async def _setup_sequence(
+        self, sequence_name: str, stream_id_tables: Iterable[str]
+    ) -> None:
+        """Set a sequence to the correct value."""
         current_stream_ids = []
         for stream_id_table in stream_id_tables:
             max_stream_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
@@ -938,26 +945,32 @@ class Porter(object):
         next_id = max(current_stream_ids) + 1
 
         def r(txn):
-            sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name, )
-            txn.execute(sql + " %s", (next_id, ))
+            sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name,)
+            txn.execute(sql + " %s", (next_id,))
 
-        await self.postgres_store.db_pool.runInteraction("_setup_%s" % (sequence_name,), r)
+        await self.postgres_store.db_pool.runInteraction(
+            "_setup_%s" % (sequence_name,), r
+        )
 
     async def _setup_auth_chain_sequence(self) -> None:
         curr_chain_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
-            table="event_auth_chains", keyvalues={}, retcol="MAX(chain_id)", allow_none=True
+            table="event_auth_chains",
+            keyvalues={},
+            retcol="MAX(chain_id)",
+            allow_none=True,
         )
 
         def r(txn):
             txn.execute(
                 "ALTER SEQUENCE event_auth_chain_id RESTART WITH %s",
-                (curr_chain_id,),
+                (curr_chain_id + 1,),
             )
 
-        await self.postgres_store.db_pool.runInteraction(
-            "_setup_event_auth_chain_id", r,
-        )
-
+        if curr_chain_id is not None:
+            await self.postgres_store.db_pool.runInteraction(
+                "_setup_event_auth_chain_id",
+                r,
+            )
 
 
 ##############################################
@@ -966,8 +979,7 @@ class Porter(object):
 
 
 class Progress(object):
-    """Used to report progress of the port
-    """
+    """Used to report progress of the port"""
 
     def __init__(self):
         self.tables = {}
@@ -992,8 +1004,7 @@ class Progress(object):
 
 
 class CursesProgress(Progress):
-    """Reports progress to a curses window
-    """
+    """Reports progress to a curses window"""
 
     def __init__(self, stdscr):
         self.stdscr = stdscr
@@ -1018,7 +1029,7 @@ class CursesProgress(Progress):
 
         self.total_processed = 0
         self.total_remaining = 0
-        for table, data in self.tables.items():
+        for data in self.tables.values():
             self.total_processed += data["num_done"] - data["start"]
             self.total_remaining += data["total"] - data["num_done"]
 
@@ -1109,8 +1120,7 @@ class CursesProgress(Progress):
 
 
 class TerminalProgress(Progress):
-    """Just prints progress to the terminal
-    """
+    """Just prints progress to the terminal"""
 
     def update(self, table, num_done):
         super(TerminalProgress, self).update(table, num_done)
diff --git a/synapse/replication/slave/storage/transactions.py b/scripts/synapse_review_recent_signups
old mode 100644
new mode 100755
similarity index 69%
rename from synapse/replication/slave/storage/transactions.py
rename to scripts/synapse_review_recent_signups
index 2091ac0df6..a36d46e14c
--- a/synapse/replication/slave/storage/transactions.py
+++ b/scripts/synapse_review_recent_signups
@@ -1,5 +1,5 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015, 2016 OpenMarket Ltd
+#!/usr/bin/env python
+# Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,10 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.storage.databases.main.transactions import TransactionStore
+from synapse._scripts.review_recent_signups import main
 
-from ._base import BaseSlavedStore
-
-
-class SlavedTransactionStore(TransactionStore, BaseSlavedStore):
-    pass
+if __name__ == "__main__":
+    main()
diff --git a/setup.cfg b/setup.cfg
index 7329eed213..e5ceb7ed19 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -18,16 +18,14 @@ ignore =
 #  E203: whitespace before ':' (which is contrary to pep8?)
 #  E731: do not assign a lambda expression, use a def
 #  E501: Line too long (black enforces this for us)
-#  B00*: Subsection of the bugbear suite (TODO: add in remaining fixes)
-ignore=W503,W504,E203,E731,E501,B006,B007,B008
+ignore=W503,W504,E203,E731,E501
 
 [isort]
 line_length = 88
-sections=FUTURE,STDLIB,COMPAT,THIRDPARTY,TWISTED,FIRSTPARTY,TESTS,LOCALFOLDER
+sections=FUTURE,STDLIB,THIRDPARTY,TWISTED,FIRSTPARTY,TESTS,LOCALFOLDER
 default_section=THIRDPARTY
 known_first_party = synapse
 known_tests=tests
-known_compat = mock
 known_twisted=twisted,OpenSSL
 multi_line_output=3
 include_trailing_comma=true
diff --git a/setup.py b/setup.py
index 29e9971dc1..c478563510 100755
--- a/setup.py
+++ b/setup.py
@@ -97,12 +97,21 @@ def exec_file(path_segments):
 # We pin black so that our tests don't start failing on new releases.
 CONDITIONAL_REQUIREMENTS["lint"] = [
     "isort==5.7.0",
-    "black==20.8b1",
+    "black==21.6b0",
     "flake8-comprehensions",
     "flake8-bugbear==21.3.2",
     "flake8",
 ]
 
+CONDITIONAL_REQUIREMENTS["dev"] = CONDITIONAL_REQUIREMENTS["lint"] + [
+    # The following are used by the release script
+    "click==7.1.2",
+    "redbaron==0.9.2",
+    "GitPython==3.1.14",
+    "commonmark==0.9.1",
+    "pygithub==1.55",
+]
+
 CONDITIONAL_REQUIREMENTS["mypy"] = ["mypy==0.812", "mypy-zope==0.2.13"]
 
 # Dependencies which are exclusively required by unit test code. This is
@@ -110,7 +119,7 @@ def exec_file(path_segments):
 # Tests assume that all optional dependencies are installed.
 #
 # parameterized_class decorator was introduced in parameterized 0.7.0
-CONDITIONAL_REQUIREMENTS["test"] = ["mock>=2.0", "parameterized>=0.7.0"]
+CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0"]
 
 setup(
     name="matrix-synapse",
@@ -123,13 +132,12 @@ def exec_file(path_segments):
     zip_safe=False,
     long_description=long_description,
     long_description_content_type="text/x-rst",
-    python_requires="~=3.5",
+    python_requires="~=3.6",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Topic :: Communications :: Chat",
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
diff --git a/stubs/frozendict.pyi b/stubs/frozendict.pyi
index 0368ba4703..24c6f3af77 100644
--- a/stubs/frozendict.pyi
+++ b/stubs/frozendict.pyi
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/stubs/txredisapi.pyi b/stubs/txredisapi.pyi
index 080ca40287..c1a06ae022 100644
--- a/stubs/txredisapi.pyi
+++ b/stubs/txredisapi.pyi
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 1d2883acf6..06d80f79b3 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018-9 New Vector Ltd
 #
@@ -22,8 +21,8 @@
 import sys
 
 # Check that we're not running on an unsupported Python version.
-if sys.version_info < (3, 5):
-    print("Synapse requires Python 3.5 or above.")
+if sys.version_info < (3, 6):
+    print("Synapse requires Python 3.6 or above.")
     sys.exit(1)
 
 # Twisted and canonicaljson will fail to import when this file is executed to
@@ -48,7 +47,7 @@
 except ImportError:
     pass
 
-__version__ = "1.31.0"
+__version__ = "1.41.1"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when
diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py
index dfe26dea6d..dae986c788 100644
--- a/synapse/_scripts/register_new_matrix_user.py
+++ b/synapse/_scripts/register_new_matrix_user.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector
 #
diff --git a/synapse/_scripts/review_recent_signups.py b/synapse/_scripts/review_recent_signups.py
new file mode 100644
index 0000000000..9de913db88
--- /dev/null
+++ b/synapse/_scripts/review_recent_signups.py
@@ -0,0 +1,174 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+import time
+from datetime import datetime
+from typing import List
+
+import attr
+
+from synapse.config._base import RootConfig, find_config_files, read_config_files
+from synapse.config.database import DatabaseConfig
+from synapse.storage.database import DatabasePool, LoggingTransaction, make_conn
+from synapse.storage.engines import create_engine
+
+
+class ReviewConfig(RootConfig):
+    "A config class that just pulls out the database config"
+    config_classes = [DatabaseConfig]
+
+
+@attr.s(auto_attribs=True)
+class UserInfo:
+    user_id: str
+    creation_ts: int
+    emails: List[str] = attr.Factory(list)
+    private_rooms: List[str] = attr.Factory(list)
+    public_rooms: List[str] = attr.Factory(list)
+    ips: List[str] = attr.Factory(list)
+
+
+def get_recent_users(txn: LoggingTransaction, since_ms: int) -> List[UserInfo]:
+    """Fetches recently registered users and some info on them."""
+
+    sql = """
+        SELECT name, creation_ts FROM users
+        WHERE
+            ? <= creation_ts
+            AND deactivated = 0
+    """
+
+    txn.execute(sql, (since_ms / 1000,))
+
+    user_infos = [UserInfo(user_id, creation_ts) for user_id, creation_ts in txn]
+
+    for user_info in user_infos:
+        user_info.emails = DatabasePool.simple_select_onecol_txn(
+            txn,
+            table="user_threepids",
+            keyvalues={"user_id": user_info.user_id, "medium": "email"},
+            retcol="address",
+        )
+
+        sql = """
+            SELECT room_id, canonical_alias, name, join_rules
+            FROM local_current_membership
+            INNER JOIN room_stats_state USING (room_id)
+            WHERE user_id = ? AND membership = 'join'
+        """
+
+        txn.execute(sql, (user_info.user_id,))
+        for room_id, canonical_alias, name, join_rules in txn:
+            if join_rules == "public":
+                user_info.public_rooms.append(canonical_alias or name or room_id)
+            else:
+                user_info.private_rooms.append(canonical_alias or name or room_id)
+
+        user_info.ips = DatabasePool.simple_select_onecol_txn(
+            txn,
+            table="user_ips",
+            keyvalues={"user_id": user_info.user_id},
+            retcol="ip",
+        )
+
+    return user_infos
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c",
+        "--config-path",
+        action="append",
+        metavar="CONFIG_FILE",
+        help="The config files for Synapse.",
+        required=True,
+    )
+    parser.add_argument(
+        "-s",
+        "--since",
+        metavar="duration",
+        help="Specify how far back to review user registrations for, defaults to 7d (i.e. 7 days).",
+        default="7d",
+    )
+    parser.add_argument(
+        "-e",
+        "--exclude-emails",
+        action="store_true",
+        help="Exclude users that have validated email addresses",
+    )
+    parser.add_argument(
+        "-u",
+        "--only-users",
+        action="store_true",
+        help="Only print user IDs that match.",
+    )
+
+    config = ReviewConfig()
+
+    config_args = parser.parse_args(sys.argv[1:])
+    config_files = find_config_files(search_paths=config_args.config_path)
+    config_dict = read_config_files(config_files)
+    config.parse_config_dict(
+        config_dict,
+    )
+
+    since_ms = time.time() * 1000 - config.parse_duration(config_args.since)
+    exclude_users_with_email = config_args.exclude_emails
+    include_context = not config_args.only_users
+
+    for database_config in config.database.databases:
+        if "main" in database_config.databases:
+            break
+
+    engine = create_engine(database_config.config)
+
+    with make_conn(database_config, engine, "review_recent_signups") as db_conn:
+        user_infos = get_recent_users(db_conn.cursor(), since_ms)
+
+    for user_info in user_infos:
+        if exclude_users_with_email and user_info.emails:
+            continue
+
+        if include_context:
+            print_public_rooms = ""
+            if user_info.public_rooms:
+                print_public_rooms = "(" + ", ".join(user_info.public_rooms[:3])
+
+                if len(user_info.public_rooms) > 3:
+                    print_public_rooms += ", ..."
+
+                print_public_rooms += ")"
+
+            print("# Created:", datetime.fromtimestamp(user_info.creation_ts))
+            print("# Email:", ", ".join(user_info.emails) or "None")
+            print("# IPs:", ", ".join(user_info.ips))
+            print(
+                "# Number joined public rooms:",
+                len(user_info.public_rooms),
+                print_public_rooms,
+            )
+            print("# Number joined private rooms:", len(user_info.private_rooms))
+            print("#")
+
+        print(user_info.user_id)
+
+        if include_context:
+            print()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/synapse/api/__init__.py b/synapse/api/__init__.py
index bfebb0f644..5e83dba2ed 100644
--- a/synapse/api/__init__.py
+++ b/synapse/api/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 26cb1bc657..4635a04e94 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,14 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import List, Optional, Tuple
+from typing import TYPE_CHECKING, Optional, Tuple
 
 import pymacaroons
 from netaddr import IPAddress
 
 from twisted.web.server import Request
 
-import synapse.types
 from synapse import event_auth
 from synapse.api.auth_blocking import AuthBlocking
 from synapse.api.constants import EventTypes, HistoryVisibility, Membership
@@ -30,29 +28,21 @@
     InvalidClientTokenError,
     MissingClientTokenError,
 )
-from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
 from synapse.appservice import ApplicationService
 from synapse.events import EventBase
 from synapse.http import get_request_user_agent
 from synapse.http.site import SynapseRequest
 from synapse.logging import opentracing as opentracing
 from synapse.storage.databases.main.registration import TokenLookupResult
-from synapse.types import StateMap, UserID
+from synapse.types import Requester, StateMap, UserID, create_requester
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.macaroons import get_value_from_macaroon, satisfy_expiry
-from synapse.util.metrics import Measure
 
-logger = logging.getLogger(__name__)
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
 
+logger = logging.getLogger(__name__)
 
-AuthEventTypes = (
-    EventTypes.Create,
-    EventTypes.Member,
-    EventTypes.PowerLevels,
-    EventTypes.JoinRules,
-    EventTypes.RoomHistoryVisibility,
-    EventTypes.ThirdPartyInvite,
-)
 
 # guests always get this device id.
 GUEST_DEVICE_ID = "guest_device"
@@ -64,40 +54,25 @@ class _InvalidMacaroonException(Exception):
 
 class Auth:
     """
-    FIXME: This class contains a mix of functions for authenticating users
-    of our client-server API and authenticating events added to room graphs.
+    This class contains functions for authenticating users of our client-server API.
     """
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.hs = hs
         self.clock = hs.get_clock()
         self.store = hs.get_datastore()
         self.state = hs.get_state_handler()
+        self._account_validity_handler = hs.get_account_validity_handler()
 
-        self.token_cache = LruCache(
+        self.token_cache: LruCache[str, Tuple[str, bool]] = LruCache(
             10000, "token_cache"
-        )  # type: LruCache[str, Tuple[str, bool]]
+        )
 
         self._auth_blocking = AuthBlocking(self.hs)
 
-        self._account_validity_enabled = hs.config.account_validity_enabled
         self._track_appservice_user_ips = hs.config.track_appservice_user_ips
         self._macaroon_secret_key = hs.config.macaroon_secret_key
-
-    async def check_from_context(
-        self, room_version: str, event, context, do_sig_check=True
-    ):
-        prev_state_ids = await context.get_prev_state_ids()
-        auth_events_ids = self.compute_auth_events(
-            event, prev_state_ids, for_verification=True
-        )
-        auth_events = await self.store.get_events(auth_events_ids)
-        auth_events = {(e.type, e.state_key): e for e in auth_events.values()}
-
-        room_version_obj = KNOWN_ROOM_VERSIONS[room_version]
-        event_auth.check(
-            room_version_obj, event, auth_events=auth_events, do_sig_check=do_sig_check
-        )
+        self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users
 
     async def check_user_in_room(
         self,
@@ -149,26 +124,13 @@ async def check_user_in_room(
 
         raise AuthError(403, "User %s not in room %s" % (user_id, room_id))
 
-    async def check_host_in_room(self, room_id, host):
-        with Measure(self.clock, "check_host_in_room"):
-            latest_event_ids = await self.store.is_host_joined(room_id, host)
-            return latest_event_ids
-
-    def can_federate(self, event, auth_events):
-        creation_event = auth_events.get((EventTypes.Create, ""))
-
-        return creation_event.content.get("m.federate", True) is True
-
-    def get_public_keys(self, invite_event):
-        return event_auth.get_public_keys(invite_event)
-
     async def get_user_by_req(
         self,
         request: SynapseRequest,
         allow_guest: bool = False,
         rights: str = "access",
         allow_expired: bool = False,
-    ) -> synapse.types.Requester:
+    ) -> Requester:
         """Get a registered user's ID.
 
         Args:
@@ -193,8 +155,8 @@ async def get_user_by_req(
 
             access_token = self.get_access_token_from_request(request)
 
-            user_id, app_service = self._get_appservice_user_id(request)
-            if user_id:
+            user_id, app_service = await self._get_appservice_user_id(request)
+            if user_id and app_service:
                 if ip_addr and self._track_appservice_user_ips:
                     await self.store.insert_client_ip(
                         user_id=user_id,
@@ -204,11 +166,11 @@ async def get_user_by_req(
                         device_id="dummy-device",  # stubbed
                     )
 
-                requester = synapse.types.create_requester(
-                    user_id, app_service=app_service
-                )
+                requester = create_requester(user_id, app_service=app_service)
 
                 request.requester = user_id
+                if user_id in self._force_tracing_for_users:
+                    opentracing.force_tracing()
                 opentracing.set_tag("authenticated_entity", user_id)
                 opentracing.set_tag("user_id", user_id)
                 opentracing.set_tag("appservice_id", app_service.id)
@@ -223,12 +185,17 @@ async def get_user_by_req(
             shadow_banned = user_info.shadow_banned
 
             # Deny the request if the user account has expired.
-            if self._account_validity_enabled and not allow_expired:
-                if await self.store.is_account_expired(
-                    user_info.user_id, self.clock.time_msec()
+            if not allow_expired:
+                if await self._account_validity_handler.is_user_expired(
+                    user_info.user_id
                 ):
+                    # Raise the error if either an account validity module has determined
+                    # the account has expired, or the legacy account validity
+                    # implementation is enabled and determined the account has expired
                     raise AuthError(
-                        403, "User account has expired", errcode=Codes.EXPIRED_ACCOUNT
+                        403,
+                        "User account has expired",
+                        errcode=Codes.EXPIRED_ACCOUNT,
                     )
 
             device_id = user_info.device_id
@@ -249,7 +216,12 @@ async def get_user_by_req(
                     errcode=Codes.GUEST_ACCESS_FORBIDDEN,
                 )
 
-            requester = synapse.types.create_requester(
+            # Mark the token as used. This is used to invalidate old refresh
+            # tokens after some time.
+            if not user_info.token_used and token_id is not None:
+                await self.store.mark_access_token_as_used(token_id)
+
+            requester = create_requester(
                 user_info.user_id,
                 token_id,
                 is_guest,
@@ -260,6 +232,8 @@ async def get_user_by_req(
             )
 
             request.requester = requester
+            if user_info.token_owner in self._force_tracing_for_users:
+                opentracing.force_tracing()
             opentracing.set_tag("authenticated_entity", user_info.token_owner)
             opentracing.set_tag("user_id", user_info.user_id)
             if device_id:
@@ -269,7 +243,41 @@ async def get_user_by_req(
         except KeyError:
             raise MissingClientTokenError()
 
-    def _get_appservice_user_id(self, request):
+    async def validate_appservice_can_control_user_id(
+        self, app_service: ApplicationService, user_id: str
+    ):
+        """Validates that the app service is allowed to control
+        the given user.
+
+        Args:
+            app_service: The app service that controls the user
+            user_id: The author MXID that the app service is controlling
+
+        Raises:
+            AuthError: If the application service is not allowed to control the user
+                (user namespace regex does not match, wrong homeserver, etc)
+                or if the user has not been registered yet.
+        """
+
+        # It's ok if the app service is trying to use the sender from their registration
+        if app_service.sender == user_id:
+            pass
+        # Check to make sure the app service is allowed to control the user
+        elif not app_service.is_interested_in_user(user_id):
+            raise AuthError(
+                403,
+                "Application service cannot masquerade as this user (%s)." % user_id,
+            )
+        # Let ASes manipulate nonexistent users (e.g. to shadow-register them)
+        # Check to make sure the user is already registered on the homeserver
+        # elif not (await self.store.get_user_by_id(user_id)):
+        #     raise AuthError(
+        #         403, "Application service has not registered this user (%s)" % user_id
+        #     )
+
+    async def _get_appservice_user_id(
+        self, request: Request
+    ) -> Tuple[Optional[str], Optional[ApplicationService]]:
         app_service = self.store.get_app_service_by_token(
             self.get_access_token_from_request(request)
         )
@@ -282,21 +290,18 @@ def _get_appservice_user_id(self, request):
             if ip_address not in app_service.ip_range_whitelist:
                 return None, None
 
+        # This will always be set by the time Twisted calls us.
+        assert request.args is not None
+
         if b"user_id" not in request.args:
             return app_service.sender, app_service
 
         user_id = request.args[b"user_id"][0].decode("utf8")
+        await self.validate_appservice_can_control_user_id(app_service, user_id)
+
         if app_service.sender == user_id:
             return app_service.sender, app_service
 
-        if not app_service.is_interested_in_user(user_id):
-            raise AuthError(403, "Application service cannot masquerade as this user.")
-        # Let ASes manipulate nonexistent users (e.g. to shadow-register them)
-        # if not (yield self.store.get_user_by_id(user_id)):
-        #     raise AuthError(
-        #         403,
-        #         "Application service has not registered this user"
-        #     )
         return user_id, app_service
 
     async def get_user_by_access_token(
@@ -390,7 +395,9 @@ async def get_user_by_access_token(
             logger.warning("Invalid macaroon in auth: %s %s", type(e), e)
             raise InvalidClientTokenError("Invalid macaroon passed.")
 
-    def _parse_and_validate_macaroon(self, token, rights="access"):
+    def _parse_and_validate_macaroon(
+        self, token: str, rights: str = "access"
+    ) -> Tuple[str, bool]:
         """Takes a macaroon and tries to parse and validate it. This is cached
         if and only if rights == access and there isn't an expiry.
 
@@ -435,15 +442,16 @@ def _parse_and_validate_macaroon(self, token, rights="access"):
 
         return user_id, guest
 
-    def validate_macaroon(self, macaroon, type_string, user_id):
+    def validate_macaroon(
+        self, macaroon: pymacaroons.Macaroon, type_string: str, user_id: str
+    ) -> None:
         """
         validate that a Macaroon is understood by and was signed by this server.
 
         Args:
-            macaroon(pymacaroons.Macaroon): The macaroon to validate
-            type_string(str): The kind of token required (e.g. "access",
-                              "delete_pusher")
-            user_id (str): The user_id required
+            macaroon: The macaroon to validate
+            type_string: The kind of token required (e.g. "access", "delete_pusher")
+            user_id: The user_id required
         """
         v = pymacaroons.Verifier()
 
@@ -468,9 +476,7 @@ def get_appservice_by_req(self, request: SynapseRequest) -> ApplicationService:
         if not service:
             logger.warning("Unrecognised appservice access token.")
             raise InvalidClientTokenError()
-        request.requester = synapse.types.create_requester(
-            service.sender, app_service=service
-        )
+        request.requester = create_requester(service.sender, app_service=service)
         return service
 
     async def is_server_admin(self, user: UserID) -> bool:
@@ -484,45 +490,7 @@ async def is_server_admin(self, user: UserID) -> bool:
         """
         return await self.store.is_server_admin(user)
 
-    def compute_auth_events(
-        self,
-        event,
-        current_state_ids: StateMap[str],
-        for_verification: bool = False,
-    ) -> List[str]:
-        """Given an event and current state return the list of event IDs used
-        to auth an event.
-
-        If `for_verification` is False then only return auth events that
-        should be added to the event's `auth_events`.
-
-        Returns:
-            List of event IDs.
-        """
-
-        if event.type == EventTypes.Create:
-            return []
-
-        # Currently we ignore the `for_verification` flag even though there are
-        # some situations where we can drop particular auth events when adding
-        # to the event's `auth_events` (e.g. joins pointing to previous joins
-        # when room is publicly joinable). Dropping event IDs has the
-        # advantage that the auth chain for the room grows slower, but we use
-        # the auth chain in state resolution v2 to order events, which means
-        # care must be taken if dropping events to ensure that it doesn't
-        # introduce undesirable "state reset" behaviour.
-        #
-        # All of which sounds a bit tricky so we don't bother for now.
-
-        auth_ids = []
-        for etype, state_key in event_auth.auth_types_for_event(event):
-            auth_ev_id = current_state_ids.get((etype, state_key))
-            if auth_ev_id:
-                auth_ids.append(auth_ev_id)
-
-        return auth_ids
-
-    async def check_can_change_room_list(self, room_id: str, user: UserID):
+    async def check_can_change_room_list(self, room_id: str, user: UserID) -> bool:
         """Determine whether the user is allowed to edit the room's entry in the
         published room list.
 
@@ -557,11 +525,11 @@ async def check_can_change_room_list(self, room_id: str, user: UserID):
         return user_level >= send_level
 
     @staticmethod
-    def has_access_token(request: Request):
+    def has_access_token(request: Request) -> bool:
         """Checks if the request has an access_token.
 
         Returns:
-            bool: False if no access_token was given, True otherwise.
+            False if no access_token was given, True otherwise.
         """
         # This will always be set by the time Twisted calls us.
         assert request.args is not None
@@ -571,13 +539,13 @@ def has_access_token(request: Request):
         return bool(query_params) or bool(auth_headers)
 
     @staticmethod
-    def get_access_token_from_request(request: Request):
+    def get_access_token_from_request(request: Request) -> str:
         """Extracts the access_token from the request.
 
         Args:
             request: The http request.
         Returns:
-            unicode: The access_token
+            The access_token
         Raises:
             MissingClientTokenError: If there isn't a single access_token in the
                 request
@@ -652,5 +620,5 @@ async def check_user_in_room_or_world_readable(
                 % (user_id, room_id),
             )
 
-    def check_auth_blocking(self, *args, **kwargs):
-        return self._auth_blocking.check_auth_blocking(*args, **kwargs)
+    async def check_auth_blocking(self, *args, **kwargs) -> None:
+        await self._auth_blocking.check_auth_blocking(*args, **kwargs)
diff --git a/synapse/api/auth_blocking.py b/synapse/api/auth_blocking.py
index d8088f524a..e6bced93d5 100644
--- a/synapse/api/auth_blocking.py
+++ b/synapse/api/auth_blocking.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,18 +13,21 @@
 # limitations under the License.
 
 import logging
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 from synapse.api.constants import LimitBlockingTypes, UserTypes
 from synapse.api.errors import Codes, ResourceLimitError
 from synapse.config.server import is_threepid_reserved
 from synapse.types import Requester
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
 class AuthBlocking:
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastore()
 
         self._server_notices_mxid = hs.config.server_notices_mxid
@@ -44,7 +46,7 @@ async def check_auth_blocking(
         threepid: Optional[dict] = None,
         user_type: Optional[str] = None,
         requester: Optional[Requester] = None,
-    ):
+    ) -> None:
         """Checks if the user should be rejected for some external reason,
         such as monthly active user limiting or global disable flag
 
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 8f37d2cf3b..e0e24fddac 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018-2019 New Vector Ltd
@@ -18,6 +17,9 @@
 
 """Contains constants from the specification."""
 
+# the max size of a (canonical-json-encoded) event
+MAX_PDU_SIZE = 65536
+
 # the "depth" field on events is limited to 2**63 - 1
 MAX_DEPTH = 2 ** 63 - 1
 
@@ -59,6 +61,14 @@ class JoinRules:
     KNOCK = "knock"
     INVITE = "invite"
     PRIVATE = "private"
+    # As defined for MSC3083.
+    RESTRICTED = "restricted"
+
+
+class RestrictedJoinRuleTypes:
+    """Understood types for the allow rules in restricted join rules."""
+
+    ROOM_MEMBERSHIP = "m.room_membership"
 
 
 class LoginType:
@@ -71,6 +81,11 @@ class LoginType:
     DUMMY = "m.login.dummy"
 
 
+# This is used in the `type` parameter for /register when called by
+# an appservice to register a new user.
+APP_SERVICE_REGISTRATION_TYPE = "m.login.application_service"
+
+
 class EventTypes:
     Member = "m.room.member"
     Create = "m.room.create"
@@ -101,13 +116,28 @@ class EventTypes:
 
     Dummy = "org.matrix.dummy_event"
 
-    MSC1772_SPACE_CHILD = "org.matrix.msc1772.space.child"
-    MSC1772_SPACE_PARENT = "org.matrix.msc1772.space.parent"
+    SpaceChild = "m.space.child"
+    SpaceParent = "m.space.parent"
+
+    MSC2716_INSERTION = "org.matrix.msc2716.insertion"
+    MSC2716_CHUNK = "org.matrix.msc2716.chunk"
+    MSC2716_MARKER = "org.matrix.msc2716.marker"
+
+
+class ToDeviceEventTypes:
+    RoomKeyRequest = "m.room_key_request"
+
+
+class DeviceKeyAlgorithms:
+    """Spec'd algorithms for the generation of per-device keys"""
+
+    ED25519 = "ed25519"
+    CURVE25519 = "curve25519"
+    SIGNED_CURVE25519 = "signed_curve25519"
 
 
 class EduTypes:
     Presence = "m.presence"
-    RoomKeyRequest = "m.room_key_request"
 
 
 class RejectedReason:
@@ -165,7 +195,23 @@ class EventContentFields:
     SELF_DESTRUCT_AFTER = "org.matrix.self_destruct_after"
 
     # cf https://github.com/matrix-org/matrix-doc/pull/1772
-    MSC1772_ROOM_TYPE = "org.matrix.msc1772.type"
+    ROOM_TYPE = "type"
+
+    # Used on normal messages to indicate they were historically imported after the fact
+    MSC2716_HISTORICAL = "org.matrix.msc2716.historical"
+    # For "insertion" events to indicate what the next chunk ID should be in
+    # order to connect to it
+    MSC2716_NEXT_CHUNK_ID = "org.matrix.msc2716.next_chunk_id"
+    # Used on "chunk" events to indicate which insertion event it connects to
+    MSC2716_CHUNK_ID = "org.matrix.msc2716.chunk_id"
+    # For "marker" events
+    MSC2716_MARKER_INSERTION = "org.matrix.msc2716.marker.insertion"
+
+
+class RoomTypes:
+    """Understood values of the room_type field of m.room.create events."""
+
+    SPACE = "m.space"
 
 
 class RoomEncryptionAlgorithms:
@@ -183,3 +229,7 @@ class HistoryVisibility:
     JOINED = "joined"
     SHARED = "shared"
     WORLD_READABLE = "world_readable"
+
+
+class ReadReceiptEventFields:
+    MSC2285_HIDDEN = "org.matrix.msc2285.hidden"
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index a71e518f90..dfb7d358c9 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -77,6 +76,9 @@ class Codes:
     INVALID_SIGNATURE = "M_INVALID_SIGNATURE"
     USER_DEACTIVATED = "M_USER_DEACTIVATED"
     BAD_ALIAS = "M_BAD_ALIAS"
+    # For restricted join rules.
+    UNABLE_AUTHORISE_JOIN = "M_UNABLE_TO_AUTHORISE_JOIN"
+    UNABLE_TO_GRANT_JOIN = "M_UNABLE_TO_GRANT_JOIN"
 
 
 class CodeMessageException(RuntimeError):
@@ -120,7 +122,7 @@ def __init__(self, location: bytes, http_code: int = http.FOUND):
         super().__init__(code=http_code, msg=msg)
         self.location = location
 
-        self.cookies = []  # type: List[bytes]
+        self.cookies: List[bytes] = []
 
 
 class SynapseError(CodeMessageException):
@@ -162,7 +164,7 @@ def __init__(
     ):
         super().__init__(code, msg, errcode)
         if additional_fields is None:
-            self._additional_fields = {}  # type: Dict
+            self._additional_fields: Dict = {}
         else:
             self._additional_fields = dict(additional_fields)
 
@@ -451,7 +453,7 @@ def __init__(self, room_version: str):
         super().__init__(
             code=400,
             msg="Your homeserver does not support the features required to "
-            "join this room",
+            "interact with this room",
             errcode=Codes.INCOMPATIBLE_ROOM_VERSION,
         )
 
diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 5caf336fd0..ad1ff6a9df 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018-2019 New Vector Ltd
@@ -290,7 +289,7 @@ def check(self, event):
             room_id = None
             ev_type = "m.presence"
             contains_url = False
-            labels = []  # type: List[str]
+            labels: List[str] = []
         else:
             sender = event.get("sender", None)
             if not sender:
diff --git a/synapse/api/presence.py b/synapse/api/presence.py
index b9a8e29460..a3bf0348d1 100644
--- a/synapse/api/presence.py
+++ b/synapse/api/presence.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py
index c3f07bc1a3..3e3d09bbd2 100644
--- a/synapse/api/ratelimiting.py
+++ b/synapse/api/ratelimiting.py
@@ -17,6 +17,7 @@
 from typing import Hashable, Optional, Tuple
 
 from synapse.api.errors import LimitExceededError
+from synapse.storage.databases.main import DataStore
 from synapse.types import Requester
 from synapse.util import Clock
 
@@ -31,10 +32,13 @@ class Ratelimiter:
         burst_count: How many actions that can be performed before being limited.
     """
 
-    def __init__(self, clock: Clock, rate_hz: float, burst_count: int):
+    def __init__(
+        self, store: DataStore, clock: Clock, rate_hz: float, burst_count: int
+    ):
         self.clock = clock
         self.rate_hz = rate_hz
         self.burst_count = burst_count
+        self.store = store
 
         # A ordered dictionary keeping track of actions, when they were last
         # performed and how often. Each entry is a mapping from a key of arbitrary type
@@ -42,64 +46,38 @@ def __init__(self, clock: Clock, rate_hz: float, burst_count: int):
         #   * How many times an action has occurred since a point in time
         #   * The point in time
         #   * The rate_hz of this particular entry. This can vary per request
-        self.actions = (
-            OrderedDict()
-        )  # type: OrderedDict[Hashable, Tuple[float, int, float]]
-
-    def can_requester_do_action(
-        self,
-        requester: Requester,
-        rate_hz: Optional[float] = None,
-        burst_count: Optional[int] = None,
-        update: bool = True,
-        _time_now_s: Optional[int] = None,
-    ) -> Tuple[bool, float]:
-        """Can the requester perform the action?
-
-        Args:
-            requester: The requester to key off when rate limiting. The user property
-                will be used.
-            rate_hz: The long term number of actions that can be performed in a second.
-                Overrides the value set during instantiation if set.
-            burst_count: How many actions that can be performed before being limited.
-                Overrides the value set during instantiation if set.
-            update: Whether to count this check as performing the action
-            _time_now_s: The current time. Optional, defaults to the current time according
-                to self.clock. Only used by tests.
-
-        Returns:
-            A tuple containing:
-                * A bool indicating if they can perform the action now
-                * The reactor timestamp for when the action can be performed next.
-                  -1 if rate_hz is less than or equal to zero
-        """
-        # Disable rate limiting of users belonging to any AS that is configured
-        # not to be rate limited in its registration file (rate_limited: true|false).
-        if requester.app_service and not requester.app_service.is_rate_limited():
-            return True, -1.0
+        self.actions: OrderedDict[Hashable, Tuple[float, int, float]] = OrderedDict()
 
-        return self.can_do_action(
-            requester.user.to_string(), rate_hz, burst_count, update, _time_now_s
-        )
-
-    def can_do_action(
+    async def can_do_action(
         self,
-        key: Hashable,
+        requester: Optional[Requester],
+        key: Optional[Hashable] = None,
         rate_hz: Optional[float] = None,
         burst_count: Optional[int] = None,
         update: bool = True,
+        n_actions: int = 1,
         _time_now_s: Optional[int] = None,
     ) -> Tuple[bool, float]:
         """Can the entity (e.g. user or IP address) perform the action?
 
+        Checks if the user has ratelimiting disabled in the database by looking
+        for null/zero values in the `ratelimit_override` table. (Non-zero
+        values aren't honoured, as they're specific to the event sending
+        ratelimiter, rather than all ratelimiters)
+
         Args:
-            key: The key we should use when rate limiting. Can be a user ID
-                (when sending events), an IP address, etc.
+            requester: The requester that is doing the action, if any. Used to check
+                if the user has ratelimits disabled in the database.
+            key: An arbitrary key used to classify an action. Defaults to the
+                requester's user ID.
             rate_hz: The long term number of actions that can be performed in a second.
                 Overrides the value set during instantiation if set.
             burst_count: How many actions that can be performed before being limited.
                 Overrides the value set during instantiation if set.
             update: Whether to count this check as performing the action
+            n_actions: The number of times the user wants to do this action. If the user
+                cannot do all of the actions, the user's action count is not incremented
+                at all.
             _time_now_s: The current time. Optional, defaults to the current time according
                 to self.clock. Only used by tests.
 
@@ -109,6 +87,30 @@ def can_do_action(
                 * The reactor timestamp for when the action can be performed next.
                   -1 if rate_hz is less than or equal to zero
         """
+        if key is None:
+            if not requester:
+                raise ValueError("Must supply at least one of `requester` or `key`")
+
+            key = requester.user.to_string()
+
+        if requester:
+            # Disable rate limiting of users belonging to any AS that is configured
+            # not to be rate limited in its registration file (rate_limited: true|false).
+            if requester.app_service and not requester.app_service.is_rate_limited():
+                return True, -1.0
+
+            # Check if ratelimiting has been disabled for the user.
+            #
+            # Note that we don't use the returned rate/burst count, as the table
+            # is specifically for the event sending ratelimiter. Instead, we
+            # only use it to (somewhat cheekily) infer whether the user should
+            # be subject to any rate limiting or not.
+            override = await self.store.get_ratelimit_for_user(
+                requester.authenticated_entity
+            )
+            if override and not override.messages_per_second:
+                return True, -1.0
+
         # Override default values if set
         time_now_s = _time_now_s if _time_now_s is not None else self.clock.time()
         rate_hz = rate_hz if rate_hz is not None else self.rate_hz
@@ -124,17 +126,20 @@ def can_do_action(
         time_delta = time_now_s - time_start
         performed_count = action_count - time_delta * rate_hz
         if performed_count < 0:
-            # Allow, reset back to count 1
-            allowed = True
+            performed_count = 0
             time_start = time_now_s
-            action_count = 1.0
-        elif performed_count > burst_count - 1.0:
+
+        # This check would be easier read as performed_count + n_actions > burst_count,
+        # but performed_count might be a very precise float (with lots of numbers
+        # following the point) in which case Python might round it up when adding it to
+        # n_actions. Writing it this way ensures it doesn't happen.
+        if performed_count > burst_count - n_actions:
             # Deny, we have exceeded our burst count
             allowed = False
         else:
             # We haven't reached our limit yet
             allowed = True
-            action_count += 1.0
+            action_count = performed_count + n_actions
 
         if update:
             self.actions[key] = (action_count, time_start, rate_hz)
@@ -175,23 +180,36 @@ def _prune_message_counts(self, time_now_s: int):
             else:
                 del self.actions[key]
 
-    def ratelimit(
+    async def ratelimit(
         self,
-        key: Hashable,
+        requester: Optional[Requester],
+        key: Optional[Hashable] = None,
         rate_hz: Optional[float] = None,
         burst_count: Optional[int] = None,
         update: bool = True,
+        n_actions: int = 1,
         _time_now_s: Optional[int] = None,
     ):
         """Checks if an action can be performed. If not, raises a LimitExceededError
 
+        Checks if the user has ratelimiting disabled in the database by looking
+        for null/zero values in the `ratelimit_override` table. (Non-zero
+        values aren't honoured, as they're specific to the event sending
+        ratelimiter, rather than all ratelimiters)
+
         Args:
-            key: An arbitrary key used to classify an action
+            requester: The requester that is doing the action, if any. Used to check for
+                if the user has ratelimits disabled.
+            key: An arbitrary key used to classify an action. Defaults to the
+                requester's user ID.
             rate_hz: The long term number of actions that can be performed in a second.
                 Overrides the value set during instantiation if set.
             burst_count: How many actions that can be performed before being limited.
                 Overrides the value set during instantiation if set.
             update: Whether to count this check as performing the action
+            n_actions: The number of times the user wants to do this action. If the user
+                cannot do all of the actions, the user's action count is not incremented
+                at all.
             _time_now_s: The current time. Optional, defaults to the current time according
                 to self.clock. Only used by tests.
 
@@ -201,11 +219,13 @@ def ratelimit(
         """
         time_now_s = _time_now_s if _time_now_s is not None else self.clock.time()
 
-        allowed, time_allowed = self.can_do_action(
+        allowed, time_allowed = await self.can_do_action(
+            requester,
             key,
             rate_hz=rate_hz,
             burst_count=burst_count,
             update=update,
+            n_actions=n_actions,
             _time_now_s=time_now_s,
         )
 
diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py
index 139fbf5524..f50fd81a7c 100644
--- a/synapse/api/room_versions.py
+++ b/synapse/api/room_versions.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Dict
+from typing import Callable, Dict, Optional
 
 import attr
 
@@ -69,11 +68,16 @@ class RoomVersion:
     limit_notifications_power_levels = attr.ib(type=bool)
     # MSC2174/MSC2176: Apply updated redaction rules algorithm.
     msc2176_redaction_rules = attr.ib(type=bool)
-    # MSC2174/MSC2176: Apply updated redaction rules algorithm.
-    msc2176_redaction_rules = attr.ib(type=bool)
+    # MSC3083: Support the 'restricted' join_rule.
+    msc3083_join_rules = attr.ib(type=bool)
     # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
     # m.room.membership event with membership 'knock'.
-    allow_knocking = attr.ib(type=bool)
+    msc2403_knocking = attr.ib(type=bool)
+    # MSC2716: Adds m.room.power_levels -> content.historical field to control
+    # whether "insertion", "chunk", "marker" events can be sent
+    msc2716_historical = attr.ib(type=bool)
+    # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
+    msc2716_redactions = attr.ib(type=bool)
 
 
 class RoomVersions:
@@ -87,7 +91,10 @@ class RoomVersions:
         strict_canonicaljson=False,
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
-        allow_knocking=False,
+        msc3083_join_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
     )
     V2 = RoomVersion(
         "2",
@@ -99,7 +106,10 @@ class RoomVersions:
         strict_canonicaljson=False,
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
-        allow_knocking=False,
+        msc3083_join_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
     )
     V3 = RoomVersion(
         "3",
@@ -111,7 +121,10 @@ class RoomVersions:
         strict_canonicaljson=False,
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
-        allow_knocking=False,
+        msc3083_join_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
     )
     V4 = RoomVersion(
         "4",
@@ -123,7 +136,10 @@ class RoomVersions:
         strict_canonicaljson=False,
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
-        allow_knocking=False,
+        msc3083_join_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
     )
     V5 = RoomVersion(
         "5",
@@ -135,7 +151,10 @@ class RoomVersions:
         strict_canonicaljson=False,
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
-        allow_knocking=False,
+        msc3083_join_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
     )
     V6 = RoomVersion(
         "6",
@@ -147,10 +166,58 @@ class RoomVersions:
         strict_canonicaljson=True,
         limit_notifications_power_levels=True,
         msc2176_redaction_rules=False,
-        allow_knocking=False,
+        msc3083_join_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+    )
+    MSC2176 = RoomVersion(
+        "org.matrix.msc2176",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.V3,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2176_redaction_rules=True,
+        msc3083_join_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
     )
     V7 = RoomVersion(
         "7",
+        RoomDisposition.STABLE,
+        EventFormatVersions.V3,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+    )
+    V8 = RoomVersion(
+        "8",
+        RoomDisposition.STABLE,
+        EventFormatVersions.V3,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+    )
+    MSC2716 = RoomVersion(
+        "org.matrix.msc2716",
         RoomDisposition.UNSTABLE,
         EventFormatVersions.V3,
         StateResolutionVersions.V2,
@@ -159,10 +226,13 @@ class RoomVersions:
         strict_canonicaljson=True,
         limit_notifications_power_levels=True,
         msc2176_redaction_rules=False,
-        allow_knocking=True,
+        msc3083_join_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=True,
+        msc2716_redactions=False,
     )
-    MSC2176 = RoomVersion(
-        "org.matrix.msc2176",
+    MSC2716v2 = RoomVersion(
+        "org.matrix.msc2716v2",
         RoomDisposition.UNSTABLE,
         EventFormatVersions.V3,
         StateResolutionVersions.V2,
@@ -170,12 +240,15 @@ class RoomVersions:
         special_case_aliases_auth=False,
         strict_canonicaljson=True,
         limit_notifications_power_levels=True,
-        msc2176_redaction_rules=True,
-        allow_knocking=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=True,
+        msc2716_redactions=True,
     )
 
 
-KNOWN_ROOM_VERSIONS = {
+KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
     v.identifier: v
     for v in (
         RoomVersions.V1,
@@ -186,5 +259,43 @@ class RoomVersions:
         RoomVersions.V6,
         RoomVersions.V7,
         RoomVersions.MSC2176,
+        RoomVersions.V7,
+        RoomVersions.MSC2716,
+        RoomVersions.V8,
     )
-}  # type: Dict[str, RoomVersion]
+}
+
+
+@attr.s(slots=True, frozen=True, auto_attribs=True)
+class RoomVersionCapability:
+    """An object which describes the unique attributes of a room version."""
+
+    identifier: str  # the identifier for this capability
+    preferred_version: Optional[RoomVersion]
+    support_check_lambda: Callable[[RoomVersion], bool]
+
+
+MSC3244_CAPABILITIES = {
+    cap.identifier: {
+        "preferred": cap.preferred_version.identifier
+        if cap.preferred_version is not None
+        else None,
+        "support": [
+            v.identifier
+            for v in KNOWN_ROOM_VERSIONS.values()
+            if cap.support_check_lambda(v)
+        ],
+    }
+    for cap in (
+        RoomVersionCapability(
+            "knock",
+            RoomVersions.V7,
+            lambda room_version: room_version.msc2403_knocking,
+        ),
+        RoomVersionCapability(
+            "restricted",
+            RoomVersions.V8,
+            lambda room_version: room_version.msc3083_join_rules,
+        ),
+    )
+}
diff --git a/synapse/api/urls.py b/synapse/api/urls.py
index 6379c86dde..4b1f213c75 100644
--- a/synapse/api/urls.py
+++ b/synapse/api/urls.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py
index d1a2cd5e19..f9940491e8 100644
--- a/synapse/app/__init__.py
+++ b/synapse/app/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 3912c8994c..50a02f51f5 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 # Copyright 2019-2021 The Matrix.org Foundation C.I.C
 #
@@ -22,26 +21,35 @@
 import sys
 import traceback
 import warnings
-from typing import Awaitable, Callable, Iterable
+from typing import TYPE_CHECKING, Awaitable, Callable, Iterable
 
 from cryptography.utils import CryptographyDeprecationWarning
 from typing_extensions import NoReturn
 
+import twisted
 from twisted.internet import defer, error, reactor
+from twisted.logger import LoggingFile, LogLevel
 from twisted.protocols.tls import TLSMemoryBIOFactory
 
 import synapse
+from synapse.api.constants import MAX_PDU_SIZE
 from synapse.app import check_bind_error
 from synapse.app.phone_stats_home import start_phone_stats_home
-from synapse.config.server import ListenerConfig
+from synapse.config.homeserver import HomeServerConfig
 from synapse.crypto import context_factory
+from synapse.events.spamcheck import load_legacy_spam_checkers
+from synapse.events.third_party_rules import load_legacy_third_party_event_rules
 from synapse.logging.context import PreserveLoggingContext
 from synapse.metrics.background_process_metrics import wrap_as_background_process
-from synapse.util.async_helpers import Linearizer
+from synapse.metrics.jemalloc import setup_jemalloc_stats
+from synapse.util.caches.lrucache import setup_expire_lru_cache_entries
 from synapse.util.daemonize import daemonize_process
 from synapse.util.rlimit import change_resource_limit
 from synapse.util.versionstring import get_version_string
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 # list of tuples of function, args list, kwargs dict
@@ -111,10 +119,9 @@ def start_reactor(
         run_command (Callable[]): callable that actually runs the reactor
     """
 
-    install_dns_limiter(reactor)
-
     def run():
         logger.info("Running")
+        setup_jemalloc_stats()
         change_resource_limit(soft_file_limit)
         if gc_thresholds:
             gc.set_threshold(*gc_thresholds)
@@ -139,7 +146,7 @@ def run():
 
 def quit_with_error(error_string: str) -> NoReturn:
     message_lines = error_string.split("\n")
-    line_length = max(len(line) for line in message_lines if len(line) < 80) + 2
+    line_length = min(max(len(line) for line in message_lines), 80) + 2
     sys.stderr.write("*" * line_length + "\n")
     for line in message_lines:
         sys.stderr.write(" %s\n" % (line.rstrip(),))
@@ -147,6 +154,30 @@ def quit_with_error(error_string: str) -> NoReturn:
     sys.exit(1)
 
 
+def handle_startup_exception(e: Exception) -> NoReturn:
+    # Exceptions that occur between setting up the logging and forking or starting
+    # the reactor are written to the logs, followed by a summary to stderr.
+    logger.exception("Exception during startup")
+    quit_with_error(
+        f"Error during initialisation:\n   {e}\nThere may be more information in the logs."
+    )
+
+
+def redirect_stdio_to_logs() -> None:
+    streams = [("stdout", LogLevel.info), ("stderr", LogLevel.error)]
+
+    for (stream, level) in streams:
+        oldStream = getattr(sys, stream)
+        loggingFile = LoggingFile(
+            logger=twisted.logger.Logger(namespace=stream),
+            level=level,
+            encoding=getattr(oldStream, "encoding", None),
+        )
+        setattr(sys, stream, loggingFile)
+
+    print("Redirected stdout/stderr to logs")
+
+
 def register_start(cb: Callable[..., Awaitable], *args, **kwargs) -> None:
     """Register a callback with the reactor, to be called once it is running
 
@@ -259,13 +290,10 @@ def refresh_certificate(hs):
     Refresh the TLS certificates that Synapse is using by re-reading them from
     disk and updating the TLS context factories to use them.
     """
-
     if not hs.config.has_tls_listener():
-        # attempt to reload the certs for the good of the tls_fingerprints
-        hs.config.read_certificate_from_disk(require_cert_and_key=False)
         return
 
-    hs.config.read_certificate_from_disk(require_cert_and_key=True)
+    hs.config.read_certificate_from_disk()
     hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config)
 
     if hs._listening_services:
@@ -289,19 +317,17 @@ def refresh_certificate(hs):
         logger.info("Context factories updated.")
 
 
-async def start(hs: "synapse.server.HomeServer", listeners: Iterable[ListenerConfig]):
+async def start(hs: "HomeServer"):
     """
     Start a Synapse server or worker.
 
-    Should be called once the reactor is running and (if we're using ACME) the
-    TLS certificates are in place.
+    Should be called once the reactor is running.
 
     Will start the main HTTP listeners and do some other startup tasks, and then
     notify systemd.
 
     Args:
         hs: homeserver instance
-        listeners: Listener configuration ('listeners' in homeserver.yaml)
     """
     # Set up the SIGHUP machinery.
     if hasattr(signal, "SIGHUP"):
@@ -336,8 +362,20 @@ def run_sighup(*args, **kwargs):
     # Start the tracer
     synapse.logging.opentracing.init_tracer(hs)  # type: ignore[attr-defined] # noqa
 
+    # Instantiate the modules so they can register their web resources to the module API
+    # before we start the listeners.
+    module_api = hs.get_module_api()
+    for module, config in hs.config.modules.loaded_modules:
+        module(config=config, api=module_api)
+
+    load_legacy_spam_checkers(hs)
+    load_legacy_third_party_event_rules(hs)
+
+    # If we've configured an expiry time for caches, start the background job now.
+    setup_expire_lru_cache_entries(hs)
+
     # It is now safe to start your Synapse.
-    hs.start_listening(listeners)
+    hs.start_listening()
     hs.get_datastore().db_pool.start_profiling()
     hs.get_pusherpool().start()
 
@@ -400,107 +438,6 @@ def setup_sdnotify(hs):
     )
 
 
-def install_dns_limiter(reactor, max_dns_requests_in_flight=100):
-    """Replaces the resolver with one that limits the number of in flight DNS
-    requests.
-
-    This is to workaround https://twistedmatrix.com/trac/ticket/9620, where we
-    can run out of file descriptors and infinite loop if we attempt to do too
-    many DNS queries at once
-
-    XXX: I'm confused by this. reactor.nameResolver does not use twisted.names unless
-    you explicitly install twisted.names as the resolver; rather it uses a GAIResolver
-    backed by the reactor's default threadpool (which is limited to 10 threads). So
-    (a) I don't understand why twisted ticket 9620 is relevant, and (b) I don't
-    understand why we would run out of FDs if we did too many lookups at once.
-    -- richvdh 2020/08/29
-    """
-    new_resolver = _LimitedHostnameResolver(
-        reactor.nameResolver, max_dns_requests_in_flight
-    )
-
-    reactor.installNameResolver(new_resolver)
-
-
-class _LimitedHostnameResolver:
-    """Wraps a IHostnameResolver, limiting the number of in-flight DNS lookups."""
-
-    def __init__(self, resolver, max_dns_requests_in_flight):
-        self._resolver = resolver
-        self._limiter = Linearizer(
-            name="dns_client_limiter", max_count=max_dns_requests_in_flight
-        )
-
-    def resolveHostName(
-        self,
-        resolutionReceiver,
-        hostName,
-        portNumber=0,
-        addressTypes=None,
-        transportSemantics="TCP",
-    ):
-        # We need this function to return `resolutionReceiver` so we do all the
-        # actual logic involving deferreds in a separate function.
-
-        # even though this is happening within the depths of twisted, we need to drop
-        # our logcontext before starting _resolve, otherwise: (a) _resolve will drop
-        # the logcontext if it returns an incomplete deferred; (b) _resolve will
-        # call the resolutionReceiver *with* a logcontext, which it won't be expecting.
-        with PreserveLoggingContext():
-            self._resolve(
-                resolutionReceiver,
-                hostName,
-                portNumber,
-                addressTypes,
-                transportSemantics,
-            )
-
-        return resolutionReceiver
-
-    @defer.inlineCallbacks
-    def _resolve(
-        self,
-        resolutionReceiver,
-        hostName,
-        portNumber=0,
-        addressTypes=None,
-        transportSemantics="TCP",
-    ):
-
-        with (yield self._limiter.queue(())):
-            # resolveHostName doesn't return a Deferred, so we need to hook into
-            # the receiver interface to get told when resolution has finished.
-
-            deferred = defer.Deferred()
-            receiver = _DeferredResolutionReceiver(resolutionReceiver, deferred)
-
-            self._resolver.resolveHostName(
-                receiver, hostName, portNumber, addressTypes, transportSemantics
-            )
-
-            yield deferred
-
-
-class _DeferredResolutionReceiver:
-    """Wraps a IResolutionReceiver and simply resolves the given deferred when
-    resolution is complete
-    """
-
-    def __init__(self, receiver, deferred):
-        self._receiver = receiver
-        self._deferred = deferred
-
-    def resolutionBegan(self, resolutionInProgress):
-        self._receiver.resolutionBegan(resolutionInProgress)
-
-    def addressResolved(self, address):
-        self._receiver.addressResolved(address)
-
-    def resolutionComplete(self):
-        self._deferred.callback(())
-        self._receiver.resolutionComplete()
-
-
 sdnotify_sockaddr = os.getenv("NOTIFY_SOCKET")
 
 
@@ -531,3 +468,25 @@ def sdnotify(state):
         # this is a bit surprising, since we don't expect to have a NOTIFY_SOCKET
         # unless systemd is expecting us to notify it.
         logger.warning("Unable to send notification to systemd: %s", e)
+
+
+def max_request_body_size(config: HomeServerConfig) -> int:
+    """Get a suitable maximum size for incoming HTTP requests"""
+
+    # Other than media uploads, the biggest request we expect to see is a fully-loaded
+    # /federation/v1/send request.
+    #
+    # The main thing in such a request is up to 50 PDUs, and up to 100 EDUs. PDUs are
+    # limited to 65536 bytes (possibly slightly more if the sender didn't use canonical
+    # json encoding); there is no specced limit to EDUs (see
+    # https://github.com/matrix-org/matrix-doc/issues/3121).
+    #
+    # in short, we somewhat arbitrarily limit requests to 200 * 64K (about 12.5M)
+    #
+    max_request_size = 200 * MAX_PDU_SIZE
+
+    # if we have a media repo enabled, we may need to allow larger uploads than that
+    if config.media.can_load_media_repo:
+        max_request_size = max(max_request_size, config.media.max_upload_size)
+
+    return max_request_size
diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py
index 9f99651aa2..7396db93c6 100644
--- a/synapse/app/admin_cmd.py
+++ b/synapse/app/admin_cmd.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -37,11 +35,9 @@
 from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.filtering import SlavedFilteringStore
 from synapse.replication.slave.storage.groups import SlavedGroupServerStore
-from synapse.replication.slave.storage.presence import SlavedPresenceStore
 from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
-from synapse.replication.slave.storage.room import RoomStore
 from synapse.server import HomeServer
 from synapse.util.logcontext import LoggingContext
 from synapse.util.versionstring import get_version_string
@@ -55,14 +51,12 @@ class AdminCmdSlavedStore(
     SlavedApplicationServiceStore,
     SlavedRegistrationStore,
     SlavedFilteringStore,
-    SlavedPresenceStore,
     SlavedGroupServerStore,
     SlavedDeviceInboxStore,
     SlavedDeviceStore,
     SlavedPushRuleStore,
     SlavedEventStore,
     SlavedClientIpStore,
-    RoomStore,
     BaseSlavedStore,
 ):
     pass
@@ -71,12 +65,6 @@ class AdminCmdSlavedStore(
 class AdminCmdServer(HomeServer):
     DATASTORE_CLASS = AdminCmdSlavedStore
 
-    def _listen_http(self, listener_config):
-        pass
-
-    def start_listening(self, listeners):
-        pass
-
 
 async def export_data_command(hs, args):
     """Export data for a user.
@@ -233,7 +221,7 @@ def start(config_options):
 
     async def run():
         with LoggingContext("command"):
-            _base.start(ss, [])
+            _base.start(ss)
             await args.func(ss, args)
 
     _base.start_worker_reactor(
diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py
index add43147b3..de1bcee0a7 100644
--- a/synapse/app/appservice.py
+++ b/synapse/app/appservice.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index add43147b3..de1bcee0a7 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index e9c098c4e7..885454ed44 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index add43147b3..de1bcee0a7 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index add43147b3..de1bcee0a7 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index add43147b3..de1bcee0a7 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py
index b2d21acefd..845e6a8220 100644
--- a/synapse/app/generic_worker.py
+++ b/synapse/app/generic_worker.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -14,12 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import contextlib
 import logging
 import sys
-from typing import Dict, Iterable, Optional, Set
-
-from typing_extensions import ContextManager
+from typing import Dict, Optional
 
 from twisted.internet import address
 from twisted.web.resource import IResource
@@ -36,29 +31,23 @@
     SERVER_KEY_V2_PREFIX,
 )
 from synapse.app import _base
-from synapse.app._base import register_start
+from synapse.app._base import (
+    handle_startup_exception,
+    max_request_body_size,
+    redirect_stdio_to_logs,
+    register_start,
+)
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.config.server import ListenerConfig
-from synapse.federation import send_queue
 from synapse.federation.transport.server import TransportLayerServer
-from synapse.handlers.presence import (
-    BasePresenceHandler,
-    PresenceState,
-    get_interested_parties,
-)
 from synapse.http.server import JsonResource, OptionsResource
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
 from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
-from synapse.replication.http.presence import (
-    ReplicationBumpPresenceActiveTime,
-    ReplicationPresenceSetState,
-)
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
@@ -70,108 +59,72 @@
 from synapse.replication.slave.storage.filtering import SlavedFilteringStore
 from synapse.replication.slave.storage.groups import SlavedGroupServerStore
 from synapse.replication.slave.storage.keys import SlavedKeyStore
-from synapse.replication.slave.storage.presence import SlavedPresenceStore
 from synapse.replication.slave.storage.profile import SlavedProfileStore
 from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
 from synapse.replication.slave.storage.pushers import SlavedPusherStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
-from synapse.replication.slave.storage.room import RoomStore
-from synapse.replication.slave.storage.transactions import SlavedTransactionStore
-from synapse.replication.tcp.client import ReplicationDataHandler
-from synapse.replication.tcp.commands import ClearUserSyncsCommand
-from synapse.replication.tcp.streams import (
-    AccountDataStream,
-    DeviceListsStream,
-    GroupServerStream,
-    PresenceStream,
-    PushersStream,
-    PushRulesStream,
-    ReceiptsStream,
-    TagAccountDataStream,
-    ToDeviceStream,
-)
 from synapse.rest.admin import register_servlets_for_media_repo
-from synapse.rest.client.v1 import events, login, room
-from synapse.rest.client.v1.initial_sync import InitialSyncRestServlet
-from synapse.rest.client.v1.profile import (
-    ProfileAvatarURLRestServlet,
-    ProfileDisplaynameRestServlet,
-    ProfileRestServlet,
-)
-from synapse.rest.client.v1.push_rule import PushRuleRestServlet
-from synapse.rest.client.v1.voip import VoipRestServlet
-from synapse.rest.client.v2_alpha import (
+from synapse.rest.client import (
     account_data,
+    events,
     groups,
+    login,
+    presence,
     read_marker,
     receipts,
+    room,
     room_keys,
     sync,
     tags,
     user_directory,
 )
-from synapse.rest.client.v2_alpha._base import client_patterns
-from synapse.rest.client.v2_alpha.account import ThreepidRestServlet
-from synapse.rest.client.v2_alpha.account_data import (
-    AccountDataServlet,
-    RoomAccountDataServlet,
-)
-from synapse.rest.client.v2_alpha.devices import DevicesRestServlet
-from synapse.rest.client.v2_alpha.keys import (
+from synapse.rest.client._base import client_patterns
+from synapse.rest.client.account import ThreepidRestServlet
+from synapse.rest.client.account_data import AccountDataServlet, RoomAccountDataServlet
+from synapse.rest.client.devices import DevicesRestServlet
+from synapse.rest.client.initial_sync import InitialSyncRestServlet
+from synapse.rest.client.keys import (
     KeyChangesServlet,
     KeyQueryServlet,
     OneTimeKeyServlet,
 )
-from synapse.rest.client.v2_alpha.register import RegisterRestServlet
-from synapse.rest.client.v2_alpha.sendtodevice import SendToDeviceRestServlet
+from synapse.rest.client.profile import (
+    ProfileAvatarURLRestServlet,
+    ProfileDisplaynameRestServlet,
+    ProfileRestServlet,
+)
+from synapse.rest.client.push_rule import PushRuleRestServlet
+from synapse.rest.client.register import RegisterRestServlet
+from synapse.rest.client.sendtodevice import SendToDeviceRestServlet
 from synapse.rest.client.versions import VersionsRestServlet
+from synapse.rest.client.voip import VoipRestServlet
 from synapse.rest.health import HealthResource
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.synapse.client import build_synapse_client_resource_tree
-from synapse.server import HomeServer, cache_in_self
+from synapse.server import HomeServer
 from synapse.storage.databases.main.censor_events import CensorEventsStore
 from synapse.storage.databases.main.client_ips import ClientIpWorkerStore
 from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyStore
+from synapse.storage.databases.main.lock import LockStore
 from synapse.storage.databases.main.media_repository import MediaRepositoryStore
 from synapse.storage.databases.main.metrics import ServerMetricsStore
 from synapse.storage.databases.main.monthly_active_users import (
     MonthlyActiveUsersWorkerStore,
 )
-from synapse.storage.databases.main.presence import UserPresenceState
-from synapse.storage.databases.main.search import SearchWorkerStore
+from synapse.storage.databases.main.presence import PresenceStore
+from synapse.storage.databases.main.room import RoomWorkerStore
+from synapse.storage.databases.main.search import SearchStore
 from synapse.storage.databases.main.stats import StatsStore
 from synapse.storage.databases.main.transactions import TransactionWorkerStore
 from synapse.storage.databases.main.ui_auth import UIAuthWorkerStore
 from synapse.storage.databases.main.user_directory import UserDirectoryStore
-from synapse.types import ReadReceipt
-from synapse.util.async_helpers import Linearizer
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.versionstring import get_version_string
 
 logger = logging.getLogger("synapse.app.generic_worker")
 
 
-class PresenceStatusStubServlet(RestServlet):
-    """If presence is disabled this servlet can be used to stub out setting
-    presence status.
-    """
-
-    PATTERNS = client_patterns("/presence/(?P<user_id>[^/]*)/status")
-
-    def __init__(self, hs):
-        super().__init__()
-        self.auth = hs.get_auth()
-
-    async def on_GET(self, request, user_id):
-        await self.auth.get_user_by_req(request)
-        return 200, {"presence": "offline", "user_id": user_id}
-
-    async def on_PUT(self, request, user_id):
-        await self.auth.get_user_by_req(request)
-        return 200, {}
-
-
 class KeyUploadServlet(RestServlet):
     """An implementation of the `KeyUploadServlet` that responds to read only
     requests, but otherwise proxies through to the master instance.
@@ -265,213 +218,6 @@ async def on_POST(self, request: Request, device_id: Optional[str]):
             return 200, {"one_time_key_counts": result}
 
 
-class _NullContextManager(ContextManager[None]):
-    """A context manager which does nothing."""
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass
-
-
-UPDATE_SYNCING_USERS_MS = 10 * 1000
-
-
-class GenericWorkerPresence(BasePresenceHandler):
-    def __init__(self, hs):
-        super().__init__(hs)
-        self.hs = hs
-        self.is_mine_id = hs.is_mine_id
-
-        self._presence_enabled = hs.config.use_presence
-
-        # The number of ongoing syncs on this process, by user id.
-        # Empty if _presence_enabled is false.
-        self._user_to_num_current_syncs = {}  # type: Dict[str, int]
-
-        self.notifier = hs.get_notifier()
-        self.instance_id = hs.get_instance_id()
-
-        # user_id -> last_sync_ms. Lists the users that have stopped syncing
-        # but we haven't notified the master of that yet
-        self.users_going_offline = {}
-
-        self._bump_active_client = ReplicationBumpPresenceActiveTime.make_client(hs)
-        self._set_state_client = ReplicationPresenceSetState.make_client(hs)
-
-        self._send_stop_syncing_loop = self.clock.looping_call(
-            self.send_stop_syncing, UPDATE_SYNCING_USERS_MS
-        )
-
-        self._busy_presence_enabled = hs.config.experimental.msc3026_enabled
-
-        hs.get_reactor().addSystemEventTrigger(
-            "before",
-            "shutdown",
-            run_as_background_process,
-            "generic_presence.on_shutdown",
-            self._on_shutdown,
-        )
-
-    def _on_shutdown(self):
-        if self._presence_enabled:
-            self.hs.get_tcp_replication().send_command(
-                ClearUserSyncsCommand(self.instance_id)
-            )
-
-    def send_user_sync(self, user_id, is_syncing, last_sync_ms):
-        if self._presence_enabled:
-            self.hs.get_tcp_replication().send_user_sync(
-                self.instance_id, user_id, is_syncing, last_sync_ms
-            )
-
-    def mark_as_coming_online(self, user_id):
-        """A user has started syncing. Send a UserSync to the master, unless they
-        had recently stopped syncing.
-
-        Args:
-            user_id (str)
-        """
-        going_offline = self.users_going_offline.pop(user_id, None)
-        if not going_offline:
-            # Safe to skip because we haven't yet told the master they were offline
-            self.send_user_sync(user_id, True, self.clock.time_msec())
-
-    def mark_as_going_offline(self, user_id):
-        """A user has stopped syncing. We wait before notifying the master as
-        its likely they'll come back soon. This allows us to avoid sending
-        a stopped syncing immediately followed by a started syncing notification
-        to the master
-
-        Args:
-            user_id (str)
-        """
-        self.users_going_offline[user_id] = self.clock.time_msec()
-
-    def send_stop_syncing(self):
-        """Check if there are any users who have stopped syncing a while ago
-        and haven't come back yet. If there are poke the master about them.
-        """
-        now = self.clock.time_msec()
-        for user_id, last_sync_ms in list(self.users_going_offline.items()):
-            if now - last_sync_ms > UPDATE_SYNCING_USERS_MS:
-                self.users_going_offline.pop(user_id, None)
-                self.send_user_sync(user_id, False, last_sync_ms)
-
-    async def user_syncing(
-        self, user_id: str, affect_presence: bool
-    ) -> ContextManager[None]:
-        """Record that a user is syncing.
-
-        Called by the sync and events servlets to record that a user has connected to
-        this worker and is waiting for some events.
-        """
-        if not affect_presence or not self._presence_enabled:
-            return _NullContextManager()
-
-        curr_sync = self._user_to_num_current_syncs.get(user_id, 0)
-        self._user_to_num_current_syncs[user_id] = curr_sync + 1
-
-        # If we went from no in flight sync to some, notify replication
-        if self._user_to_num_current_syncs[user_id] == 1:
-            self.mark_as_coming_online(user_id)
-
-        def _end():
-            # We check that the user_id is in user_to_num_current_syncs because
-            # user_to_num_current_syncs may have been cleared if we are
-            # shutting down.
-            if user_id in self._user_to_num_current_syncs:
-                self._user_to_num_current_syncs[user_id] -= 1
-
-                # If we went from one in flight sync to non, notify replication
-                if self._user_to_num_current_syncs[user_id] == 0:
-                    self.mark_as_going_offline(user_id)
-
-        @contextlib.contextmanager
-        def _user_syncing():
-            try:
-                yield
-            finally:
-                _end()
-
-        return _user_syncing()
-
-    async def notify_from_replication(self, states, stream_id):
-        parties = await get_interested_parties(self.store, states)
-        room_ids_to_states, users_to_states = parties
-
-        self.notifier.on_new_event(
-            "presence_key",
-            stream_id,
-            rooms=room_ids_to_states.keys(),
-            users=users_to_states.keys(),
-        )
-
-    async def process_replication_rows(self, token, rows):
-        states = [
-            UserPresenceState(
-                row.user_id,
-                row.state,
-                row.last_active_ts,
-                row.last_federation_update_ts,
-                row.last_user_sync_ts,
-                row.status_msg,
-                row.currently_active,
-            )
-            for row in rows
-        ]
-
-        for state in states:
-            self.user_to_current_state[state.user_id] = state
-
-        stream_id = token
-        await self.notify_from_replication(states, stream_id)
-
-    def get_currently_syncing_users_for_replication(self) -> Iterable[str]:
-        return [
-            user_id
-            for user_id, count in self._user_to_num_current_syncs.items()
-            if count > 0
-        ]
-
-    async def set_state(self, target_user, state, ignore_status_msg=False):
-        """Set the presence state of the user."""
-        presence = state["presence"]
-
-        valid_presence = (
-            PresenceState.ONLINE,
-            PresenceState.UNAVAILABLE,
-            PresenceState.OFFLINE,
-            PresenceState.BUSY,
-        )
-
-        if presence not in valid_presence or (
-            presence == PresenceState.BUSY and not self._busy_presence_enabled
-        ):
-            raise SynapseError(400, "Invalid presence state")
-
-        user_id = target_user.to_string()
-
-        # If presence is disabled, no-op
-        if not self.hs.config.use_presence:
-            return
-
-        # Proxy request to master
-        await self._set_state_client(
-            user_id=user_id, state=state, ignore_status_msg=ignore_status_msg
-        )
-
-    async def bump_presence_active_time(self, user):
-        """We've seen the user do something that indicates they're interacting
-        with the app.
-        """
-        # If presence is disabled, no-op
-        if not self.hs.config.use_presence:
-            return
-
-        # Proxy request to master
-        user_id = user.to_string()
-        await self._bump_active_client(user_id=user_id)
-
-
 class GenericWorkerSlavedStore(
     # FIXME(#3714): We need to add UserDirectoryStore as we write directly
     # rather than going via the correct worker.
@@ -479,6 +225,7 @@ class GenericWorkerSlavedStore(
     StatsStore,
     UIAuthWorkerStore,
     EndToEndRoomKeyStore,
+    PresenceStore,
     SlavedDeviceInboxStore,
     SlavedDeviceStore,
     SlavedReceiptsStore,
@@ -490,20 +237,19 @@ class GenericWorkerSlavedStore(
     ClientIpWorkerStore,
     SlavedEventStore,
     SlavedKeyStore,
-    RoomStore,
+    RoomWorkerStore,
     DirectoryStore,
     SlavedApplicationServiceStore,
     SlavedRegistrationStore,
-    SlavedTransactionStore,
     SlavedProfileStore,
     SlavedClientIpStore,
-    SlavedPresenceStore,
     SlavedFilteringStore,
     MonthlyActiveUsersWorkerStore,
     MediaRepositoryStore,
     ServerMetricsStore,
-    SearchWorkerStore,
+    SearchStore,
     TransactionWorkerStore,
+    LockStore,
     BaseSlavedStore,
 ):
     pass
@@ -523,7 +269,7 @@ def _listen_http(self, listener_config: ListenerConfig):
             site_tag = port
 
         # We always include a health resource.
-        resources = {"/health": HealthResource()}  # type: Dict[str, IResource]
+        resources: Dict[str, IResource] = {"/health": HealthResource()}
 
         for res in listener_config.http_options.resources:
             for name in res.names:
@@ -565,10 +311,7 @@ def _listen_http(self, listener_config: ListenerConfig):
 
                     user_directory.register_servlets(self, resource)
 
-                    # If presence is disabled, use the stub servlet that does
-                    # not allow sending presence
-                    if not self.config.use_presence:
-                        PresenceStatusStubServlet(self).register(resource)
+                    presence.register_servlets(self, resource)
 
                     groups.register_servlets(self, resource)
 
@@ -617,6 +360,10 @@ def _listen_http(self, listener_config: ListenerConfig):
                 if name == "replication":
                     resources[REPLICATION_PREFIX] = ReplicationRestResource(self)
 
+        # Attach additional resources registered by modules.
+        resources.update(self._module_web_resources)
+        self._module_web_resources_consumed = True
+
         root_resource = create_resource_tree(resources, OptionsResource())
 
         _base.listen_tcp(
@@ -628,14 +375,16 @@ def _listen_http(self, listener_config: ListenerConfig):
                 listener_config,
                 root_resource,
                 self.version_string,
+                max_request_body_size=max_request_body_size(self.config),
+                reactor=self.get_reactor(),
             ),
             reactor=self.get_reactor(),
         )
 
         logger.info("Synapse worker now listening on port %d", port)
 
-    def start_listening(self, listeners: Iterable[ListenerConfig]):
-        for listener in listeners:
+    def start_listening(self):
+        for listener in self.config.worker_listeners:
             if listener.type == "http":
                 self._listen_http(listener)
             elif listener.type == "manhole":
@@ -643,12 +392,10 @@ def start_listening(self, listeners: Iterable[ListenerConfig]):
                     listener.bind_addresses, listener.port, manhole_globals={"hs": self}
                 )
             elif listener.type == "metrics":
-                if not self.get_config().enable_metrics:
+                if not self.config.enable_metrics:
                     logger.warning(
-                        (
-                            "Metrics listener configured, but "
-                            "enable_metrics is not True!"
-                        )
+                        "Metrics listener configured, but "
+                        "enable_metrics is not True!"
                     )
                 else:
                     _base.listen_metrics(listener.bind_addresses, listener.port)
@@ -657,234 +404,6 @@ def start_listening(self, listeners: Iterable[ListenerConfig]):
 
         self.get_tcp_replication().start_replication(self)
 
-    @cache_in_self
-    def get_replication_data_handler(self):
-        return GenericWorkerReplicationHandler(self)
-
-    @cache_in_self
-    def get_presence_handler(self):
-        return GenericWorkerPresence(self)
-
-
-class GenericWorkerReplicationHandler(ReplicationDataHandler):
-    def __init__(self, hs):
-        super().__init__(hs)
-
-        self.store = hs.get_datastore()
-        self.presence_handler = hs.get_presence_handler()  # type: GenericWorkerPresence
-        self.notifier = hs.get_notifier()
-
-        self.notify_pushers = hs.config.start_pushers
-        self.pusher_pool = hs.get_pusherpool()
-
-        self.send_handler = None  # type: Optional[FederationSenderHandler]
-        if hs.config.send_federation:
-            self.send_handler = FederationSenderHandler(hs)
-
-    async def on_rdata(self, stream_name, instance_name, token, rows):
-        await super().on_rdata(stream_name, instance_name, token, rows)
-        await self._process_and_notify(stream_name, instance_name, token, rows)
-
-    async def _process_and_notify(self, stream_name, instance_name, token, rows):
-        try:
-            if self.send_handler:
-                await self.send_handler.process_replication_rows(
-                    stream_name, token, rows
-                )
-
-            if stream_name == PushRulesStream.NAME:
-                self.notifier.on_new_event(
-                    "push_rules_key", token, users=[row.user_id for row in rows]
-                )
-            elif stream_name in (AccountDataStream.NAME, TagAccountDataStream.NAME):
-                self.notifier.on_new_event(
-                    "account_data_key", token, users=[row.user_id for row in rows]
-                )
-            elif stream_name == ReceiptsStream.NAME:
-                self.notifier.on_new_event(
-                    "receipt_key", token, rooms=[row.room_id for row in rows]
-                )
-                await self.pusher_pool.on_new_receipts(
-                    token, token, {row.room_id for row in rows}
-                )
-            elif stream_name == ToDeviceStream.NAME:
-                entities = [row.entity for row in rows if row.entity.startswith("@")]
-                if entities:
-                    self.notifier.on_new_event("to_device_key", token, users=entities)
-            elif stream_name == DeviceListsStream.NAME:
-                all_room_ids = set()  # type: Set[str]
-                for row in rows:
-                    if row.entity.startswith("@"):
-                        room_ids = await self.store.get_rooms_for_user(row.entity)
-                        all_room_ids.update(room_ids)
-                self.notifier.on_new_event("device_list_key", token, rooms=all_room_ids)
-            elif stream_name == PresenceStream.NAME:
-                await self.presence_handler.process_replication_rows(token, rows)
-            elif stream_name == GroupServerStream.NAME:
-                self.notifier.on_new_event(
-                    "groups_key", token, users=[row.user_id for row in rows]
-                )
-            elif stream_name == PushersStream.NAME:
-                for row in rows:
-                    if row.deleted:
-                        self.stop_pusher(row.user_id, row.app_id, row.pushkey)
-                    else:
-                        await self.start_pusher(row.user_id, row.app_id, row.pushkey)
-        except Exception:
-            logger.exception("Error processing replication")
-
-    async def on_position(self, stream_name: str, instance_name: str, token: int):
-        await super().on_position(stream_name, instance_name, token)
-        # Also call on_rdata to ensure that stream positions are properly reset.
-        await self.on_rdata(stream_name, instance_name, token, [])
-
-    def stop_pusher(self, user_id, app_id, pushkey):
-        if not self.notify_pushers:
-            return
-
-        key = "%s:%s" % (app_id, pushkey)
-        pushers_for_user = self.pusher_pool.pushers.get(user_id, {})
-        pusher = pushers_for_user.pop(key, None)
-        if pusher is None:
-            return
-        logger.info("Stopping pusher %r / %r", user_id, key)
-        pusher.on_stop()
-
-    async def start_pusher(self, user_id, app_id, pushkey):
-        if not self.notify_pushers:
-            return
-
-        key = "%s:%s" % (app_id, pushkey)
-        logger.info("Starting pusher %r / %r", user_id, key)
-        return await self.pusher_pool.start_pusher_by_id(app_id, pushkey, user_id)
-
-    def on_remote_server_up(self, server: str):
-        """Called when get a new REMOTE_SERVER_UP command."""
-
-        # Let's wake up the transaction queue for the server in case we have
-        # pending stuff to send to it.
-        if self.send_handler:
-            self.send_handler.wake_destination(server)
-
-
-class FederationSenderHandler:
-    """Processes the fedration replication stream
-
-    This class is only instantiate on the worker responsible for sending outbound
-    federation transactions. It receives rows from the replication stream and forwards
-    the appropriate entries to the FederationSender class.
-    """
-
-    def __init__(self, hs: GenericWorkerServer):
-        self.store = hs.get_datastore()
-        self._is_mine_id = hs.is_mine_id
-        self.federation_sender = hs.get_federation_sender()
-        self._hs = hs
-
-        # Stores the latest position in the federation stream we've gotten up
-        # to. This is always set before we use it.
-        self.federation_position = None
-
-        self._fed_position_linearizer = Linearizer(name="_fed_position_linearizer")
-
-    def wake_destination(self, server: str):
-        self.federation_sender.wake_destination(server)
-
-    async def process_replication_rows(self, stream_name, token, rows):
-        # The federation stream contains things that we want to send out, e.g.
-        # presence, typing, etc.
-        if stream_name == "federation":
-            send_queue.process_rows_for_federation(self.federation_sender, rows)
-            await self.update_token(token)
-
-        # ... and when new receipts happen
-        elif stream_name == ReceiptsStream.NAME:
-            await self._on_new_receipts(rows)
-
-        # ... as well as device updates and messages
-        elif stream_name == DeviceListsStream.NAME:
-            # The entities are either user IDs (starting with '@') whose devices
-            # have changed, or remote servers that we need to tell about
-            # changes.
-            hosts = {row.entity for row in rows if not row.entity.startswith("@")}
-            for host in hosts:
-                self.federation_sender.send_device_messages(host)
-
-        elif stream_name == ToDeviceStream.NAME:
-            # The to_device stream includes stuff to be pushed to both local
-            # clients and remote servers, so we ignore entities that start with
-            # '@' (since they'll be local users rather than destinations).
-            hosts = {row.entity for row in rows if not row.entity.startswith("@")}
-            for host in hosts:
-                self.federation_sender.send_device_messages(host)
-
-    async def _on_new_receipts(self, rows):
-        """
-        Args:
-            rows (Iterable[synapse.replication.tcp.streams.ReceiptsStream.ReceiptsStreamRow]):
-                new receipts to be processed
-        """
-        for receipt in rows:
-            # we only want to send on receipts for our own users
-            if not self._is_mine_id(receipt.user_id):
-                continue
-            receipt_info = ReadReceipt(
-                receipt.room_id,
-                receipt.receipt_type,
-                receipt.user_id,
-                [receipt.event_id],
-                receipt.data,
-            )
-            await self.federation_sender.send_read_receipt(receipt_info)
-
-    async def update_token(self, token):
-        """Update the record of where we have processed to in the federation stream.
-
-        Called after we have processed a an update received over replication. Sends
-        a FEDERATION_ACK back to the master, and stores the token that we have processed
-         in `federation_stream_position` so that we can restart where we left off.
-        """
-        self.federation_position = token
-
-        # We save and send the ACK to master asynchronously, so we don't block
-        # processing on persistence. We don't need to do this operation for
-        # every single RDATA we receive, we just need to do it periodically.
-
-        if self._fed_position_linearizer.is_queued(None):
-            # There is already a task queued up to save and send the token, so
-            # no need to queue up another task.
-            return
-
-        run_as_background_process("_save_and_send_ack", self._save_and_send_ack)
-
-    async def _save_and_send_ack(self):
-        """Save the current federation position in the database and send an ACK
-        to master with where we're up to.
-        """
-        try:
-            # We linearize here to ensure we don't have races updating the token
-            #
-            # XXX this appears to be redundant, since the ReplicationCommandHandler
-            # has a linearizer which ensures that we only process one line of
-            # replication data at a time. Should we remove it, or is it doing useful
-            # service for robustness? Or could we replace it with an assertion that
-            # we're not being re-entered?
-
-            with (await self._fed_position_linearizer.queue(None)):
-                # We persist and ack the same position, so we take a copy of it
-                # here as otherwise it can get modified from underneath us.
-                current_position = self.federation_position
-
-                await self.store.update_federation_out_pos(
-                    "federation", current_position
-                )
-
-                # We ACK this token over replication so that the master can drop
-                # its in memory queues
-                self._hs.get_tcp_replication().send_federation_ack(current_position)
-        except Exception:
-            logger.exception("Error updating federation stream position")
-
 
 def start(config_options):
     try:
@@ -941,6 +460,10 @@ def start(config_options):
         config.server.update_user_directory = False
 
     synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts
+    synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage
+
+    if config.server.gc_seconds:
+        synapse.metrics.MIN_TIME_BETWEEN_GCS = config.server.gc_seconds
 
     hs = GenericWorkerServer(
         config.server_name,
@@ -950,13 +473,20 @@ def start(config_options):
 
     setup_logging(hs, config, use_worker_options=True)
 
-    hs.setup()
+    try:
+        hs.setup()
+
+        # Ensure the replication streamer is always started in case we write to any
+        # streams. Will no-op if no streams can be written to by this worker.
+        hs.get_replication_streamer()
+    except Exception as e:
+        handle_startup_exception(e)
 
-    # Ensure the replication streamer is always started in case we write to any
-    # streams. Will no-op if no streams can be written to by this worker.
-    hs.get_replication_streamer()
+    register_start(_base.start, hs)
 
-    register_start(_base.start, hs, config.worker_listeners)
+    # redirect stdio to the logs, if configured.
+    if not hs.config.no_redirect_stdio:
+        redirect_stdio_to_logs()
 
     _base.start_worker_reactor("synapse-generic-worker", config)
 
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 3bfe9d507f..7dae163c1a 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 #
@@ -18,7 +16,7 @@
 import logging
 import os
 import sys
-from typing import Iterable, Iterator
+from typing import Iterator
 
 from twisted.internet import reactor
 from twisted.web.resource import EncodingResourceWrapper, IResource
@@ -37,7 +35,14 @@
     WEB_CLIENT_PREFIX,
 )
 from synapse.app import _base
-from synapse.app._base import listen_ssl, listen_tcp, quit_with_error, register_start
+from synapse.app._base import (
+    handle_startup_exception,
+    listen_ssl,
+    listen_tcp,
+    max_request_body_size,
+    redirect_stdio_to_logs,
+    register_start,
+)
 from synapse.config._base import ConfigError
 from synapse.config.emailconfig import ThreepidBehaviour
 from synapse.config.homeserver import HomeServerConfig
@@ -64,8 +69,6 @@
 from synapse.rest.well_known import WellKnownResource
 from synapse.server import HomeServer
 from synapse.storage import DataStore
-from synapse.storage.engines import IncorrectDatabaseSetup
-from synapse.storage.prepare_database import UpgradeDatabaseException
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.module_loader import load_module
 from synapse.util.versionstring import get_version_string
@@ -119,6 +122,10 @@ def _listener_http(self, config: HomeServerConfig, listener_config: ListenerConf
                 )
             resources[path] = resource
 
+        # Attach additional resources registered by modules.
+        resources.update(self._module_web_resources)
+        self._module_web_resources_consumed = True
+
         # try to find something useful to redirect '/' to
         if WEB_CLIENT_PREFIX in resources:
             root_resource = RootOptionsRedirectResource(WEB_CLIENT_PREFIX)
@@ -127,19 +134,21 @@ def _listener_http(self, config: HomeServerConfig, listener_config: ListenerConf
         else:
             root_resource = OptionsResource()
 
-        root_resource = create_resource_tree(resources, root_resource)
+        site = SynapseSite(
+            "synapse.access.%s.%s" % ("https" if tls else "http", site_tag),
+            site_tag,
+            listener_config,
+            create_resource_tree(resources, root_resource),
+            self.version_string,
+            max_request_body_size=max_request_body_size(self.config),
+            reactor=self.get_reactor(),
+        )
 
         if tls:
             ports = listen_ssl(
                 bind_addresses,
                 port,
-                SynapseSite(
-                    "synapse.access.https.%s" % (site_tag,),
-                    site_tag,
-                    listener_config,
-                    root_resource,
-                    self.version_string,
-                ),
+                site,
                 self.tls_server_context_factory,
                 reactor=self.get_reactor(),
             )
@@ -149,13 +158,7 @@ def _listener_http(self, config: HomeServerConfig, listener_config: ListenerConf
             ports = listen_tcp(
                 bind_addresses,
                 port,
-                SynapseSite(
-                    "synapse.access.http.%s" % (site_tag,),
-                    site_tag,
-                    listener_config,
-                    root_resource,
-                    self.version_string,
-                ),
+                site,
                 reactor=self.get_reactor(),
             )
             logger.info("Synapse now listening on TCP port %d", port)
@@ -192,7 +195,7 @@ def _configure_named_resource(self, name, compress=False):
                 }
             )
 
-            if self.get_config().threepid_behaviour_email == ThreepidBehaviour.LOCAL:
+            if self.config.threepid_behaviour_email == ThreepidBehaviour.LOCAL:
                 from synapse.rest.synapse.client.password_reset import (
                     PasswordResetSubmitTokenResource,
                 )
@@ -231,7 +234,7 @@ def _configure_named_resource(self, name, compress=False):
             )
 
         if name in ["media", "federation", "client"]:
-            if self.get_config().enable_media_repo:
+            if self.config.enable_media_repo:
                 media_repo = self.get_media_repository_resource()
                 resources.update(
                     {MEDIA_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo}
@@ -245,7 +248,7 @@ def _configure_named_resource(self, name, compress=False):
             resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self)
 
         if name == "webclient":
-            webclient_loc = self.get_config().web_client_location
+            webclient_loc = self.config.web_client_location
 
             if webclient_loc is None:
                 logger.warning(
@@ -266,7 +269,7 @@ def _configure_named_resource(self, name, compress=False):
                 # https://twistedmatrix.com/trac/ticket/7678
                 resources[WEB_CLIENT_PREFIX] = File(webclient_loc)
 
-        if name == "metrics" and self.get_config().enable_metrics:
+        if name == "metrics" and self.config.enable_metrics:
             resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
 
         if name == "replication":
@@ -274,18 +277,18 @@ def _configure_named_resource(self, name, compress=False):
 
         return resources
 
-    def start_listening(self, listeners: Iterable[ListenerConfig]):
-        config = self.get_config()
-
-        if config.redis_enabled:
+    def start_listening(self):
+        if self.config.redis_enabled:
             # If redis is enabled we connect via the replication command handler
             # in the same way as the workers (since we're effectively a client
             # rather than a server).
             self.get_tcp_replication().start_replication(self)
 
-        for listener in listeners:
+        for listener in self.config.server.listeners:
             if listener.type == "http":
-                self._listening_services.extend(self._listener_http(config, listener))
+                self._listening_services.extend(
+                    self._listener_http(self.config, listener)
+                )
             elif listener.type == "manhole":
                 _base.listen_manhole(
                     listener.bind_addresses, listener.port, manhole_globals={"hs": self}
@@ -299,12 +302,10 @@ def start_listening(self, listeners: Iterable[ListenerConfig]):
                 for s in services:
                     reactor.addSystemEventTrigger("before", "shutdown", s.stopListening)
             elif listener.type == "metrics":
-                if not self.get_config().enable_metrics:
+                if not self.config.enable_metrics:
                     logger.warning(
-                        (
-                            "Metrics listener configured, but "
-                            "enable_metrics is not True!"
-                        )
+                        "Metrics listener configured, but "
+                        "enable_metrics is not True!"
                     )
                 else:
                     _base.listen_metrics(listener.bind_addresses, listener.port)
@@ -340,6 +341,10 @@ def setup(config_options):
         sys.exit(0)
 
     events.USE_FROZEN_DICTS = config.use_frozen_dicts
+    synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage
+
+    if config.server.gc_seconds:
+        synapse.metrics.MIN_TIME_BETWEEN_GCS = config.server.gc_seconds
 
     hs = SynapseHomeServer(
         config.server_name,
@@ -353,67 +358,17 @@ def setup(config_options):
 
     try:
         hs.setup()
-    except IncorrectDatabaseSetup as e:
-        quit_with_error(str(e))
-    except UpgradeDatabaseException as e:
-        quit_with_error("Failed to upgrade database: %s" % (e,))
-
-    async def do_acme() -> bool:
-        """
-        Reprovision an ACME certificate, if it's required.
-
-        Returns:
-            Whether the cert has been updated.
-        """
-        acme = hs.get_acme_handler()
-
-        # Check how long the certificate is active for.
-        cert_days_remaining = hs.config.is_disk_cert_valid(allow_self_signed=False)
-
-        # We want to reprovision if cert_days_remaining is None (meaning no
-        # certificate exists), or the days remaining number it returns
-        # is less than our re-registration threshold.
-        provision = False
-
-        if (
-            cert_days_remaining is None
-            or cert_days_remaining < hs.config.acme_reprovision_threshold
-        ):
-            provision = True
-
-        if provision:
-            await acme.provision_certificate()
-
-        return provision
-
-    async def reprovision_acme():
-        """
-        Provision a certificate from ACME, if required, and reload the TLS
-        certificate if it's renewed.
-        """
-        reprovisioned = await do_acme()
-        if reprovisioned:
-            _base.refresh_certificate(hs)
+    except Exception as e:
+        handle_startup_exception(e)
 
     async def start():
-        # Run the ACME provisioning code, if it's enabled.
-        if hs.config.acme_enabled:
-            acme = hs.get_acme_handler()
-            # Start up the webservices which we will respond to ACME
-            # challenges with, and then provision.
-            await acme.start_listening()
-            await do_acme()
-
-            # Check if it needs to be reprovisioned every day.
-            hs.get_clock().looping_call(reprovision_acme, 24 * 60 * 60 * 1000)
-
         # Load the OIDC provider metadatas, if OIDC is enabled.
         if hs.config.oidc_enabled:
             oidc = hs.get_oidc_handler()
             # Loading the provider metadata also ensures the provider config is valid.
             await oidc.load_metadata()
 
-        await _base.start(hs, config.listeners)
+        await _base.start(hs)
 
         hs.get_datastore().db_pool.updates.start_doing_background_updates()
 
@@ -495,6 +450,11 @@ def main():
         # check base requirements
         check_requirements()
         hs = setup(sys.argv[1:])
+
+        # redirect stdio to the logs, if configured.
+        if not hs.config.no_redirect_stdio:
+            redirect_stdio_to_logs()
+
         run(hs)
 
 
diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py
index add43147b3..de1bcee0a7 100644
--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py
index 8f86cecb76..86ad7337a9 100644
--- a/synapse/app/phone_stats_home.py
+++ b/synapse/app/phone_stats_home.py
@@ -71,6 +71,8 @@ async def phone_stats_home(hs, stats, stats_process=_stats_process):
     # General statistics
     #
 
+    store = hs.get_datastore()
+
     stats["homeserver"] = hs.config.server_name
     stats["server_context"] = hs.config.server_context
     stats["timestamp"] = now
@@ -79,34 +81,38 @@ async def phone_stats_home(hs, stats, stats_process=_stats_process):
     stats["python_version"] = "{}.{}.{}".format(
         version.major, version.minor, version.micro
     )
-    stats["total_users"] = await hs.get_datastore().count_all_users()
+    stats["total_users"] = await store.count_all_users()
 
-    total_nonbridged_users = await hs.get_datastore().count_nonbridged_users()
+    total_nonbridged_users = await store.count_nonbridged_users()
     stats["total_nonbridged_users"] = total_nonbridged_users
 
-    daily_user_type_results = await hs.get_datastore().count_daily_user_type()
+    daily_user_type_results = await store.count_daily_user_type()
     for name, count in daily_user_type_results.items():
         stats["daily_user_type_" + name] = count
 
-    room_count = await hs.get_datastore().get_room_count()
+    room_count = await store.get_room_count()
     stats["total_room_count"] = room_count
 
-    stats["daily_active_users"] = await hs.get_datastore().count_daily_users()
-    stats["monthly_active_users"] = await hs.get_datastore().count_monthly_users()
-    daily_active_e2ee_rooms = await hs.get_datastore().count_daily_active_e2ee_rooms()
+    stats["daily_active_users"] = await store.count_daily_users()
+    stats["monthly_active_users"] = await store.count_monthly_users()
+    daily_active_e2ee_rooms = await store.count_daily_active_e2ee_rooms()
     stats["daily_active_e2ee_rooms"] = daily_active_e2ee_rooms
-    stats["daily_e2ee_messages"] = await hs.get_datastore().count_daily_e2ee_messages()
-    daily_sent_e2ee_messages = await hs.get_datastore().count_daily_sent_e2ee_messages()
+    stats["daily_e2ee_messages"] = await store.count_daily_e2ee_messages()
+    daily_sent_e2ee_messages = await store.count_daily_sent_e2ee_messages()
     stats["daily_sent_e2ee_messages"] = daily_sent_e2ee_messages
-    stats["daily_active_rooms"] = await hs.get_datastore().count_daily_active_rooms()
-    stats["daily_messages"] = await hs.get_datastore().count_daily_messages()
-    daily_sent_messages = await hs.get_datastore().count_daily_sent_messages()
+    stats["daily_active_rooms"] = await store.count_daily_active_rooms()
+    stats["daily_messages"] = await store.count_daily_messages()
+    daily_sent_messages = await store.count_daily_sent_messages()
     stats["daily_sent_messages"] = daily_sent_messages
 
-    r30_results = await hs.get_datastore().count_r30_users()
+    r30_results = await store.count_r30_users()
     for name, count in r30_results.items():
         stats["r30_users_" + name] = count
 
+    r30v2_results = await store.count_r30v2_users()
+    for name, count in r30v2_results.items():
+        stats["r30v2_users_" + name] = count
+
     stats["cache_factor"] = hs.config.caches.global_factor
     stats["event_cache_size"] = hs.config.caches.event_cache_size
 
@@ -115,8 +121,8 @@ async def phone_stats_home(hs, stats, stats_process=_stats_process):
     #
 
     # This only reports info about the *main* database.
-    stats["database_engine"] = hs.get_datastore().db_pool.engine.module.__name__
-    stats["database_server_version"] = hs.get_datastore().db_pool.engine.server_version
+    stats["database_engine"] = store.db_pool.engine.module.__name__
+    stats["database_server_version"] = store.db_pool.engine.server_version
 
     #
     # Logging configuration
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index add43147b3..de1bcee0a7 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index add43147b3..de1bcee0a7 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index 503d44f687..14bde27179 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py
index 0bfc5e445f..6504c6bd3f 100644
--- a/synapse/appservice/__init__.py
+++ b/synapse/appservice/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py
index 8c5d178e05..935f24263c 100644
--- a/synapse/appservice/api.py
+++ b/synapse/appservice/api.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -89,9 +88,9 @@ def __init__(self, hs):
         super().__init__(hs)
         self.clock = hs.get_clock()
 
-        self.protocol_meta_cache = ResponseCache(
+        self.protocol_meta_cache: ResponseCache[Tuple[str, str]] = ResponseCache(
             hs.get_clock(), "as_protocol_meta", timeout_ms=HOUR_IN_MS
-        )  # type: ResponseCache[Tuple[str, str]]
+        )
 
     async def query_user(self, service, user_id):
         if service.url is None:
diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py
index 366c476f80..6a2ce99b55 100644
--- a/synapse/appservice/scheduler.py
+++ b/synapse/appservice/scheduler.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -49,7 +48,7 @@
 components.
 """
 import logging
-from typing import List
+from typing import List, Optional
 
 from synapse.appservice import ApplicationService, ApplicationServiceState
 from synapse.events import EventBase
@@ -191,11 +190,11 @@ async def send(
         self,
         service: ApplicationService,
         events: List[EventBase],
-        ephemeral: List[JsonDict] = [],
+        ephemeral: Optional[List[JsonDict]] = None,
     ):
         try:
             txn = await self.store.create_appservice_txn(
-                service=service, events=events, ephemeral=ephemeral
+                service=service, events=events, ephemeral=ephemeral or []
             )
             service_is_up = await self._is_service_up(service)
             if service_is_up:
diff --git a/synapse/config/__init__.py b/synapse/config/__init__.py
index 1e76e9559d..d2f889159e 100644
--- a/synapse/config/__init__.py
+++ b/synapse/config/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/config/__main__.py b/synapse/config/__main__.py
index 65043d5b5b..b5b6735a8f 100644
--- a/synapse/config/__main__.py
+++ b/synapse/config/__main__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index ba9cd63cf2..2cc242782a 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -238,13 +237,14 @@ def read_template(self, filename: str) -> jinja2.Template:
     def read_templates(
         self,
         filenames: List[str],
-        custom_template_directory: Optional[str] = None,
+        custom_template_directories: Optional[Iterable[str]] = None,
     ) -> List[jinja2.Template]:
         """Load a list of template files from disk using the given variables.
 
         This function will attempt to load the given templates from the default Synapse
-        template directory. If `custom_template_directory` is supplied, that directory
-        is tried first.
+        template directory. If `custom_template_directories` is supplied, any directory
+        in this list is tried (in the order they appear in the list) before trying
+        Synapse's default directory.
 
         Files read are treated as Jinja templates. The templates are not rendered yet
         and have autoescape enabled.
@@ -252,8 +252,8 @@ def read_templates(
         Args:
             filenames: A list of template filenames to read.
 
-            custom_template_directory: A directory to try to look for the templates
-                before using the default Synapse template directory instead.
+            custom_template_directories: A list of directory to try to look for the
+                templates before using the default Synapse template directory instead.
 
         Raises:
             ConfigError: if the file's path is incorrect or otherwise cannot be read.
@@ -261,20 +261,26 @@ def read_templates(
         Returns:
             A list of jinja2 templates.
         """
-        search_directories = [self.default_template_dir]
-
-        # The loader will first look in the custom template directory (if specified) for the
-        # given filename. If it doesn't find it, it will use the default template dir instead
-        if custom_template_directory:
-            # Check that the given template directory exists
-            if not self.path_exists(custom_template_directory):
-                raise ConfigError(
-                    "Configured template directory does not exist: %s"
-                    % (custom_template_directory,)
-                )
+        search_directories = []
+
+        # The loader will first look in the custom template directories (if specified)
+        # for the given filename. If it doesn't find it, it will use the default
+        # template dir instead.
+        if custom_template_directories is not None:
+            for custom_template_directory in custom_template_directories:
+                # Check that the given template directory exists
+                if not self.path_exists(custom_template_directory):
+                    raise ConfigError(
+                        "Configured template directory does not exist: %s"
+                        % (custom_template_directory,)
+                    )
 
-            # Search the custom template directory as well
-            search_directories.insert(0, custom_template_directory)
+                # Search the custom template directory as well
+                search_directories.append(custom_template_directory)
+
+        # Append the default directory at the end of the list so Jinja can fallback on it
+        # if a template is missing from any custom directory.
+        search_directories.append(self.default_template_dir)
 
         # TODO: switch to synapse.util.templates.build_jinja_env
         loader = jinja2.FileSystemLoader(search_directories)
@@ -406,7 +412,6 @@ def generate_config(
         listeners=None,
         tls_certificate_path=None,
         tls_private_key_path=None,
-        acme_domain=None,
     ):
         """
         Build a default configuration file
@@ -458,9 +463,6 @@ def generate_config(
 
             tls_private_key_path (str|None): The path to the tls private key.
 
-            acme_domain (str|None): The domain acme will try to validate. If
-                specified acme will be enabled.
-
         Returns:
             str: the yaml config file
         """
@@ -478,7 +480,6 @@ def generate_config(
                 listeners=listeners,
                 tls_certificate_path=tls_certificate_path,
                 tls_private_key_path=tls_private_key_path,
-                acme_domain=acme_domain,
             ).values()
         )
 
diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi
index ddec356a07..06fbd1166b 100644
--- a/synapse/config/_base.pyi
+++ b/synapse/config/_base.pyi
@@ -5,18 +5,21 @@ from synapse.config import (
     api,
     appservice,
     auth,
+    cache,
     captcha,
     cas,
-    consent_config,
+    consent,
     database,
     emailconfig,
     experimental,
+    federation,
     groups,
-    jwt_config,
+    jwt,
     key,
     logger,
     metrics,
-    oidc_config,
+    modules,
+    oidc,
     password_auth_providers,
     push,
     ratelimiting,
@@ -24,9 +27,9 @@ from synapse.config import (
     registration,
     repository,
     room_directory,
-    saml2_config,
+    saml2,
     server,
-    server_notices_config,
+    server_notices,
     spam_checker,
     sso,
     stats,
@@ -65,11 +68,11 @@ class RootConfig:
     api: api.ApiConfig
     appservice: appservice.AppServiceConfig
     key: key.KeyConfig
-    saml2: saml2_config.SAML2Config
+    saml2: saml2.SAML2Config
     cas: cas.CasConfig
     sso: sso.SSOConfig
-    oidc: oidc_config.OIDCConfig
-    jwt: jwt_config.JWTConfig
+    oidc: oidc.OIDCConfig
+    jwt: jwt.JWTConfig
     auth: auth.AuthConfig
     email: emailconfig.EmailConfig
     worker: workers.WorkerConfig
@@ -78,13 +81,16 @@ class RootConfig:
     spamchecker: spam_checker.SpamCheckerConfig
     groups: groups.GroupsConfig
     userdirectory: user_directory.UserDirectoryConfig
-    consent: consent_config.ConsentConfig
+    consent: consent.ConsentConfig
     stats: stats.StatsConfig
-    servernotices: server_notices_config.ServerNoticesConfig
+    servernotices: server_notices.ServerNoticesConfig
     roomdirectory: room_directory.RoomDirectoryConfig
     thirdpartyrules: third_party_event_rules.ThirdPartyRulesConfig
     tracer: tracer.TracerConfig
     redis: redis.RedisConfig
+    modules: modules.ModulesConfig
+    caches: cache.CacheConfig
+    federation: federation.FederationConfig
 
     config_classes: List = ...
     def __init__(self) -> None: ...
@@ -111,7 +117,6 @@ class RootConfig:
         database_conf: Optional[Any] = ...,
         tls_certificate_path: Optional[str] = ...,
         tls_private_key_path: Optional[str] = ...,
-        acme_domain: Optional[str] = ...,
     ): ...
     @classmethod
     def load_or_generate_config(cls, description: Any, argv: Any): ...
diff --git a/synapse/config/_util.py b/synapse/config/_util.py
index 8fce7f6bb1..3edb4b7106 100644
--- a/synapse/config/_util.py
+++ b/synapse/config/_util.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/config/account_validity.py b/synapse/config/account_validity.py
index 6d107944a3..8b96883132 100644
--- a/synapse/config/account_validity.py
+++ b/synapse/config/account_validity.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,6 +18,21 @@ class AccountValidityConfig(Config):
     section = "account_validity"
 
     def read_config(self, config, **kwargs):
+        """Parses the old account validity config. The config format looks like this:
+
+        account_validity:
+            enabled: true
+            period: 6w
+            renew_at: 1w
+            renew_email_subject: "Renew your %(app)s account"
+            template_dir: "res/templates"
+            account_renewed_html_path: "account_renewed.html"
+            invalid_token_html_path: "invalid_token.html"
+
+        We expect admins to use modules for this feature (which is why it doesn't appear
+        in the sample config file), but we want to keep support for it around for a bit
+        for backwards compatibility.
+        """
         account_validity_config = config.get("account_validity") or {}
         self.account_validity_enabled = account_validity_config.get("enabled", False)
         self.account_validity_renew_by_email_enabled = (
@@ -54,18 +68,19 @@ def read_config(self, config, **kwargs):
                 raise ConfigError("Can't send renewal emails without 'public_baseurl'")
 
         # Load account validity templates.
-        # We do this here instead of in AccountValidityConfig as read_templates
-        # relies on state that hasn't been initialised in AccountValidityConfig
+        account_validity_template_dir = account_validity_config.get("template_dir")
         account_renewed_template_filename = account_validity_config.get(
             "account_renewed_html_path", "account_renewed.html"
         )
-        account_previously_renewed_template_filename = account_validity_config.get(
-            "account_previously_renewed_html_path", "account_previously_renewed.html"
-        )
         invalid_token_template_filename = account_validity_config.get(
             "invalid_token_html_path", "invalid_token.html"
         )
-        custom_template_directory = account_validity_config.get("template_dir")
+
+        # Read and store template content
+        custom_template_directories = (
+            self.root.server.custom_template_directory,
+            account_validity_template_dir,
+        )
 
         (
             self.account_validity_account_renewed_template,
@@ -74,76 +89,8 @@ def read_config(self, config, **kwargs):
         ) = self.read_templates(
             [
                 account_renewed_template_filename,
-                account_previously_renewed_template_filename,
+                "account_previously_renewed.html",
                 invalid_token_template_filename,
             ],
-            custom_template_directory=custom_template_directory,
+            (td for td in custom_template_directories if td),
         )
-
-    def generate_config_section(self, **kwargs):
-        return """\
-        ## Account Validity ##
-        #
-        # Optional account validity configuration. This allows for accounts to be denied
-        # any request after a given period.
-        #
-        # Once this feature is enabled, Synapse will look for registered users without an
-        # expiration date at startup and will add one to every account it found using the
-        # current settings at that time.
-        # This means that, if a validity period is set, and Synapse is restarted (it will
-        # then derive an expiration date from the current validity period), and some time
-        # after that the validity period changes and Synapse is restarted, the users'
-        # expiration dates won't be updated unless their account is manually renewed. This
-        # date will be randomly selected within a range [now + period - d ; now + period],
-        # where d is equal to 10% of the validity period.
-        #
-        account_validity:
-          # The account validity feature is disabled by default. Uncomment the
-          # following line to enable it.
-          #
-          #enabled: true
-
-          # The period after which an account is valid after its registration. When
-          # renewing the account, its validity period will be extended by this amount
-          # of time. This parameter is required when using the account validity
-          # feature.
-          #
-          #period: 6w
-
-          # The amount of time before an account's expiry date at which Synapse will
-          # send an email to the account's email address with a renewal link. By
-          # default, no such emails are sent.
-          #
-          # If you enable this setting, you will also need to fill out the 'email' and
-          # 'public_baseurl' configuration sections.
-          #
-          #renew_at: 1w
-
-          # The subject of the email sent out with the renewal link. '%(app)s' can be
-          # used as a placeholder for the 'app_name' parameter from the 'email'
-          # section.
-          #
-          # Note that the placeholder must be written '%(app)s', including the
-          # trailing 's'.
-          #
-          # If this is not set, a default value is used.
-          #
-          #renew_email_subject: "Renew your %(app)s account"
-
-          # Directory in which Synapse will try to find templates for the HTML files to
-          # serve to the user when trying to renew an account. If not set, default
-          # templates from within the Synapse package will be used.
-          #
-          #template_dir: "res/templates"
-
-          # File within 'template_dir' giving the HTML to be displayed to the user after
-          # they successfully renewed their account. If not set, default text is used.
-          #
-          #account_renewed_html_path: "account_renewed.html"
-
-          # File within 'template_dir' giving the HTML to be displayed when the user
-          # tries to renew an account with an invalid renewal token. If not set,
-          # default text is used.
-          #
-          #invalid_token_html_path: "invalid_token.html"
-        """
diff --git a/synapse/config/api.py b/synapse/config/api.py
index 0638ed8d2e..972f1ffc76 100644
--- a/synapse/config/api.py
+++ b/synapse/config/api.py
@@ -1,5 +1,4 @@
-# Copyright 2015, 2016 OpenMarket Ltd
-# Copyright 2020 The Matrix.org Foundation C.I.C.
+# Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,42 +12,129 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.api.constants import EventTypes
+import logging
+from typing import Iterable
 
-from ._base import Config
+from synapse.api.constants import EventTypes
+from synapse.config._base import Config, ConfigError
+from synapse.config._util import validate_config
+from synapse.types import JsonDict
 
-# The default types of room state to send to users to are invited to or knock on a room.
-DEFAULT_ROOM_STATE_TYPES = [
-    EventTypes.JoinRules,
-    EventTypes.CanonicalAlias,
-    EventTypes.RoomAvatar,
-    EventTypes.RoomEncryption,
-    EventTypes.Name,
-]
+logger = logging.getLogger(__name__)
 
 
 class ApiConfig(Config):
     section = "api"
 
-    def read_config(self, config, **kwargs):
-        self.room_invite_state_types = config.get(
-            "room_invite_state_types", DEFAULT_ROOM_STATE_TYPES
+    def read_config(self, config: JsonDict, **kwargs):
+        validate_config(_MAIN_SCHEMA, config, ())
+        self.room_prejoin_state = list(self._get_prejoin_state_types(config))
+
+    def generate_config_section(cls, **kwargs) -> str:
+        formatted_default_state_types = "\n".join(
+            "           # - %s" % (t,) for t in _DEFAULT_PREJOIN_STATE_TYPES
         )
 
-    def generate_config_section(cls, **kwargs):
         return """\
         ## API Configuration ##
 
-        # A list of event types from a room that will be given to users when they
-        # are invited to a room. This allows clients to display information about the
-        # room that they've been invited to, without actually being in the room yet.
+        # Controls for the state that is shared with users who receive an invite
+        # to a room
         #
-        #room_invite_state_types:
-        #  - "{JoinRules}"
-        #  - "{CanonicalAlias}"
-        #  - "{RoomAvatar}"
-        #  - "{RoomEncryption}"
-        #  - "{Name}"
-        """.format(
-            **vars(EventTypes)
-        )
+        room_prejoin_state:
+           # By default, the following state event types are shared with users who
+           # receive invites to the room:
+           #
+%(formatted_default_state_types)s
+           #
+           # Uncomment the following to disable these defaults (so that only the event
+           # types listed in 'additional_event_types' are shared). Defaults to 'false'.
+           #
+           #disable_default_event_types: true
+
+           # Additional state event types to share with users when they are invited
+           # to a room.
+           #
+           # By default, this list is empty (so only the default event types are shared).
+           #
+           #additional_event_types:
+           #  - org.example.custom.event.type
+        """ % {
+            "formatted_default_state_types": formatted_default_state_types
+        }
+
+    def _get_prejoin_state_types(self, config: JsonDict) -> Iterable[str]:
+        """Get the event types to include in the prejoin state
+
+        Parses the config and returns an iterable of the event types to be included.
+        """
+        room_prejoin_state_config = config.get("room_prejoin_state") or {}
+
+        # backwards-compatibility support for room_invite_state_types
+        if "room_invite_state_types" in config:
+            # if both "room_invite_state_types" and "room_prejoin_state" are set, then
+            # we don't really know what to do.
+            if room_prejoin_state_config:
+                raise ConfigError(
+                    "Can't specify both 'room_invite_state_types' and 'room_prejoin_state' "
+                    "in config"
+                )
+
+            logger.warning(_ROOM_INVITE_STATE_TYPES_WARNING)
+
+            yield from config["room_invite_state_types"]
+            return
+
+        if not room_prejoin_state_config.get("disable_default_event_types"):
+            yield from _DEFAULT_PREJOIN_STATE_TYPES
+
+        yield from room_prejoin_state_config.get("additional_event_types", [])
+
+
+_ROOM_INVITE_STATE_TYPES_WARNING = """\
+WARNING: The 'room_invite_state_types' configuration setting is now deprecated,
+and replaced with 'room_prejoin_state'. New features may not work correctly
+unless 'room_invite_state_types' is removed. See the sample configuration file for
+details of 'room_prejoin_state'.
+--------------------------------------------------------------------------------
+"""
+
+_DEFAULT_PREJOIN_STATE_TYPES = [
+    EventTypes.JoinRules,
+    EventTypes.CanonicalAlias,
+    EventTypes.RoomAvatar,
+    EventTypes.RoomEncryption,
+    EventTypes.Name,
+    # Per MSC1772.
+    EventTypes.Create,
+]
+
+
+# room_prejoin_state can either be None (as it is in the default config), or
+# an object containing other config settings
+_ROOM_PREJOIN_STATE_CONFIG_SCHEMA = {
+    "oneOf": [
+        {
+            "type": "object",
+            "properties": {
+                "disable_default_event_types": {"type": "boolean"},
+                "additional_event_types": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                },
+            },
+        },
+        {"type": "null"},
+    ]
+}
+
+# the legacy room_invite_state_types setting
+_ROOM_INVITE_STATE_TYPES_SCHEMA = {"type": "array", "items": {"type": "string"}}
+
+_MAIN_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "room_prejoin_state": _ROOM_PREJOIN_STATE_CONFIG_SCHEMA,
+        "room_invite_state_types": _ROOM_INVITE_STATE_TYPES_SCHEMA,
+    },
+}
diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py
index 746fc3cc02..1ebea88db2 100644
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -57,14 +57,14 @@ def load_appservices(hostname, config_files):
         return []
 
     # Dicts of value -> filename
-    seen_as_tokens = {}  # type: Dict[str, str]
-    seen_ids = {}  # type: Dict[str, str]
+    seen_as_tokens: Dict[str, str] = {}
+    seen_ids: Dict[str, str] = {}
 
     appservices = []
 
     for config_file in config_files:
         try:
-            with open(config_file, "r") as f:
+            with open(config_file) as f:
                 appservice = _load_appservice(hostname, yaml.safe_load(f), config_file)
                 if appservice.id in seen_ids:
                     raise ConfigError(
diff --git a/synapse/config/auth.py b/synapse/config/auth.py
index 9aabaadf9e..53809cee2e 100644
--- a/synapse/config/auth.py
+++ b/synapse/config/auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -104,6 +103,10 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs):
             # the user-interactive authentication process, by allowing for multiple
             # (and potentially different) operations to use the same validation session.
             #
+            # This is ignored for potentially "dangerous" operations (including
+            # deactivating an account, modifying an account password, and
+            # adding a 3PID).
+            #
             # Uncomment below to allow for credential validation to last for 15
             # seconds.
             #
diff --git a/synapse/config/cache.py b/synapse/config/cache.py
index 4e8abbf88a..d119427ad8 100644
--- a/synapse/config/cache.py
+++ b/synapse/config/cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,13 +17,15 @@
 import threading
 from typing import Callable, Dict
 
+from synapse.python_dependencies import DependencyException, check_requirements
+
 from ._base import Config, ConfigError
 
 # The prefix for all cache factor-related environment variables
 _CACHE_PREFIX = "SYNAPSE_CACHE_FACTOR"
 
 # Map from canonicalised cache name to cache.
-_CACHES = {}  # type: Dict[str, Callable[[float], None]]
+_CACHES: Dict[str, Callable[[float], None]] = {}
 
 # a lock on the contents of _CACHES
 _CACHES_LOCK = threading.Lock()
@@ -115,42 +116,57 @@ def generate_config_section(self, **kwargs):
         #event_cache_size: 10K
 
         caches:
-           # Controls the global cache factor, which is the default cache factor
-           # for all caches if a specific factor for that cache is not otherwise
-           # set.
-           #
-           # This can also be set by the "SYNAPSE_CACHE_FACTOR" environment
-           # variable. Setting by environment variable takes priority over
-           # setting through the config file.
-           #
-           # Defaults to 0.5, which will half the size of all caches.
-           #
-           #global_factor: 1.0
-
-           # A dictionary of cache name to cache factor for that individual
-           # cache. Overrides the global cache factor for a given cache.
-           #
-           # These can also be set through environment variables comprised
-           # of "SYNAPSE_CACHE_FACTOR_" + the name of the cache in capital
-           # letters and underscores. Setting by environment variable
-           # takes priority over setting through the config file.
-           # Ex. SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0
-           #
-           # Some caches have '*' and other characters that are not
-           # alphanumeric or underscores. These caches can be named with or
-           # without the special characters stripped. For example, to specify
-           # the cache factor for `*stateGroupCache*` via an environment
-           # variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`.
-           #
-           per_cache_factors:
-             #get_users_who_share_room_with_user: 2.0
+          # Controls the global cache factor, which is the default cache factor
+          # for all caches if a specific factor for that cache is not otherwise
+          # set.
+          #
+          # This can also be set by the "SYNAPSE_CACHE_FACTOR" environment
+          # variable. Setting by environment variable takes priority over
+          # setting through the config file.
+          #
+          # Defaults to 0.5, which will half the size of all caches.
+          #
+          #global_factor: 1.0
+
+          # A dictionary of cache name to cache factor for that individual
+          # cache. Overrides the global cache factor for a given cache.
+          #
+          # These can also be set through environment variables comprised
+          # of "SYNAPSE_CACHE_FACTOR_" + the name of the cache in capital
+          # letters and underscores. Setting by environment variable
+          # takes priority over setting through the config file.
+          # Ex. SYNAPSE_CACHE_FACTOR_GET_USERS_WHO_SHARE_ROOM_WITH_USER=2.0
+          #
+          # Some caches have '*' and other characters that are not
+          # alphanumeric or underscores. These caches can be named with or
+          # without the special characters stripped. For example, to specify
+          # the cache factor for `*stateGroupCache*` via an environment
+          # variable would be `SYNAPSE_CACHE_FACTOR_STATEGROUPCACHE=2.0`.
+          #
+          per_cache_factors:
+            #get_users_who_share_room_with_user: 2.0
+
+          # Controls how long an entry can be in a cache without having been
+          # accessed before being evicted. Defaults to None, which means
+          # entries are never evicted based on time.
+          #
+          #expiry_time: 30m
+
+          # Controls how long the results of a /sync request are cached for after
+          # a successful response is returned. A higher duration can help clients with
+          # intermittent connections, at the cost of higher memory usage.
+          #
+          # By default, this is zero, which means that sync responses are not cached
+          # at all.
+          #
+          #sync_response_cache_duration: 2m
         """
 
     def read_config(self, config, **kwargs):
         self.event_cache_size = self.parse_size(
             config.get("event_cache_size", _DEFAULT_EVENT_CACHE_SIZE)
         )
-        self.cache_factors = {}  # type: Dict[str, float]
+        self.cache_factors: Dict[str, float] = {}
 
         cache_config = config.get("caches") or {}
         self.global_factor = cache_config.get(
@@ -190,6 +206,25 @@ def read_config(self, config, **kwargs):
                 )
             self.cache_factors[cache] = factor
 
+        self.track_memory_usage = cache_config.get("track_memory_usage", False)
+        if self.track_memory_usage:
+            try:
+                check_requirements("cache_memory")
+            except DependencyException as e:
+                raise ConfigError(
+                    e.message  # noqa: B306, DependencyException.message is a property
+                )
+
+        expiry_time = cache_config.get("expiry_time")
+        if expiry_time:
+            self.expiry_time_msec = self.parse_duration(expiry_time)
+        else:
+            self.expiry_time_msec = None
+
+        self.sync_response_cache_duration = self.parse_duration(
+            cache_config.get("sync_response_cache_duration", 0)
+        )
+
         # Resize all caches (if necessary) with the new factors we've loaded
         self.resize_all_caches()
 
diff --git a/synapse/config/cas.py b/synapse/config/cas.py
index dbf5085965..901f4123e1 100644
--- a/synapse/config/cas.py
+++ b/synapse/config/cas.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/config/consent_config.py b/synapse/config/consent.py
similarity index 97%
rename from synapse/config/consent_config.py
rename to synapse/config/consent.py
index c47f364b14..b05a9bd97f 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,7 +22,7 @@
 # User Consent configuration
 #
 # for detailed instructions, see
-# https://github.com/matrix-org/synapse/blob/master/docs/consent_tracking.md
+# https://matrix-org.github.io/synapse/latest/consent_tracking.html
 #
 # Parts of this section are required if enabling the 'consent' resource under
 # 'listeners', in particular 'template_dir' and 'version'.
diff --git a/synapse/config/database.py b/synapse/config/database.py
index e7889b9c20..651e31b576 100644
--- a/synapse/config/database.py
+++ b/synapse/config/database.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -34,6 +33,9 @@
 # 'name' gives the database engine to use: either 'sqlite3' (for SQLite) or
 # 'psycopg2' (for PostgreSQL).
 #
+# 'txn_limit' gives the maximum number of transactions to run per connection
+# before reconnecting. Defaults to 0, which means no limit.
+#
 # 'args' gives options which are passed through to the database engine,
 # except for options starting 'cp_', which are used to configure the Twisted
 # connection pool. For a reference to valid arguments, see:
@@ -54,15 +56,18 @@
 #
 #database:
 #  name: psycopg2
+#  txn_limit: 10000
 #  args:
 #    user: synapse_user
 #    password: secretpassword
 #    database: synapse
 #    host: localhost
+#    port: 5432
 #    cp_min: 5
 #    cp_max: 10
 #
-# For more information on using Synapse with Postgres, see `docs/postgres.md`.
+# For more information on using Synapse with Postgres,
+# see https://matrix-org.github.io/synapse/latest/postgres.html.
 #
 database:
   name: sqlite3
diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py
index 5431691831..4477419196 100644
--- a/synapse/config/emailconfig.py
+++ b/synapse/config/emailconfig.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -40,12 +39,13 @@
     "messages_from_person_and_others": "[%(app)s] You have messages on %(app)s from %(person)s and others...",
     "invite_from_person": "[%(app)s] %(person)s has invited you to chat on %(app)s...",
     "invite_from_person_to_room": "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s...",
+    "invite_from_person_to_space": "[%(app)s] %(person)s has invited you to join the %(space)s space on %(app)s...",
     "password_reset": "[%(server_name)s] Password reset",
     "email_validation": "[%(server_name)s] Validate your email",
 }
 
 
-@attr.s
+@attr.s(slots=True, frozen=True)
 class EmailSubjectConfig:
     message_from_person_in_room = attr.ib(type=str)
     message_from_person = attr.ib(type=str)
@@ -55,6 +55,7 @@ class EmailSubjectConfig:
     messages_from_person_and_others = attr.ib(type=str)
     invite_from_person = attr.ib(type=str)
     invite_from_person_to_room = attr.ib(type=str)
+    invite_from_person_to_space = attr.ib(type=str)
     password_reset = attr.ib(type=str)
     email_validation = attr.ib(type=str)
 
@@ -79,6 +80,12 @@ def read_config(self, config, **kwargs):
         self.require_transport_security = email_config.get(
             "require_transport_security", False
         )
+        self.enable_smtp_tls = email_config.get("enable_tls", True)
+        if self.require_transport_security and not self.enable_smtp_tls:
+            raise ConfigError(
+                "email.require_transport_security requires email.enable_tls to be true"
+            )
+
         if "app_name" in email_config:
             self.email_app_name = email_config["app_name"]
         else:
@@ -135,9 +142,9 @@ def read_config(self, config, **kwargs):
 
                 # trusted_third_party_id_servers does not contain a scheme whereas
                 # account_threepid_delegate_email is expected to. Presume https
-                self.account_threepid_delegate_email = (
+                self.account_threepid_delegate_email: Optional[str] = (
                     "https://" + first_trusted_identity_server
-                )  # type: Optional[str]
+                )
                 self.using_identity_server_from_trusted_list = True
             else:
                 raise ConfigError(
@@ -250,7 +257,14 @@ def read_config(self, config, **kwargs):
                     registration_template_success_html,
                     add_threepid_template_success_html,
                 ],
-                template_dir,
+                (
+                    td
+                    for td in (
+                        self.root.server.custom_template_directory,
+                        template_dir,
+                    )
+                    if td
+                ),  # Filter out template_dir if not provided
             )
 
             # Render templates that do not contain any placeholders
@@ -290,7 +304,14 @@ def read_config(self, config, **kwargs):
                 self.email_notif_template_text,
             ) = self.read_templates(
                 [notif_template_html, notif_template_text],
-                template_dir,
+                (
+                    td
+                    for td in (
+                        self.root.server.custom_template_directory,
+                        template_dir,
+                    )
+                    if td
+                ),  # Filter out template_dir if not provided
             )
 
             self.email_notif_for_new_users = email_config.get(
@@ -313,7 +334,14 @@ def read_config(self, config, **kwargs):
                 self.account_validity_template_text,
             ) = self.read_templates(
                 [expiry_template_html, expiry_template_text],
-                template_dir,
+                (
+                    td
+                    for td in (
+                        self.root.server.custom_template_directory,
+                        template_dir,
+                    )
+                    if td
+                ),  # Filter out template_dir if not provided
             )
 
         subjects_config = email_config.get("subjects", {})
@@ -345,6 +373,9 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs):
             """\
         # Configuration for sending emails from Synapse.
         #
+        # Server admins can configure custom templates for email content. See
+        # https://matrix-org.github.io/synapse/latest/templates.html for more information.
+        #
         email:
           # The hostname of the outgoing SMTP server to use. Defaults to 'localhost'.
           #
@@ -367,6 +398,14 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs):
           #
           #require_transport_security: true
 
+          # Uncomment the following to disable TLS for SMTP.
+          #
+          # By default, if the server supports TLS, it will be used, and the server
+          # must present a certificate that is valid for 'smtp_host'. If this option
+          # is set to false, TLS will not be used.
+          #
+          #enable_tls: false
+
           # notif_from defines the "From" address to use when sending emails.
           # It must be set if email sending is enabled.
           #
@@ -413,49 +452,6 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs):
           #
           #invite_client_location: https://app.element.io
 
-          # Directory in which Synapse will try to find the template files below.
-          # If not set, or the files named below are not found within the template
-          # directory, default templates from within the Synapse package will be used.
-          #
-          # Synapse will look for the following templates in this directory:
-          #
-          # * The contents of email notifications of missed events: 'notif_mail.html' and
-          #   'notif_mail.txt'.
-          #
-          # * The contents of account expiry notice emails: 'notice_expiry.html' and
-          #   'notice_expiry.txt'.
-          #
-          # * The contents of password reset emails sent by the homeserver:
-          #   'password_reset.html' and 'password_reset.txt'
-          #
-          # * An HTML page that a user will see when they follow the link in the password
-          #   reset email. The user will be asked to confirm the action before their
-          #   password is reset: 'password_reset_confirmation.html'
-          #
-          # * HTML pages for success and failure that a user will see when they confirm
-          #   the password reset flow using the page above: 'password_reset_success.html'
-          #   and 'password_reset_failure.html'
-          #
-          # * The contents of address verification emails sent during registration:
-          #   'registration.html' and 'registration.txt'
-          #
-          # * HTML pages for success and failure that a user will see when they follow
-          #   the link in an address verification email sent during registration:
-          #   'registration_success.html' and 'registration_failure.html'
-          #
-          # * The contents of address verification emails sent when an address is added
-          #   to a Matrix account: 'add_threepid.html' and 'add_threepid.txt'
-          #
-          # * HTML pages for success and failure that a user will see when they follow
-          #   the link in an address verification email sent when an address is added
-          #   to a Matrix account: 'add_threepid_success.html' and
-          #   'add_threepid_failure.html'
-          #
-          # You can see the default templates at:
-          # https://github.com/matrix-org/synapse/tree/master/synapse/res/templates
-          #
-          #template_dir: "res/templates"
-
           # Subjects to use when sending emails from Synapse.
           #
           # The placeholder '%%(app)s' will be replaced with the value of the 'app_name'
diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py
index 001bddc6f6..907df9591a 100644
--- a/synapse/config/experimental.py
+++ b/synapse/config/experimental.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions
 from synapse.config._base import Config
 from synapse.types import JsonDict
 
@@ -26,15 +24,20 @@ class ExperimentalConfig(Config):
     def read_config(self, config: JsonDict, **kwargs):
         experimental = config.get("experimental_features") or {}
 
-        # MSC2403 (room knocking)
-        self.msc2403_enabled = experimental.get("msc2403_enabled", False)  # type: bool
-        if self.msc2403_enabled:
-            # Enable the MSC2403 unstable room version
-            KNOWN_ROOM_VERSIONS.update({RoomVersions.V7.identifier: RoomVersions.V7})
-
         # MSC2858 (multiple SSO identity providers)
-        self.msc2858_enabled = experimental.get("msc2858_enabled", False)  # type: bool
-        # Spaces (MSC1772, MSC2946, etc)
-        self.spaces_enabled = experimental.get("spaces_enabled", False)  # type: bool
+        self.msc2858_enabled: bool = experimental.get("msc2858_enabled", False)
+
         # MSC3026 (busy presence state)
-        self.msc3026_enabled = experimental.get("msc3026_enabled", False)  # type: bool
+        self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)
+
+        # MSC2716 (backfill existing history)
+        self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False)
+
+        # MSC2285 (hidden read receipts)
+        self.msc2285_enabled: bool = experimental.get("msc2285_enabled", False)
+
+        # MSC3244 (room version capabilities)
+        self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True)
+
+        # MSC3266 (room summary api)
+        self.msc3266_enabled: bool = experimental.get("msc3266_enabled", False)
diff --git a/synapse/config/federation.py b/synapse/config/federation.py
index 55e4db5442..7d64993e22 100644
--- a/synapse/config/federation.py
+++ b/synapse/config/federation.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,7 +22,7 @@ class FederationConfig(Config):
 
     def read_config(self, config, **kwargs):
         # FIXME: federation_domain_whitelist needs sytests
-        self.federation_domain_whitelist = None  # type: Optional[dict]
+        self.federation_domain_whitelist: Optional[dict] = None
         federation_domain_whitelist = config.get("federation_domain_whitelist", None)
 
         if federation_domain_whitelist is not None:
@@ -45,6 +44,10 @@ def read_config(self, config, **kwargs):
             "allow_profile_lookup_over_federation", True
         )
 
+        self.allow_device_name_lookup_over_federation = config.get(
+            "allow_device_name_lookup_over_federation", True
+        )
+
     def generate_config_section(self, config_dir_path, server_name, **kwargs):
         return """\
         ## Federation ##
@@ -76,6 +79,12 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs):
         # on this homeserver. Defaults to 'true'.
         #
         #allow_profile_lookup_over_federation: false
+
+        # Uncomment to disable device display name lookup over federation. By default, the
+        # Federation API allows other homeservers to obtain device display names of any user
+        # on this homeserver. Defaults to 'true'.
+        #
+        #allow_device_name_lookup_over_federation: false
         """
 
 
diff --git a/synapse/config/groups.py b/synapse/config/groups.py
index 7b7860ea71..15c2e64bda 100644
--- a/synapse/config/groups.py
+++ b/synapse/config/groups.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index 58961679ff..1f42a51857 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
+# Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,17 +19,18 @@
 from .cache import CacheConfig
 from .captcha import CaptchaConfig
 from .cas import CasConfig
-from .consent_config import ConsentConfig
+from .consent import ConsentConfig
 from .database import DatabaseConfig
 from .emailconfig import EmailConfig
 from .experimental import ExperimentalConfig
 from .federation import FederationConfig
 from .groups import GroupsConfig
-from .jwt_config import JWTConfig
+from .jwt import JWTConfig
 from .key import KeyConfig
 from .logger import LoggingConfig
 from .metrics import MetricsConfig
-from .oidc_config import OIDCConfig
+from .modules import ModulesConfig
+from .oidc import OIDCConfig
 from .password_auth_providers import PasswordAuthProviderConfig
 from .push import PushConfig
 from .ratelimiting import RatelimitConfig
@@ -40,9 +39,9 @@
 from .repository import ContentRepositoryConfig
 from .room import RoomConfig
 from .room_directory import RoomDirectoryConfig
-from .saml2_config import SAML2Config
+from .saml2 import SAML2Config
 from .server import ServerConfig
-from .server_notices_config import ServerNoticesConfig
+from .server_notices import ServerNoticesConfig
 from .spam_checker import SpamCheckerConfig
 from .sso import SSOConfig
 from .stats import StatsConfig
@@ -57,8 +56,8 @@
 class HomeServerConfig(RootConfig):
 
     config_classes = [
+        ModulesConfig,
         ServerConfig,
-        ExperimentalConfig,
         TlsConfig,
         FederationConfig,
         CacheConfig,
@@ -95,4 +94,5 @@ class HomeServerConfig(RootConfig):
         TracerConfig,
         WorkerConfig,
         RedisConfig,
+        ExperimentalConfig,
     ]
diff --git a/synapse/config/jwt_config.py b/synapse/config/jwt.py
similarity index 97%
rename from synapse/config/jwt_config.py
rename to synapse/config/jwt.py
index f30330abb6..9d295f5856 100644
--- a/synapse/config/jwt_config.py
+++ b/synapse/config/jwt.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015 Niklas Riekenbrauck
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -65,7 +64,7 @@ def generate_config_section(self, **kwargs):
         # Note that this is a non-standard login type and client support is
         # expected to be non-existent.
         #
-        # See https://github.com/matrix-org/synapse/blob/master/docs/jwt.md.
+        # See https://matrix-org.github.io/synapse/latest/jwt.html.
         #
         #jwt_config:
             # Uncomment the following to enable authorization using JSON web
diff --git a/synapse/config/key.py b/synapse/config/key.py
index 350ff1d665..94a9063043 100644
--- a/synapse/config/key.py
+++ b/synapse/config/key.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index 999aecce5c..4a398a7932 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -32,7 +31,6 @@
 )
 
 import synapse
-from synapse.app import _base as appbase
 from synapse.logging._structured import setup_structured_logging
 from synapse.logging.context import LoggingContextFilter
 from synapse.logging.filter import MetadataFilter
@@ -51,7 +49,7 @@
 # be ingested by ELK stacks. See [2] for details.
 #
 # [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
-# [2]: https://github.com/matrix-org/synapse/blob/master/docs/structured_logging.md
+# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
 
 version: 1
 
@@ -69,18 +67,31 @@
         backupCount: 3  # Does not include the current log file.
         encoding: utf8
 
-    # Default to buffering writes to log file for efficiency. This means that
-    # will be a delay for INFO/DEBUG logs to get written, but WARNING/ERROR
-    # logs will still be flushed immediately.
+    # Default to buffering writes to log file for efficiency.
+    # WARNING/ERROR logs will still be flushed immediately, but there will be a
+    # delay (of up to `period` seconds, or until the buffer is full with
+    # `capacity` messages) before INFO/DEBUG logs get written.
     buffer:
-        class: logging.handlers.MemoryHandler
+        class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler
         target: file
-        # The capacity is the number of log lines that are buffered before
-        # being written to disk. Increasing this will lead to better
+
+        # The capacity is the maximum number of log lines that are buffered
+        # before being written to disk. Increasing this will lead to better
         # performance, at the expensive of it taking longer for log lines to
         # be written to disk.
+        # This parameter is required.
         capacity: 10
-        flushLevel: 30  # Flush for WARNING logs as well
+
+        # Logs with a level at or above the flush level will cause the buffer to
+        # be flushed immediately.
+        # Default value: 40 (ERROR)
+        # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG)
+        flushLevel: 30  # Flush immediately for WARNING logs and higher
+
+        # The period of time, in seconds, between forced flushes.
+        # Messages will not be delayed for longer than this time.
+        # Default value: 5 seconds
+        period: 5
 
     # A handler that writes logs to stderr. Unused by default, but can be used
     # instead of "buffer" and "file" in the logger handlers.
@@ -261,9 +272,7 @@ def _log(event: dict) -> None:
         finally:
             threadlocal.active = False
 
-    logBeginner.beginLoggingTo([_log], redirectStandardIO=not config.no_redirect_stdio)
-    if not config.no_redirect_stdio:
-        print("Redirected stdout/stderr to logs")
+    logBeginner.beginLoggingTo([_log], redirectStandardIO=False)
 
 
 def _load_logging_config(log_config_path: str) -> None:
@@ -319,6 +328,8 @@ def setup_logging(
     # Perform one-time logging configuration.
     _setup_stdlib_logging(config, log_config_path, logBeginner=logBeginner)
     # Add a SIGHUP handler to reload the logging configuration, if one is available.
+    from synapse.app import _base as appbase
+
     appbase.register_sighup(_reload_logging_config, log_config_path)
 
     # Log immediately so we can grep backwards.
diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py
index 2b289f4208..7ac82edb0e 100644
--- a/synapse/config/metrics.py
+++ b/synapse/config/metrics.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
diff --git a/synapse/config/modules.py b/synapse/config/modules.py
new file mode 100644
index 0000000000..ae0821e5a5
--- /dev/null
+++ b/synapse/config/modules.py
@@ -0,0 +1,49 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict, List, Tuple
+
+from synapse.config._base import Config, ConfigError
+from synapse.util.module_loader import load_module
+
+
+class ModulesConfig(Config):
+    section = "modules"
+
+    def read_config(self, config: dict, **kwargs):
+        self.loaded_modules: List[Tuple[Any, Dict]] = []
+
+        configured_modules = config.get("modules") or []
+        for i, module in enumerate(configured_modules):
+            config_path = ("modules", "<item %i>" % i)
+            if not isinstance(module, dict):
+                raise ConfigError("expected a mapping", config_path)
+
+            self.loaded_modules.append(load_module(module, config_path))
+
+    def generate_config_section(self, **kwargs):
+        return """
+            ## Modules ##
+
+            # Server admins can expand Synapse's functionality with external modules.
+            #
+            # See https://matrix-org.github.io/synapse/latest/modules.html for more
+            # documentation on how to configure or create custom modules for Synapse.
+            #
+            modules:
+                # - module: my_super_module.MySuperClass
+                #   config:
+                #       do_thing: true
+                # - module: my_other_super_module.SomeClass
+                #   config: {}
+            """
diff --git a/synapse/config/oidc_config.py b/synapse/config/oidc.py
similarity index 96%
rename from synapse/config/oidc_config.py
rename to synapse/config/oidc.py
index 05733ec41d..ba89d11cf0 100644
--- a/synapse/config/oidc_config.py
+++ b/synapse/config/oidc.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Quentin Gliech
 # Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 #
@@ -15,20 +14,23 @@
 # limitations under the License.
 
 from collections import Counter
-from typing import Iterable, List, Mapping, Optional, Tuple, Type
+from typing import Collection, Iterable, List, Mapping, Optional, Tuple, Type
 
 import attr
 
 from synapse.config._util import validate_config
 from synapse.config.sso import SsoAttributeRequirement
 from synapse.python_dependencies import DependencyException, check_requirements
-from synapse.types import Collection, JsonDict
+from synapse.types import JsonDict
 from synapse.util.module_loader import load_module
 from synapse.util.stringutils import parse_and_validate_mxc_uri
 
 from ._base import Config, ConfigError, read_file
 
-DEFAULT_USER_MAPPING_PROVIDER = "synapse.handlers.oidc_handler.JinjaOidcMappingProvider"
+DEFAULT_USER_MAPPING_PROVIDER = "synapse.handlers.oidc.JinjaOidcMappingProvider"
+# The module that JinjaOidcMappingProvider is in was renamed, we want to
+# transparently handle both the same.
+LEGACY_USER_MAPPING_PROVIDER = "synapse.handlers.oidc_handler.JinjaOidcMappingProvider"
 
 
 class OIDCConfig(Config):
@@ -164,7 +166,7 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs):
         #
         #       module: The class name of a custom mapping module. Default is
         #           {mapping_provider!r}.
-        #           See https://github.com/matrix-org/synapse/blob/master/docs/sso_mapping_providers.md#openid-mapping-providers
+        #           See https://matrix-org.github.io/synapse/latest/sso_mapping_providers.html#openid-mapping-providers
         #           for information on implementing a custom mapping provider.
         #
         #       config: Configuration for the mapping provider module. This section will
@@ -215,7 +217,7 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs):
         #     - attribute: groups
         #       value: "admin"
         #
-        # See https://github.com/matrix-org/synapse/blob/master/docs/openid.md
+        # See https://matrix-org.github.io/synapse/latest/openid.html
         # for information on how to configure these options.
         #
         # For backwards compatibility, it is also possible to configure a single OIDC
@@ -404,6 +406,8 @@ def _parse_oidc_config_dict(
     """
     ump_config = oidc_config.get("user_mapping_provider", {})
     ump_config.setdefault("module", DEFAULT_USER_MAPPING_PROVIDER)
+    if ump_config.get("module") == LEGACY_USER_MAPPING_PROVIDER:
+        ump_config["module"] = DEFAULT_USER_MAPPING_PROVIDER
     ump_config.setdefault("config", {})
 
     (
@@ -456,7 +460,7 @@ def _parse_oidc_config_dict(
             ) from e
 
     client_secret_jwt_key_config = oidc_config.get("client_secret_jwt_key")
-    client_secret_jwt_key = None  # type: Optional[OidcProviderClientSecretJwtKey]
+    client_secret_jwt_key: Optional[OidcProviderClientSecretJwtKey] = None
     if client_secret_jwt_key_config is not None:
         keyfile = client_secret_jwt_key_config.get("key_file")
         if keyfile:
diff --git a/synapse/config/password_auth_providers.py b/synapse/config/password_auth_providers.py
index 85d07c4f8f..0f5b2b3977 100644
--- a/synapse/config/password_auth_providers.py
+++ b/synapse/config/password_auth_providers.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 Openmarket
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,7 +25,7 @@ class PasswordAuthProviderConfig(Config):
     section = "authproviders"
 
     def read_config(self, config, **kwargs):
-        self.password_providers = []  # type: List[Any]
+        self.password_providers: List[Any] = []
         providers = []
 
         # We want to be backwards compatible with the old `ldap_config`
@@ -58,7 +57,7 @@ def generate_config_section(self, **kwargs):
         # ex. LDAP, external tokens, etc.
         #
         # For more information and known implementations, please see
-        # https://github.com/matrix-org/synapse/blob/master/docs/password_auth_providers.md
+        # https://matrix-org.github.io/synapse/latest/password_auth_providers.html
         #
         # Note: instances wishing to use SAML or CAS authentication should
         # instead use the `saml2_config` or `cas_config` options,
diff --git a/synapse/config/push.py b/synapse/config/push.py
index 7831a2ef79..6ef8491caf 100644
--- a/synapse/config/push.py
+++ b/synapse/config/push.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 New Vector Ltd
 #
diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py
index 19322372a9..8fdaa59326 100644
--- a/synapse/config/ratelimiting.py
+++ b/synapse/config/ratelimiting.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Dict
+from typing import Dict, Optional
 
 from ._base import Config
 
@@ -21,8 +21,10 @@ class RateLimitConfig:
     def __init__(
         self,
         config: Dict[str, float],
-        defaults={"per_second": 0.17, "burst_count": 3.0},
+        defaults: Optional[Dict[str, float]] = None,
     ):
+        defaults = defaults or {"per_second": 0.17, "burst_count": 3.0}
+
         self.per_second = config.get("per_second", defaults["per_second"])
         self.burst_count = int(config.get("burst_count", defaults["burst_count"]))
 
diff --git a/synapse/config/redis.py b/synapse/config/redis.py
index 1373302335..33104af734 100644
--- a/synapse/config/redis.py
+++ b/synapse/config/redis.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index b49e6609ce..6e9f405312 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -154,6 +153,27 @@ def read_config(self, config, **kwargs):
             session_lifetime = self.parse_duration(session_lifetime)
         self.session_lifetime = session_lifetime
 
+        # The `access_token_lifetime` applies for tokens that can be renewed
+        # using a refresh token, as per MSC2918. If it is `None`, the refresh
+        # token mechanism is disabled.
+        #
+        # Since it is incompatible with the `session_lifetime` mechanism, it is set to
+        # `None` by default if a `session_lifetime` is set.
+        access_token_lifetime = config.get(
+            "access_token_lifetime", "5m" if session_lifetime is None else None
+        )
+        if access_token_lifetime is not None:
+            access_token_lifetime = self.parse_duration(access_token_lifetime)
+        self.access_token_lifetime = access_token_lifetime
+
+        if session_lifetime is not None and access_token_lifetime is not None:
+            raise ConfigError(
+                "The refresh token mechanism is incompatible with the "
+                "`session_lifetime` option. Consider disabling the "
+                "`session_lifetime` option or disabling the refresh token "
+                "mechanism by removing the `access_token_lifetime` option."
+            )
+
         # The success template used during fallback auth.
         self.fallback_success_template = self.read_template("auth_success.html")
 
@@ -243,9 +263,9 @@ def generate_config_section(self, generate_secrets=False, **kwargs):
         #
         #allowed_local_3pids:
         #  - medium: email
-        #    pattern: '.*@matrix\\.org'
+        #    pattern: '^[^@]+@matrix\\.org$'
         #  - medium: email
-        #    pattern: '.*@vector\\.im'
+        #    pattern: '^[^@]+@vector\\.im$'
         #  - medium: msisdn
         #    pattern: '\\+44'
 
@@ -478,4 +498,4 @@ def add_arguments(parser):
 
     def read_arguments(self, args):
         if args.enable_registration is not None:
-            self.enable_registration = bool(strtobool(str(args.enable_registration)))
+            self.enable_registration = strtobool(str(args.enable_registration))
diff --git a/synapse/config/repository.py b/synapse/config/repository.py
index 6cade50acc..06a5cef76b 100644
--- a/synapse/config/repository.py
+++ b/synapse/config/repository.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014, 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import os
 from collections import namedtuple
 from typing import Dict, List
+from urllib.request import getproxies_environment  # type: ignore
 
 from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set
 from synapse.python_dependencies import DependencyException, check_requirements
@@ -23,6 +24,8 @@
 
 from ._base import Config, ConfigError
 
+logger = logging.getLogger(__name__)
+
 DEFAULT_THUMBNAIL_SIZES = [
     {"width": 32, "height": 32, "method": "crop"},
     {"width": 96, "height": 96, "method": "crop"},
@@ -37,6 +40,9 @@
         #    method: %(method)s
 """
 
+HTTP_PROXY_SET_WARNING = """\
+The Synapse config url_preview_ip_range_blacklist will be ignored as an HTTP(s) proxy is configured."""
+
 ThumbnailRequirement = namedtuple(
     "ThumbnailRequirement", ["width", "height", "method", "media_type"]
 )
@@ -63,7 +69,7 @@ def parse_thumbnail_requirements(thumbnail_sizes):
         Dictionary mapping from media type string to list of
         ThumbnailRequirement tuples.
     """
-    requirements = {}  # type: Dict[str, List]
+    requirements: Dict[str, List] = {}
     for size in thumbnail_sizes:
         width = size["width"]
         height = size["height"]
@@ -71,6 +77,7 @@ def parse_thumbnail_requirements(thumbnail_sizes):
         jpeg_thumbnail = ThumbnailRequirement(width, height, method, "image/jpeg")
         png_thumbnail = ThumbnailRequirement(width, height, method, "image/png")
         requirements.setdefault("image/jpeg", []).append(jpeg_thumbnail)
+        requirements.setdefault("image/jpg", []).append(jpeg_thumbnail)
         requirements.setdefault("image/webp", []).append(jpeg_thumbnail)
         requirements.setdefault("image/gif", []).append(png_thumbnail)
         requirements.setdefault("image/png", []).append(png_thumbnail)
@@ -147,7 +154,7 @@ def read_config(self, config, **kwargs):
         #
         # We don't create the storage providers here as not all workers need
         # them to be started.
-        self.media_storage_providers = []  # type: List[tuple]
+        self.media_storage_providers: List[tuple] = []
 
         for i, provider_config in enumerate(storage_providers):
             # We special case the module "file_system" so as not to need to
@@ -186,12 +193,17 @@ def read_config(self, config, **kwargs):
                     e.message  # noqa: B306, DependencyException.message is a property
                 )
 
+            proxy_env = getproxies_environment()
             if "url_preview_ip_range_blacklist" not in config:
-                raise ConfigError(
-                    "For security, you must specify an explicit target IP address "
-                    "blacklist in url_preview_ip_range_blacklist for url previewing "
-                    "to work"
-                )
+                if "http" not in proxy_env or "https" not in proxy_env:
+                    raise ConfigError(
+                        "For security, you must specify an explicit target IP address "
+                        "blacklist in url_preview_ip_range_blacklist for url previewing "
+                        "to work"
+                    )
+            else:
+                if "http" in proxy_env or "https" in proxy_env:
+                    logger.warning("".join(HTTP_PROXY_SET_WARNING))
 
             # we always blacklist '0.0.0.0' and '::', which are supposed to be
             # unroutable addresses.
@@ -254,6 +266,10 @@ def generate_config_section(self, data_dir_path, **kwargs):
 
         # The largest allowed upload size in bytes
         #
+        # If you are using a reverse proxy you may also need to set this value in
+        # your reverse proxy's config. Notably Nginx has a small max body size by default.
+        # See https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
+        #
         #max_upload_size: 50M
 
         # The largest allowed size for a user avatar. If not defined, no
@@ -318,6 +334,8 @@ def generate_config_section(self, data_dir_path, **kwargs):
         # This must be specified if url_preview_enabled is set. It is recommended that
         # you uncomment the following list as a starting point.
         #
+        # Note: The value is ignored when an HTTP proxy is in use
+        #
         #url_preview_ip_range_blacklist:
 %(ip_range_blacklist)s
 
diff --git a/synapse/config/room.py b/synapse/config/room.py
index 692d7a1936..d889d90dbc 100644
--- a/synapse/config/room.py
+++ b/synapse/config/room.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py
index 2dd719c388..56981cac79 100644
--- a/synapse/config/room_directory.py
+++ b/synapse/config/room_directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/config/saml2_config.py b/synapse/config/saml2.py
similarity index 95%
rename from synapse/config/saml2_config.py
rename to synapse/config/saml2.py
index 6db9cb5ced..05e983625d 100644
--- a/synapse/config/saml2_config.py
+++ b/synapse/config/saml2.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
@@ -26,7 +25,10 @@
 
 logger = logging.getLogger(__name__)
 
-DEFAULT_USER_MAPPING_PROVIDER = (
+DEFAULT_USER_MAPPING_PROVIDER = "synapse.handlers.saml.DefaultSamlMappingProvider"
+# The module that DefaultSamlMappingProvider is in was renamed, we want to
+# transparently handle both the same.
+LEGACY_USER_MAPPING_PROVIDER = (
     "synapse.handlers.saml_handler.DefaultSamlMappingProvider"
 )
 
@@ -98,6 +100,8 @@ def read_config(self, config, **kwargs):
 
         # Use the default user mapping provider if not set
         ump_dict.setdefault("module", DEFAULT_USER_MAPPING_PROVIDER)
+        if ump_dict.get("module") == LEGACY_USER_MAPPING_PROVIDER:
+            ump_dict["module"] = DEFAULT_USER_MAPPING_PROVIDER
 
         # Ensure a config is present
         ump_dict["config"] = ump_dict.get("config") or {}
@@ -160,7 +164,13 @@ def read_config(self, config, **kwargs):
         config_path = saml2_config.get("config_path", None)
         if config_path is not None:
             mod = load_python_module(config_path)
-            _dict_merge(merge_dict=mod.CONFIG, into_dict=saml2_config_dict)
+            config = getattr(mod, "CONFIG", None)
+            if config is None:
+                raise ConfigError(
+                    "Config path specified by saml2_config.config_path does not "
+                    "have a CONFIG property."
+                )
+            _dict_merge(merge_dict=config, into_dict=saml2_config_dict)
 
         import saml2.config
 
diff --git a/synapse/config/server.py b/synapse/config/server.py
index c8b1a25004..9276d5c3f4 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -20,13 +19,14 @@
 import os.path
 import re
 from textwrap import indent
-from typing import Any, Dict, Iterable, List, Optional, Set
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
 
 import attr
 import yaml
 from netaddr import AddrFormatError, IPNetwork, IPSet
 
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
+from synapse.util.module_loader import load_module
 from synapse.util.stringutils import parse_and_validate_server_name
 
 from ._base import Config, ConfigError
@@ -153,7 +153,7 @@ def generate_ip_set(
 METRICS_PORT_WARNING = """\
 The metrics_port configuration option is deprecated in Synapse 0.31 in favour of
 a listener. Please see
-https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
+https://matrix-org.github.io/synapse/latest/metrics-howto.html
 on how to configure the new listener.
 --------------------------------------------------------------------------------"""
 
@@ -235,10 +235,27 @@ def read_config(self, config, **kwargs):
         self.print_pidfile = config.get("print_pidfile")
         self.user_agent_suffix = config.get("user_agent_suffix")
         self.use_frozen_dicts = config.get("use_frozen_dicts", False)
+
         self.public_baseurl = config.get("public_baseurl")
+        if self.public_baseurl is not None:
+            if self.public_baseurl[-1] != "/":
+                self.public_baseurl += "/"
 
         # Whether to enable user presence.
-        self.use_presence = config.get("use_presence", True)
+        presence_config = config.get("presence") or {}
+        self.use_presence = presence_config.get("enabled")
+        if self.use_presence is None:
+            self.use_presence = config.get("use_presence", True)
+
+        # Custom presence router module
+        self.presence_router_module_class = None
+        self.presence_router_config = None
+        presence_router_config = presence_config.get("presence_router")
+        if presence_router_config:
+            (
+                self.presence_router_module_class,
+                self.presence_router_config,
+            ) = load_module(presence_router_config, ("presence", "presence_router"))
 
         # Whether to update the user directory or not. This should be set to
         # false only if we are updating the user directory in a worker
@@ -380,23 +397,22 @@ def read_config(self, config, **kwargs):
         self.ip_range_whitelist = generate_ip_set(
             config.get("ip_range_whitelist", ()), config_path=("ip_range_whitelist",)
         )
-
         # The federation_ip_range_blacklist is used for backwards-compatibility
-        # and only applies to federation and identity servers. If it is not given,
-        # default to ip_range_blacklist.
-        federation_ip_range_blacklist = config.get(
-            "federation_ip_range_blacklist", ip_range_blacklist
-        )
-        # Always blacklist 0.0.0.0, ::
-        self.federation_ip_range_blacklist = generate_ip_set(
-            federation_ip_range_blacklist,
-            ["0.0.0.0", "::"],
-            config_path=("federation_ip_range_blacklist",),
-        )
-
-        if self.public_baseurl is not None:
-            if self.public_baseurl[-1] != "/":
-                self.public_baseurl += "/"
+        # and only applies to federation and identity servers.
+        if "federation_ip_range_blacklist" in config:
+            # Always blacklist 0.0.0.0, ::
+            self.federation_ip_range_blacklist = generate_ip_set(
+                config["federation_ip_range_blacklist"],
+                ["0.0.0.0", "::"],
+                config_path=("federation_ip_range_blacklist",),
+            )
+            # 'federation_ip_range_whitelist' was never a supported configuration option.
+            self.federation_ip_range_whitelist = None
+        else:
+            # No backwards-compatiblity requrired, as federation_ip_range_blacklist
+            # is not given. Default to ip_range_blacklist and ip_range_whitelist.
+            self.federation_ip_range_blacklist = self.ip_range_blacklist
+            self.federation_ip_range_whitelist = self.ip_range_whitelist
 
         # (undocumented) option for torturing the worker-mode replication a bit,
         # for testing. The value defines the number of milliseconds to pause before
@@ -495,7 +511,7 @@ def read_config(self, config, **kwargs):
                 " greater than 'allowed_lifetime_max'"
             )
 
-        self.retention_purge_jobs = []  # type: List[Dict[str, Optional[int]]]
+        self.retention_purge_jobs: List[Dict[str, Optional[int]]] = []
         for purge_job_config in retention_config.get("purge_jobs", []):
             interval_config = purge_job_config.get("interval")
 
@@ -565,6 +581,7 @@ def read_config(self, config, **kwargs):
             _warn_if_webclient_configured(self.listeners)
 
         self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None))
+        self.gc_seconds = self.read_gc_intervals(config.get("gc_min_interval", None))
 
         @attr.s
         class LimitRemoteRoomsConfig:
@@ -677,23 +694,21 @@ class LimitRemoteRoomsConfig:
         # not included in the sample configuration file on purpose as it's a temporary
         # hack, so that some users can trial the new defaults without impacting every
         # user on the homeserver.
-        users_new_default_push_rules = (
+        users_new_default_push_rules: list = (
             config.get("users_new_default_push_rules") or []
-        )  # type: list
+        )
         if not isinstance(users_new_default_push_rules, list):
             raise ConfigError("'users_new_default_push_rules' must be a list")
 
         # Turn the list into a set to improve lookup speed.
-        self.users_new_default_push_rules = set(
-            users_new_default_push_rules
-        )  # type: set
+        self.users_new_default_push_rules: set = set(users_new_default_push_rules)
 
         # Whitelist of domain names that given next_link parameters must have
-        next_link_domain_whitelist = config.get(
+        next_link_domain_whitelist: Optional[List[str]] = config.get(
             "next_link_domain_whitelist"
-        )  # type: Optional[List[str]]
+        )
 
-        self.next_link_domain_whitelist = None  # type: Optional[Set[str]]
+        self.next_link_domain_whitelist: Optional[Set[str]] = None
         if next_link_domain_whitelist is not None:
             if not isinstance(next_link_domain_whitelist, list):
                 raise ConfigError("'next_link_domain_whitelist' must be a list")
@@ -701,6 +716,18 @@ class LimitRemoteRoomsConfig:
             # Turn the list into a set to improve lookup speed.
             self.next_link_domain_whitelist = set(next_link_domain_whitelist)
 
+        templates_config = config.get("templates") or {}
+        if not isinstance(templates_config, dict):
+            raise ConfigError("The 'templates' section must be a dictionary")
+
+        self.custom_template_directory = templates_config.get(
+            "custom_template_directory"
+        )
+        if self.custom_template_directory is not None and not isinstance(
+            self.custom_template_directory, str
+        ):
+            raise ConfigError("'custom_template_directory' must be a string")
+
     def has_tls_listener(self) -> bool:
         return any(listener.tls for listener in self.listeners)
 
@@ -800,7 +827,7 @@ def generate_config_section(
         # In most cases you should avoid using a matrix specific subdomain such as
         # matrix.example.com or synapse.example.com as the server_name for the same
         # reasons you wouldn't use user@email.example.com as your email address.
-        # See https://github.com/matrix-org/synapse/blob/master/docs/delegate.md
+        # See https://matrix-org.github.io/synapse/latest/delegate.html
         # for information on how to host Synapse on a subdomain while preserving
         # a clean server_name.
         #
@@ -840,9 +867,28 @@ def generate_config_section(
         #
         #soft_file_limit: 0
 
-        # Set to false to disable presence tracking on this homeserver.
+        # Presence tracking allows users to see the state (e.g online/offline)
+        # of other local and remote users.
         #
-        #use_presence: false
+        presence:
+          # Uncomment to disable presence tracking on this homeserver. This option
+          # replaces the previous top-level 'use_presence' option.
+          #
+          #enabled: false
+
+          # Presence routers are third-party modules that can specify additional logic
+          # to where presence updates from users are routed.
+          #
+          presence_router:
+            # The custom module's class. Uncomment to use a custom presence router module.
+            #
+            #module: "my_custom_router.PresenceRouter"
+
+            # Configuration options of the custom module. Refer to your module's
+            # documentation for available options.
+            #
+            #config:
+            #  example_option: 'something'
 
         # Whether to require authentication to retrieve profile data (avatars,
         # display names) of other users through the client API. Defaults to
@@ -891,6 +937,16 @@ def generate_config_section(
         #
         #gc_thresholds: [700, 10, 10]
 
+        # The minimum time in seconds between each GC for a generation, regardless of
+        # the GC thresholds. This ensures that we don't do GC too frequently.
+        #
+        # A value of `[1s, 10s, 30s]` indicates that a second must pass between consecutive
+        # generation 0 GCs, etc.
+        #
+        # Defaults to `[1s, 10s, 30s]`.
+        #
+        #gc_min_interval: [0.5s, 30s, 1m]
+
         # Set the limit on the returned events in the timeline in the get
         # and sync operations. The default value is 100. -1 means no upper limit.
         #
@@ -922,6 +978,8 @@ def generate_config_section(
         #
         # This option replaces federation_ip_range_blacklist in Synapse v1.25.0.
         #
+        # Note: The value is ignored when an HTTP proxy is in use
+        #
         #ip_range_blacklist:
 %(ip_range_blacklist)s
 
@@ -948,9 +1006,9 @@ def generate_config_section(
         #       'all local interfaces'.
         #
         #   type: the type of listener. Normally 'http', but other valid options are:
-        #       'manhole' (see docs/manhole.md),
-        #       'metrics' (see docs/metrics-howto.md),
-        #       'replication' (see docs/workers.md).
+        #       'manhole' (see https://matrix-org.github.io/synapse/latest/manhole.html),
+        #       'metrics' (see https://matrix-org.github.io/synapse/latest/metrics-howto.html),
+        #       'replication' (see https://matrix-org.github.io/synapse/latest/workers.html).
         #
         #   tls: set to true to enable TLS for this listener. Will use the TLS
         #       key/cert specified in tls_private_key_path / tls_certificate_path.
@@ -975,8 +1033,8 @@ def generate_config_section(
         #   client: the client-server API (/_matrix/client), and the synapse admin
         #       API (/_synapse/admin). Also implies 'media' and 'static'.
         #
-        #   consent: user consent forms (/_matrix/consent). See
-        #       docs/consent_tracking.md.
+        #   consent: user consent forms (/_matrix/consent).
+        #       See https://matrix-org.github.io/synapse/latest/consent_tracking.html.
         #
         #   federation: the server-server API (/_matrix/federation). Also implies
         #       'media', 'keys', 'openid'
@@ -985,12 +1043,13 @@ def generate_config_section(
         #
         #   media: the media API (/_matrix/media).
         #
-        #   metrics: the metrics interface. See docs/metrics-howto.md.
+        #   metrics: the metrics interface.
+        #       See https://matrix-org.github.io/synapse/latest/metrics-howto.html.
         #
         #   openid: OpenID authentication.
         #
-        #   replication: the HTTP replication API (/_synapse/replication). See
-        #       docs/workers.md.
+        #   replication: the HTTP replication API (/_synapse/replication).
+        #       See https://matrix-org.github.io/synapse/latest/workers.html.
         #
         #   static: static resources under synapse/static (/_matrix/static). (Mostly
         #       useful for 'fallback authentication'.)
@@ -1010,7 +1069,7 @@ def generate_config_section(
           # that unwraps TLS.
           #
           # If you plan to use a reverse proxy, please see
-          # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md.
+          # https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
           #
           %(unsecure_http_bindings)s
 
@@ -1311,6 +1370,19 @@ def generate_config_section(
         # all domains.
         #
         #next_link_domain_whitelist: ["matrix.org"]
+
+        # Templates to use when generating email or HTML page contents.
+        #
+        templates:
+          # Directory in which Synapse will try to find template files to use to generate
+          # email or HTML page contents.
+          # If not set, or a file is not found within the template directory, a default
+          # template from within the Synapse package will be used.
+          #
+          # See https://matrix-org.github.io/synapse/latest/templates.html for more
+          # information about using custom templates.
+          #
+          #custom_template_directory: /path/to/custom/templates/
         """
             % locals()
         )
@@ -1347,6 +1419,24 @@ def add_arguments(parser):
             help="Turn on the twisted telnet manhole service on the given port.",
         )
 
+    def read_gc_intervals(self, durations) -> Optional[Tuple[float, float, float]]:
+        """Reads the three durations for the GC min interval option, returning seconds."""
+        if durations is None:
+            return None
+
+        try:
+            if len(durations) != 3:
+                raise ValueError()
+            return (
+                self.parse_duration(durations[0]) / 1000,
+                self.parse_duration(durations[1]) / 1000,
+                self.parse_duration(durations[2]) / 1000,
+            )
+        except Exception:
+            raise ConfigError(
+                "Value of `gc_min_interval` must be a list of three durations if set"
+            )
+
 
 def is_threepid_reserved(reserved_threepids, threepid):
     """Check the threepid against the reserved threepid config
diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices.py
similarity index 99%
rename from synapse/config/server_notices_config.py
rename to synapse/config/server_notices.py
index 57f69dc8e2..48bf3241b6 100644
--- a/synapse/config/server_notices_config.py
+++ b/synapse/config/server_notices.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/config/spam_checker.py b/synapse/config/spam_checker.py
index 3d05abc158..a233a9ce03 100644
--- a/synapse/config/spam_checker.py
+++ b/synapse/config/spam_checker.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 from typing import Any, Dict, List, Tuple
 
 from synapse.config import ConfigError
@@ -20,12 +20,21 @@
 
 from ._base import Config
 
+logger = logging.getLogger(__name__)
+
+LEGACY_SPAM_CHECKER_WARNING = """
+This server is using a spam checker module that is implementing the deprecated spam
+checker interface. Please check with the module's maintainer to see if a new version
+supporting Synapse's generic modules system is available.
+For more information, please see https://matrix-org.github.io/synapse/latest/modules.html
+---------------------------------------------------------------------------------------"""
+
 
 class SpamCheckerConfig(Config):
     section = "spamchecker"
 
     def read_config(self, config, **kwargs):
-        self.spam_checkers = []  # type: List[Tuple[Any, Dict]]
+        self.spam_checkers: List[Tuple[Any, Dict]] = []
 
         spam_checkers = config.get("spam_checker") or []
         if isinstance(spam_checkers, dict):
@@ -44,17 +53,7 @@ def read_config(self, config, **kwargs):
         else:
             raise ConfigError("spam_checker syntax is incorrect")
 
-    def generate_config_section(self, **kwargs):
-        return """\
-        # Spam checkers are third-party modules that can block specific actions
-        # of local users, such as creating rooms and registering undesirable
-        # usernames, as well as remote users by redacting incoming events.
-        #
-        spam_checker:
-           #- module: "my_custom_project.SuperSpamChecker"
-           #  config:
-           #    example_option: 'things'
-           #- module: "some_other_project.BadEventStopper"
-           #  config:
-           #    example_stop_events_from: ['@bad:example.com']
-        """
+        # If this configuration is being used in any way, warn the admin that it is going
+        # away soon.
+        if self.spam_checkers:
+            logger.warning(LEGACY_SPAM_CHECKER_WARNING)
diff --git a/synapse/config/sso.py b/synapse/config/sso.py
index 243cc681e8..fe1177ab81 100644
--- a/synapse/config/sso.py
+++ b/synapse/config/sso.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -40,12 +39,17 @@ class SSOConfig(Config):
     section = "sso"
 
     def read_config(self, config, **kwargs):
-        sso_config = config.get("sso") or {}  # type: Dict[str, Any]
+        sso_config: Dict[str, Any] = config.get("sso") or {}
 
         # The sso-specific template_dir
         self.sso_template_dir = sso_config.get("template_dir")
 
         # Read templates from disk
+        custom_template_directories = (
+            self.root.server.custom_template_directory,
+            self.sso_template_dir,
+        )
+
         (
             self.sso_login_idp_picker_template,
             self.sso_redirect_confirm_template,
@@ -64,7 +68,7 @@ def read_config(self, config, **kwargs):
                 "sso_auth_success.html",
                 "sso_auth_bad_user.html",
             ],
-            self.sso_template_dir,
+            (td for td in custom_template_directories if td),
         )
 
         # These templates have no placeholders, so render them here
@@ -75,6 +79,10 @@ def read_config(self, config, **kwargs):
 
         self.sso_client_whitelist = sso_config.get("client_whitelist") or []
 
+        self.sso_update_profile_information = (
+            sso_config.get("update_profile_information") or False
+        )
+
         # Attempt to also whitelist the server's login fallback, since that fallback sets
         # the redirect URL to itself (so it can process the login token then return
         # gracefully to the client). This would make it pointless to ask the user for
@@ -91,6 +99,9 @@ def generate_config_section(self, **kwargs):
         # Additional settings to use with single-sign on systems such as OpenID Connect,
         # SAML2 and CAS.
         #
+        # Server admins can configure custom templates for pages related to SSO. See
+        # https://matrix-org.github.io/synapse/latest/templates.html for more information.
+        #
         sso:
             # A list of client URLs which are whitelisted so that the user does not
             # have to confirm giving access to their account to the URL. Any client
@@ -112,166 +123,14 @@ def generate_config_section(self, **kwargs):
             #  - https://riot.im/develop
             #  - https://my.custom.client/
 
-            # Directory in which Synapse will try to find the template files below.
-            # If not set, or the files named below are not found within the template
-            # directory, default templates from within the Synapse package will be used.
-            #
-            # Synapse will look for the following templates in this directory:
-            #
-            # * HTML page to prompt the user to choose an Identity Provider during
-            #   login: 'sso_login_idp_picker.html'.
-            #
-            #   This is only used if multiple SSO Identity Providers are configured.
-            #
-            #   When rendering, this template is given the following variables:
-            #     * redirect_url: the URL that the user will be redirected to after
-            #       login.
-            #
-            #     * server_name: the homeserver's name.
-            #
-            #     * providers: a list of available Identity Providers. Each element is
-            #       an object with the following attributes:
-            #
-            #         * idp_id: unique identifier for the IdP
-            #         * idp_name: user-facing name for the IdP
-            #         * idp_icon: if specified in the IdP config, an MXC URI for an icon
-            #              for the IdP
-            #         * idp_brand: if specified in the IdP config, a textual identifier
-            #              for the brand of the IdP
-            #
-            #   The rendered HTML page should contain a form which submits its results
-            #   back as a GET request, with the following query parameters:
-            #
-            #     * redirectUrl: the client redirect URI (ie, the `redirect_url` passed
-            #       to the template)
-            #
-            #     * idp: the 'idp_id' of the chosen IDP.
-            #
-            # * HTML page to prompt new users to enter a userid and confirm other
-            #   details: 'sso_auth_account_details.html'. This is only shown if the
-            #   SSO implementation (with any user_mapping_provider) does not return
-            #   a localpart.
-            #
-            #   When rendering, this template is given the following variables:
-            #
-            #     * server_name: the homeserver's name.
-            #
-            #     * idp: details of the SSO Identity Provider that the user logged in
-            #       with: an object with the following attributes:
-            #
-            #         * idp_id: unique identifier for the IdP
-            #         * idp_name: user-facing name for the IdP
-            #         * idp_icon: if specified in the IdP config, an MXC URI for an icon
-            #              for the IdP
-            #         * idp_brand: if specified in the IdP config, a textual identifier
-            #              for the brand of the IdP
-            #
-            #     * user_attributes: an object containing details about the user that
-            #       we received from the IdP. May have the following attributes:
-            #
-            #         * display_name: the user's display_name
-            #         * emails: a list of email addresses
-            #
-            #   The template should render a form which submits the following fields:
-            #
-            #     * username: the localpart of the user's chosen user id
-            #
-            # * HTML page allowing the user to consent to the server's terms and
-            #   conditions. This is only shown for new users, and only if
-            #   `user_consent.require_at_registration` is set.
-            #
-            #   When rendering, this template is given the following variables:
-            #
-            #     * server_name: the homeserver's name.
-            #
-            #     * user_id: the user's matrix proposed ID.
-            #
-            #     * user_profile.display_name: the user's proposed display name, if any.
-            #
-            #     * consent_version: the version of the terms that the user will be
-            #       shown
-            #
-            #     * terms_url: a link to the page showing the terms.
-            #
-            #   The template should render a form which submits the following fields:
-            #
-            #     * accepted_version: the version of the terms accepted by the user
-            #       (ie, 'consent_version' from the input variables).
-            #
-            # * HTML page for a confirmation step before redirecting back to the client
-            #   with the login token: 'sso_redirect_confirm.html'.
-            #
-            #   When rendering, this template is given the following variables:
-            #
-            #     * redirect_url: the URL the user is about to be redirected to.
-            #
-            #     * display_url: the same as `redirect_url`, but with the query
-            #                    parameters stripped. The intention is to have a
-            #                    human-readable URL to show to users, not to use it as
-            #                    the final address to redirect to.
-            #
-            #     * server_name: the homeserver's name.
-            #
-            #     * new_user: a boolean indicating whether this is the user's first time
-            #          logging in.
-            #
-            #     * user_id: the user's matrix ID.
-            #
-            #     * user_profile.avatar_url: an MXC URI for the user's avatar, if any.
-            #           None if the user has not set an avatar.
-            #
-            #     * user_profile.display_name: the user's display name. None if the user
-            #           has not set a display name.
-            #
-            # * HTML page which notifies the user that they are authenticating to confirm
-            #   an operation on their account during the user interactive authentication
-            #   process: 'sso_auth_confirm.html'.
-            #
-            #   When rendering, this template is given the following variables:
-            #     * redirect_url: the URL the user is about to be redirected to.
-            #
-            #     * description: the operation which the user is being asked to confirm
-            #
-            #     * idp: details of the Identity Provider that we will use to confirm
-            #       the user's identity: an object with the following attributes:
-            #
-            #         * idp_id: unique identifier for the IdP
-            #         * idp_name: user-facing name for the IdP
-            #         * idp_icon: if specified in the IdP config, an MXC URI for an icon
-            #              for the IdP
-            #         * idp_brand: if specified in the IdP config, a textual identifier
-            #              for the brand of the IdP
-            #
-            # * HTML page shown after a successful user interactive authentication session:
-            #   'sso_auth_success.html'.
-            #
-            #   Note that this page must include the JavaScript which notifies of a successful authentication
-            #   (see https://matrix.org/docs/spec/client_server/r0.6.0#fallback).
-            #
-            #   This template has no additional variables.
-            #
-            # * HTML page shown after a user-interactive authentication session which
-            #   does not map correctly onto the expected user: 'sso_auth_bad_user.html'.
-            #
-            #   When rendering, this template is given the following variables:
-            #     * server_name: the homeserver's name.
-            #     * user_id_to_verify: the MXID of the user that we are trying to
-            #       validate.
-            #
-            # * HTML page shown during single sign-on if a deactivated user (according to Synapse's database)
-            #   attempts to login: 'sso_account_deactivated.html'.
-            #
-            #   This template has no additional variables.
-            #
-            # * HTML page to display to users if something goes wrong during the
-            #   OpenID Connect authentication process: 'sso_error.html'.
-            #
-            #   When rendering, this template is given two variables:
-            #     * error: the technical name of the error
-            #     * error_description: a human-readable message for the error
+            # Uncomment to keep a user's profile fields in sync with information from
+            # the identity provider. Currently only syncing the displayname is
+            # supported. Fields are checked on every SSO login, and are updated
+            # if necessary.
             #
-            # You can see the default templates at:
-            # https://github.com/matrix-org/synapse/tree/master/synapse/res/templates
+            # Note that enabling this option will override user profile information,
+            # regardless of whether users have opted-out of syncing that
+            # information when first signing in. Defaults to false.
             #
-            #template_dir: "res/templates"
+            #update_profile_information: true
         """
diff --git a/synapse/config/stats.py b/synapse/config/stats.py
index 2258329a52..6f253e00c0 100644
--- a/synapse/config/stats.py
+++ b/synapse/config/stats.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -39,20 +38,16 @@ class StatsConfig(Config):
 
     def read_config(self, config, **kwargs):
         self.stats_enabled = True
-        self.stats_bucket_size = 86400 * 1000
         stats_config = config.get("stats", None)
         if stats_config:
             self.stats_enabled = stats_config.get("enabled", self.stats_enabled)
-            self.stats_bucket_size = self.parse_duration(
-                stats_config.get("bucket_size", "1d")
-            )
         if not self.stats_enabled:
             logger.warning(ROOM_STATS_DISABLED_WARN)
 
     def generate_config_section(self, config_dir_path, server_name, **kwargs):
         return """
         # Settings for local room and user statistics collection. See
-        # docs/room_and_user_statistics.md.
+        # https://matrix-org.github.io/synapse/latest/room_and_user_statistics.html.
         #
         stats:
           # Uncomment the following to disable room and user statistics. Note that doing
@@ -60,9 +55,4 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs):
           # correctly.
           #
           #enabled: false
-
-          # The size of each timeslice in the room_stats_historical and
-          # user_stats_historical tables, as a time period. Defaults to "1d".
-          #
-          #bucket_size: 1h
         """
diff --git a/synapse/config/third_party_event_rules.py b/synapse/config/third_party_event_rules.py
index c04e1c4e07..a3fae02420 100644
--- a/synapse/config/third_party_event_rules.py
+++ b/synapse/config/third_party_event_rules.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -29,18 +28,3 @@ def read_config(self, config, **kwargs):
             self.third_party_event_rules = load_module(
                 provider, ("third_party_event_rules",)
             )
-
-    def generate_config_section(self, **kwargs):
-        return """\
-        # Server admins can define a Python module that implements extra rules for
-        # allowing or denying incoming events. In order to work, this module needs to
-        # override the methods defined in synapse/events/third_party_rules.py.
-        #
-        # This feature is designed to be used in closed federations only, where each
-        # participating server enforces the same rules.
-        #
-        #third_party_event_rules:
-        #  module: "my_custom_project.SuperRulesSet"
-        #  config:
-        #    example_option: 'things'
-        """
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index ad37b93c02..5679f05e42 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,12 +14,8 @@
 
 import logging
 import os
-import warnings
 from datetime import datetime
-from hashlib import sha256
-from typing import List, Optional
-
-from unpaddedbase64 import encode_base64
+from typing import List, Optional, Pattern
 
 from OpenSSL import SSL, crypto
 from twisted.internet._sslverify import Certificate, trustRootFromCertificates
@@ -30,45 +25,12 @@
 
 logger = logging.getLogger(__name__)
 
-ACME_SUPPORT_ENABLED_WARN = """\
-This server uses Synapse's built-in ACME support. Note that ACME v1 has been
-deprecated by Let's Encrypt, and that Synapse doesn't currently support ACME v2,
-which means that this feature will not work with Synapse installs set up after
-November 2019, and that it may stop working on June 2020 for installs set up
-before that date.
-
-For more info and alternative solutions, see
-https://github.com/matrix-org/synapse/blob/master/docs/ACME.md#deprecation-of-acme-v1
---------------------------------------------------------------------------------"""
-
 
 class TlsConfig(Config):
     section = "tls"
 
     def read_config(self, config: dict, config_dir_path: str, **kwargs):
 
-        acme_config = config.get("acme", None)
-        if acme_config is None:
-            acme_config = {}
-
-        self.acme_enabled = acme_config.get("enabled", False)
-
-        if self.acme_enabled:
-            logger.warning(ACME_SUPPORT_ENABLED_WARN)
-
-        # hyperlink complains on py2 if this is not a Unicode
-        self.acme_url = str(
-            acme_config.get("url", "https://acme-v01.api.letsencrypt.org/directory")
-        )
-        self.acme_port = acme_config.get("port", 80)
-        self.acme_bind_addresses = acme_config.get("bind_addresses", ["::", "0.0.0.0"])
-        self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30)
-        self.acme_domain = acme_config.get("domain", config.get("server_name"))
-
-        self.acme_account_key_file = self.abspath(
-            acme_config.get("account_key_file", config_dir_path + "/client.key")
-        )
-
         self.tls_certificate_file = self.abspath(config.get("tls_certificate_path"))
         self.tls_private_key_file = self.abspath(config.get("tls_private_key_path"))
 
@@ -84,13 +46,6 @@ def read_config(self, config: dict, config_dir_path: str, **kwargs):
                     "configured."
                 )
 
-        self._original_tls_fingerprints = config.get("tls_fingerprints", [])
-
-        if self._original_tls_fingerprints is None:
-            self._original_tls_fingerprints = []
-
-        self.tls_fingerprints = list(self._original_tls_fingerprints)
-
         # Whether to verify certificates on outbound federation traffic
         self.federation_verify_certificates = config.get(
             "federation_verify_certificates", True
@@ -111,10 +66,8 @@ def read_config(self, config: dict, config_dir_path: str, **kwargs):
         if self.federation_client_minimum_tls_version == "1.3":
             if getattr(SSL, "OP_NO_TLSv1_3", None) is None:
                 raise ConfigError(
-                    (
-                        "federation_client_minimum_tls_version cannot be 1.3, "
-                        "your OpenSSL does not support it"
-                    )
+                    "federation_client_minimum_tls_version cannot be 1.3, "
+                    "your OpenSSL does not support it"
                 )
 
         # Whitelist of domains to not verify certificates for
@@ -125,7 +78,7 @@ def read_config(self, config: dict, config_dir_path: str, **kwargs):
             fed_whitelist_entries = []
 
         # Support globs (*) in whitelist values
-        self.federation_certificate_verification_whitelist = []  # type: List[str]
+        self.federation_certificate_verification_whitelist: List[Pattern] = []
         for entry in fed_whitelist_entries:
             try:
                 entry_regex = glob_to_regex(entry.encode("ascii").decode("ascii"))
@@ -177,8 +130,8 @@ def read_config(self, config: dict, config_dir_path: str, **kwargs):
             "use_insecure_ssl_client_just_for_testing_do_not_use"
         )
 
-        self.tls_certificate = None  # type: Optional[crypto.X509]
-        self.tls_private_key = None  # type: Optional[crypto.PKey]
+        self.tls_certificate: Optional[crypto.X509] = None
+        self.tls_private_key: Optional[crypto.PKey] = None
 
     def is_disk_cert_valid(self, allow_self_signed=True):
         """
@@ -226,41 +179,12 @@ def is_disk_cert_valid(self, allow_self_signed=True):
         days_remaining = (expires_on - now).days
         return days_remaining
 
-    def read_certificate_from_disk(self, require_cert_and_key: bool):
+    def read_certificate_from_disk(self):
         """
         Read the certificates and private key from disk.
-
-        Args:
-            require_cert_and_key: set to True to throw an error if the certificate
-                and key file are not given
         """
-        if require_cert_and_key:
-            self.tls_private_key = self.read_tls_private_key()
-            self.tls_certificate = self.read_tls_certificate()
-        elif self.tls_certificate_file:
-            # we only need the certificate for the tls_fingerprints. Reload it if we
-            # can, but it's not a fatal error if we can't.
-            try:
-                self.tls_certificate = self.read_tls_certificate()
-            except Exception as e:
-                logger.info(
-                    "Unable to read TLS certificate (%s). Ignoring as no "
-                    "tls listeners enabled.",
-                    e,
-                )
-
-        self.tls_fingerprints = list(self._original_tls_fingerprints)
-
-        if self.tls_certificate:
-            # Check that our own certificate is included in the list of fingerprints
-            # and include it if it is not.
-            x509_certificate_bytes = crypto.dump_certificate(
-                crypto.FILETYPE_ASN1, self.tls_certificate
-            )
-            sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest())
-            sha256_fingerprints = {f["sha256"] for f in self.tls_fingerprints}
-            if sha256_fingerprint not in sha256_fingerprints:
-                self.tls_fingerprints.append({"sha256": sha256_fingerprint})
+        self.tls_private_key = self.read_tls_private_key()
+        self.tls_certificate = self.read_tls_certificate()
 
     def generate_config_section(
         self,
@@ -269,11 +193,9 @@ def generate_config_section(
         data_dir_path,
         tls_certificate_path,
         tls_private_key_path,
-        acme_domain,
-        **kwargs
+        **kwargs,
     ):
-        """If the acme_domain is specified acme will be enabled.
-        If the TLS paths are not specified the default will be certs in the
+        """If the TLS paths are not specified the default will be certs in the
         config directory"""
 
         base_key_name = os.path.join(config_dir_path, server_name)
@@ -283,28 +205,15 @@ def generate_config_section(
                 "Please specify both a cert path and a key path or neither."
             )
 
-        tls_enabled = (
-            "" if tls_certificate_path and tls_private_key_path or acme_domain else "#"
-        )
+        tls_enabled = "" if tls_certificate_path and tls_private_key_path else "#"
 
         if not tls_certificate_path:
             tls_certificate_path = base_key_name + ".tls.crt"
         if not tls_private_key_path:
             tls_private_key_path = base_key_name + ".tls.key"
 
-        acme_enabled = bool(acme_domain)
-        acme_domain = "matrix.example.com"
-
-        default_acme_account_file = os.path.join(data_dir_path, "acme_account.key")
-
-        # this is to avoid the max line length. Sorrynotsorry
-        proxypassline = (
-            "ProxyPass /.well-known/acme-challenge "
-            "http://localhost:8009/.well-known/acme-challenge"
-        )
-
         # flake8 doesn't recognise that variables are used in the below string
-        _ = tls_enabled, proxypassline, acme_enabled, default_acme_account_file
+        _ = tls_enabled
 
         return (
             """\
@@ -314,13 +223,9 @@ def generate_config_section(
         # This certificate, as of Synapse 1.0, will need to be a valid and verifiable
         # certificate, signed by a recognised Certificate Authority.
         #
-        # See 'ACME support' below to enable auto-provisioning this certificate via
-        # Let's Encrypt.
-        #
-        # If supplying your own, be sure to use a `.pem` file that includes the
-        # full certificate chain including any intermediate certificates (for
-        # instance, if using certbot, use `fullchain.pem` as your certificate,
-        # not `cert.pem`).
+        # Be sure to use a `.pem` file that includes the full certificate chain including
+        # any intermediate certificates (for instance, if using certbot, use
+        # `fullchain.pem` as your certificate, not `cert.pem`).
         #
         %(tls_enabled)stls_certificate_path: "%(tls_certificate_path)s"
 
@@ -370,107 +275,6 @@ def generate_config_section(
         #  - myCA1.pem
         #  - myCA2.pem
         #  - myCA3.pem
-
-        # ACME support: This will configure Synapse to request a valid TLS certificate
-        # for your configured `server_name` via Let's Encrypt.
-        #
-        # Note that ACME v1 is now deprecated, and Synapse currently doesn't support
-        # ACME v2. This means that this feature currently won't work with installs set
-        # up after November 2019. For more info, and alternative solutions, see
-        # https://github.com/matrix-org/synapse/blob/master/docs/ACME.md#deprecation-of-acme-v1
-        #
-        # Note that provisioning a certificate in this way requires port 80 to be
-        # routed to Synapse so that it can complete the http-01 ACME challenge.
-        # By default, if you enable ACME support, Synapse will attempt to listen on
-        # port 80 for incoming http-01 challenges - however, this will likely fail
-        # with 'Permission denied' or a similar error.
-        #
-        # There are a couple of potential solutions to this:
-        #
-        #  * If you already have an Apache, Nginx, or similar listening on port 80,
-        #    you can configure Synapse to use an alternate port, and have your web
-        #    server forward the requests. For example, assuming you set 'port: 8009'
-        #    below, on Apache, you would write:
-        #
-        #    %(proxypassline)s
-        #
-        #  * Alternatively, you can use something like `authbind` to give Synapse
-        #    permission to listen on port 80.
-        #
-        acme:
-            # ACME support is disabled by default. Set this to `true` and uncomment
-            # tls_certificate_path and tls_private_key_path above to enable it.
-            #
-            enabled: %(acme_enabled)s
-
-            # Endpoint to use to request certificates. If you only want to test,
-            # use Let's Encrypt's staging url:
-            #     https://acme-staging.api.letsencrypt.org/directory
-            #
-            #url: https://acme-v01.api.letsencrypt.org/directory
-
-            # Port number to listen on for the HTTP-01 challenge. Change this if
-            # you are forwarding connections through Apache/Nginx/etc.
-            #
-            port: 80
-
-            # Local addresses to listen on for incoming connections.
-            # Again, you may want to change this if you are forwarding connections
-            # through Apache/Nginx/etc.
-            #
-            bind_addresses: ['::', '0.0.0.0']
-
-            # How many days remaining on a certificate before it is renewed.
-            #
-            reprovision_threshold: 30
-
-            # The domain that the certificate should be for. Normally this
-            # should be the same as your Matrix domain (i.e., 'server_name'), but,
-            # by putting a file at 'https://<server_name>/.well-known/matrix/server',
-            # you can delegate incoming traffic to another server. If you do that,
-            # you should give the target of the delegation here.
-            #
-            # For example: if your 'server_name' is 'example.com', but
-            # 'https://example.com/.well-known/matrix/server' delegates to
-            # 'matrix.example.com', you should put 'matrix.example.com' here.
-            #
-            # If not set, defaults to your 'server_name'.
-            #
-            domain: %(acme_domain)s
-
-            # file to use for the account key. This will be generated if it doesn't
-            # exist.
-            #
-            # If unspecified, we will use CONFDIR/client.key.
-            #
-            account_key_file: %(default_acme_account_file)s
-
-        # List of allowed TLS fingerprints for this server to publish along
-        # with the signing keys for this server. Other matrix servers that
-        # make HTTPS requests to this server will check that the TLS
-        # certificates returned by this server match one of the fingerprints.
-        #
-        # Synapse automatically adds the fingerprint of its own certificate
-        # to the list. So if federation traffic is handled directly by synapse
-        # then no modification to the list is required.
-        #
-        # If synapse is run behind a load balancer that handles the TLS then it
-        # will be necessary to add the fingerprints of the certificates used by
-        # the loadbalancers to this list if they are different to the one
-        # synapse is using.
-        #
-        # Homeservers are permitted to cache the list of TLS fingerprints
-        # returned in the key responses up to the "valid_until_ts" returned in
-        # key. It may be necessary to publish the fingerprints of a new
-        # certificate and wait until the "valid_until_ts" of the previous key
-        # responses have passed before deploying it.
-        #
-        # You can calculate a fingerprint from a given TLS listener via:
-        # openssl s_client -connect $host:$port < /dev/null 2> /dev/null |
-        #   openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '='
-        # or by checking matrix.org/federationtester/api/report?server_name=$host
-        #
-        #tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
         """
             # Lowercase the string representation of boolean values
             % {
@@ -482,8 +286,6 @@ def generate_config_section(
     def read_tls_certificate(self) -> crypto.X509:
         """Reads the TLS certificate from the configured file, and returns it
 
-        Also checks if it is self-signed, and warns if so
-
         Returns:
             The certificate
         """
@@ -492,16 +294,6 @@ def read_tls_certificate(self) -> crypto.X509:
         cert_pem = self.read_file(cert_path, "tls_certificate_path")
         cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
 
-        # Check if it is self-signed, and issue a warning if so.
-        if cert.get_issuer() == cert.get_subject():
-            warnings.warn(
-                (
-                    "Self-signed TLS certificates will not be accepted by Synapse 1.0. "
-                    "Please either provide a valid certificate, or use Synapse's ACME "
-                    "support to provision one."
-                )
-            )
-
         return cert
 
     def read_tls_private_key(self) -> crypto.PKey:
diff --git a/synapse/config/tracer.py b/synapse/config/tracer.py
index 727a1e7008..21b9a88353 100644
--- a/synapse/config/tracer.py
+++ b/synapse/config/tracer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.d
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Set
+
 from synapse.python_dependencies import DependencyException, check_requirements
 
 from ._base import Config, ConfigError
@@ -33,6 +34,8 @@ def read_config(self, config, **kwargs):
             {"sampler": {"type": "const", "param": 1}, "logging": False},
         )
 
+        self.force_tracing_for_users: Set[str] = set()
+
         if not self.opentracer_enabled:
             return
 
@@ -49,6 +52,19 @@ def read_config(self, config, **kwargs):
         if not isinstance(self.opentracer_whitelist, list):
             raise ConfigError("Tracer homeserver_whitelist config is malformed")
 
+        force_tracing_for_users = opentracing_config.get("force_tracing_for_users", [])
+        if not isinstance(force_tracing_for_users, list):
+            raise ConfigError(
+                "Expected a list", ("opentracing", "force_tracing_for_users")
+            )
+        for i, u in enumerate(force_tracing_for_users):
+            if not isinstance(u, str):
+                raise ConfigError(
+                    "Expected a string",
+                    ("opentracing", "force_tracing_for_users", f"index {i}"),
+                )
+            self.force_tracing_for_users.add(u)
+
     def generate_config_section(cls, **kwargs):
         return """\
         ## Opentracing ##
@@ -65,7 +81,8 @@ def generate_config_section(cls, **kwargs):
             #enabled: true
 
             # The list of homeservers we wish to send and receive span contexts and span baggage.
-            # See docs/opentracing.rst
+            # See https://matrix-org.github.io/synapse/latest/opentracing.html.
+            #
             # This is a list of regexes which are matched against the server_name of the
             # homeserver.
             #
@@ -74,19 +91,26 @@ def generate_config_section(cls, **kwargs):
             #homeserver_whitelist:
             #  - ".*"
 
+            # A list of the matrix IDs of users whose requests will always be traced,
+            # even if the tracing system would otherwise drop the traces due to
+            # probabilistic sampling.
+            #
+            # By default, the list is empty.
+            #
+            #force_tracing_for_users:
+            #  - "@user1:server_name"
+            #  - "@user2:server_name"
+
             # Jaeger can be configured to sample traces at different rates.
             # All configuration options provided by Jaeger can be set here.
-            # Jaeger's configuration mostly related to trace sampling which
+            # Jaeger's configuration is mostly related to trace sampling which
             # is documented here:
-            # https://www.jaegertracing.io/docs/1.13/sampling/.
+            # https://www.jaegertracing.io/docs/latest/sampling/.
             #
             #jaeger_config:
             #  sampler:
             #    type: const
             #    param: 1
-
-            #  Logging whether spans were started and reported
-            #
             #  logging:
             #    false
         """
diff --git a/synapse/config/user_directory.py b/synapse/config/user_directory.py
index ab7770e472..f1beb87aea 100644
--- a/synapse/config/user_directory.py
+++ b/synapse/config/user_directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -54,7 +53,7 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs):
             #
             # If you set it true, you'll have to rebuild the user_directory search
             # indexes, see:
-            # https://github.com/matrix-org/synapse/blob/master/docs/user_directory.md
+            # https://matrix-org.github.io/synapse/latest/user_directory.html
             #
             # Uncomment to return search results containing all known users, even if that
             # user does not share a room with the requester.
diff --git a/synapse/config/workers.py b/synapse/config/workers.py
index ac92375a85..462630201d 100644
--- a/synapse/config/workers.py
+++ b/synapse/config/workers.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -65,6 +64,14 @@ class WriterLocations:
     Attributes:
         events: The instances that write to the event and backfill streams.
         typing: The instance that writes to the typing stream.
+        to_device: The instances that write to the to_device stream. Currently
+            can only be a single instance.
+        account_data: The instances that write to the account data streams. Currently
+            can only be a single instance.
+        receipts: The instances that write to the receipts stream. Currently
+            can only be a single instance.
+        presence: The instances that write to the presence stream. Currently
+            can only be a single instance.
     """
 
     events = attr.ib(
@@ -86,6 +93,11 @@ class WriterLocations:
         type=List[str],
         converter=_instance_to_list_converter,
     )
+    presence = attr.ib(
+        default=["master"],
+        type=List[str],
+        converter=_instance_to_list_converter,
+    )
 
 
 class WorkerConfig(Config):
@@ -189,7 +201,14 @@ def read_config(self, config, **kwargs):
 
         # Check that the configured writers for events and typing also appears in
         # `instance_map`.
-        for stream in ("events", "typing", "to_device", "account_data", "receipts"):
+        for stream in (
+            "events",
+            "typing",
+            "to_device",
+            "account_data",
+            "receipts",
+            "presence",
+        ):
             instances = _instance_to_list_converter(getattr(self.writers, stream))
             for instance in instances:
                 if instance != "master" and instance not in self.instance_map:
@@ -216,6 +235,11 @@ def read_config(self, config, **kwargs):
         if len(self.writers.events) == 0:
             raise ConfigError("Must specify at least one instance to handle `events`.")
 
+        if len(self.writers.presence) != 1:
+            raise ConfigError(
+                "Must only specify one instance to handle `presence` messages."
+            )
+
         self.events_shard_config = RoutableShardedWorkerHandlingConfig(
             self.writers.events
         )
diff --git a/synapse/crypto/__init__.py b/synapse/crypto/__init__.py
index bfebb0f644..5e83dba2ed 100644
--- a/synapse/crypto/__init__.py
+++ b/synapse/crypto/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py
index 8fb116ae18..0f2b632e47 100644
--- a/synapse/crypto/event_signing.py
+++ b/synapse/crypto/event_signing.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index d5fb51513b..9e9b1c1c86 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017, 2018 New Vector Ltd
 #
@@ -17,8 +16,7 @@
 import abc
 import logging
 import urllib
-from collections import defaultdict
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple
 
 import attr
 from signedjson.key import (
@@ -43,17 +41,14 @@
     SynapseError,
 )
 from synapse.config.key import TrustedKeyServer
-from synapse.logging.context import (
-    PreserveLoggingContext,
-    make_deferred_yieldable,
-    preserve_fn,
-    run_in_background,
-)
+from synapse.events import EventBase
+from synapse.events.utils import prune_event_dict
+from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.storage.keys import FetchKeyResult
 from synapse.types import JsonDict
 from synapse.util import unwrapFirstError
 from synapse.util.async_helpers import yieldable_gather_results
-from synapse.util.metrics import Measure
+from synapse.util.batching_queue import BatchingQueue
 from synapse.util.retryutils import NotRetryingDestination
 
 if TYPE_CHECKING:
@@ -70,40 +65,98 @@ class VerifyJsonRequest:
     Attributes:
         server_name: The name of the server to verify against.
 
-        json_object: The JSON object to verify.
+        get_json_object: A callback to fetch the JSON object to verify.
+            A callback is used to allow deferring the creation of the JSON
+            object to verify until needed, e.g. for events we can defer
+            creating the redacted copy. This reduces the memory usage when
+            there are large numbers of in flight requests.
 
         minimum_valid_until_ts: time at which we require the signing key to
             be valid. (0 implies we don't care)
 
-        request_name: The name of the request.
-
         key_ids: The set of key_ids to that could be used to verify the JSON object
-
-        key_ready (Deferred[str, str, nacl.signing.VerifyKey]):
-            A deferred (server_name, key_id, verify_key) tuple that resolves when
-            a verify key has been fetched. The deferreds' callbacks are run with no
-            logcontext.
-
-            If we are unable to find a key which satisfies the request, the deferred
-            errbacks with an M_UNAUTHORIZED SynapseError.
     """
 
     server_name = attr.ib(type=str)
-    json_object = attr.ib(type=JsonDict)
+    get_json_object = attr.ib(type=Callable[[], JsonDict])
     minimum_valid_until_ts = attr.ib(type=int)
-    request_name = attr.ib(type=str)
-    key_ids = attr.ib(init=False, type=List[str])
-    key_ready = attr.ib(default=attr.Factory(defer.Deferred), type=defer.Deferred)
+    key_ids = attr.ib(type=List[str])
+
+    @staticmethod
+    def from_json_object(
+        server_name: str,
+        json_object: JsonDict,
+        minimum_valid_until_ms: int,
+    ):
+        """Create a VerifyJsonRequest to verify all signatures on a signed JSON
+        object for the given server.
+        """
+        key_ids = signature_ids(json_object, server_name)
+        return VerifyJsonRequest(
+            server_name,
+            lambda: json_object,
+            minimum_valid_until_ms,
+            key_ids=key_ids,
+        )
+
+    @staticmethod
+    def from_event(
+        server_name: str,
+        event: EventBase,
+        minimum_valid_until_ms: int,
+    ):
+        """Create a VerifyJsonRequest to verify all signatures on an event
+        object for the given server.
+        """
+        key_ids = list(event.signatures.get(server_name, []))
+        return VerifyJsonRequest(
+            server_name,
+            # We defer creating the redacted json object, as it uses a lot more
+            # memory than the Event object itself.
+            lambda: prune_event_dict(event.room_version, event.get_pdu_json()),
+            minimum_valid_until_ms,
+            key_ids=key_ids,
+        )
 
-    def __attrs_post_init__(self):
-        self.key_ids = signature_ids(self.json_object, self.server_name)
+    def to_fetch_key_request(self) -> "_FetchKeyRequest":
+        """Create a key fetch request for all keys needed to satisfy the
+        verification request.
+        """
+        return _FetchKeyRequest(
+            server_name=self.server_name,
+            minimum_valid_until_ts=self.minimum_valid_until_ts,
+            key_ids=self.key_ids,
+        )
 
 
 class KeyLookupError(ValueError):
     pass
 
 
+@attr.s(slots=True)
+class _FetchKeyRequest:
+    """A request for keys for a given server.
+
+    We will continue to try and fetch until we have all the keys listed under
+    `key_ids` (with an appropriate `valid_until_ts` property) or we run out of
+    places to fetch keys from.
+
+    Attributes:
+        server_name: The name of the server that owns the keys.
+        minimum_valid_until_ts: The timestamp which the keys must be valid until.
+        key_ids: The IDs of the keys to attempt to fetch
+    """
+
+    server_name = attr.ib(type=str)
+    minimum_valid_until_ts = attr.ib(type=int)
+    key_ids = attr.ib(type=List[str])
+
+
 class Keyring:
+    """Handles verifying signed JSON objects and fetching the keys needed to do
+    so.
+    """
+
     def __init__(
         self, hs: "HomeServer", key_fetchers: "Optional[Iterable[KeyFetcher]]" = None
     ):
@@ -117,22 +170,24 @@ def __init__(
             )
         self._key_fetchers = key_fetchers
 
-        # map from server name to Deferred. Has an entry for each server with
-        # an ongoing key download; the Deferred completes once the download
-        # completes.
-        #
-        # These are regular, logcontext-agnostic Deferreds.
-        self.key_downloads = {}  # type: Dict[str, defer.Deferred]
+        self._server_queue: BatchingQueue[
+            _FetchKeyRequest, Dict[str, Dict[str, FetchKeyResult]]
+        ] = BatchingQueue(
+            "keyring_server",
+            clock=hs.get_clock(),
+            process_batch_callback=self._inner_fetch_key_requests,
+        )
 
-    def verify_json_for_server(
+    async def verify_json_for_server(
         self,
         server_name: str,
         json_object: JsonDict,
         validity_time: int,
-        request_name: str,
-    ) -> defer.Deferred:
+    ) -> None:
         """Verify that a JSON object has been signed by a given server
 
+        Completes if the the object was correctly signed, otherwise raises.
+
         Args:
             server_name: name of the server which must have signed this object
 
@@ -140,360 +195,269 @@ def verify_json_for_server(
 
             validity_time: timestamp at which we require the signing key to
                 be valid. (0 implies we don't care)
-
-            request_name: an identifier for this json object (eg, an event id)
-                for logging.
-
-        Returns:
-            Deferred[None]: completes if the the object was correctly signed, otherwise
-                errbacks with an error
         """
-        req = VerifyJsonRequest(server_name, json_object, validity_time, request_name)
-        requests = (req,)
-        return make_deferred_yieldable(self._verify_objects(requests)[0])
+        request = VerifyJsonRequest.from_json_object(
+            server_name,
+            json_object,
+            validity_time,
+        )
+        return await self.process_request(request)
 
     def verify_json_objects_for_server(
-        self, server_and_json: Iterable[Tuple[str, dict, int, str]]
+        self, server_and_json: Iterable[Tuple[str, dict, int]]
     ) -> List[defer.Deferred]:
         """Bulk verifies signatures of json objects, bulk fetching keys as
         necessary.
 
         Args:
             server_and_json:
-                Iterable of (server_name, json_object, validity_time, request_name)
+                Iterable of (server_name, json_object, validity_time)
                 tuples.
 
                 validity_time is a timestamp at which the signing key must be
                 valid.
 
-                request_name is an identifier for this json object (eg, an event id)
-                for logging.
-
         Returns:
             List<Deferred[None]>: for each input triplet, a deferred indicating success
                 or failure to verify each json object's signature for the given
                 server_name. The deferreds run their callbacks in the sentinel
                 logcontext.
         """
-        return self._verify_objects(
-            VerifyJsonRequest(server_name, json_object, validity_time, request_name)
-            for server_name, json_object, validity_time, request_name in server_and_json
-        )
-
-    def _verify_objects(
-        self, verify_requests: Iterable[VerifyJsonRequest]
-    ) -> List[defer.Deferred]:
-        """Does the work of verify_json_[objects_]for_server
-
-
-        Args:
-            verify_requests: Iterable of verification requests.
-
-        Returns:
-            List<Deferred[None]>: for each input item, a deferred indicating success
-                or failure to verify each json object's signature for the given
-                server_name. The deferreds run their callbacks in the sentinel
-                logcontext.
-        """
-        # a list of VerifyJsonRequests which are awaiting a key lookup
-        key_lookups = []
-        handle = preserve_fn(_handle_key_deferred)
-
-        def process(verify_request: VerifyJsonRequest) -> defer.Deferred:
-            """Process an entry in the request list
-
-            Adds a key request to key_lookups, and returns a deferred which
-            will complete or fail (in the sentinel context) when verification completes.
-            """
-            if not verify_request.key_ids:
-                return defer.fail(
-                    SynapseError(
-                        400,
-                        "Not signed by %s" % (verify_request.server_name,),
-                        Codes.UNAUTHORIZED,
-                    )
-                )
-
-            logger.debug(
-                "Verifying %s for %s with key_ids %s, min_validity %i",
-                verify_request.request_name,
-                verify_request.server_name,
-                verify_request.key_ids,
-                verify_request.minimum_valid_until_ts,
+        return [
+            run_in_background(
+                self.process_request,
+                VerifyJsonRequest.from_json_object(
+                    server_name,
+                    json_object,
+                    validity_time,
+                ),
             )
+            for server_name, json_object, validity_time in server_and_json
+        ]
 
-            # add the key request to the queue, but don't start it off yet.
-            key_lookups.append(verify_request)
-
-            # now run _handle_key_deferred, which will wait for the key request
-            # to complete and then do the verification.
-            #
-            # We want _handle_key_request to log to the right context, so we
-            # wrap it with preserve_fn (aka run_in_background)
-            return handle(verify_request)
-
-        results = [process(r) for r in verify_requests]
-
-        if key_lookups:
-            run_in_background(self._start_key_lookups, key_lookups)
-
-        return results
-
-    async def _start_key_lookups(
-        self, verify_requests: List[VerifyJsonRequest]
+    async def verify_event_for_server(
+        self,
+        server_name: str,
+        event: EventBase,
+        validity_time: int,
     ) -> None:
-        """Sets off the key fetches for each verify request
-
-        Once each fetch completes, verify_request.key_ready will be resolved.
+        await self.process_request(
+            VerifyJsonRequest.from_event(
+                server_name,
+                event,
+                validity_time,
+            )
+        )
 
-        Args:
-            verify_requests:
+    async def process_request(self, verify_request: VerifyJsonRequest) -> None:
+        """Processes the `VerifyJsonRequest`. Raises if the object is not signed
+        by the server, the signatures don't match or we failed to fetch the
+        necessary keys.
         """
 
-        try:
-            # map from server name to a set of outstanding request ids
-            server_to_request_ids = {}  # type: Dict[str, Set[int]]
-
-            for verify_request in verify_requests:
-                server_name = verify_request.server_name
-                request_id = id(verify_request)
-                server_to_request_ids.setdefault(server_name, set()).add(request_id)
-
-            # Wait for any previous lookups to complete before proceeding.
-            await self.wait_for_previous_lookups(server_to_request_ids.keys())
-
-            # take out a lock on each of the servers by sticking a Deferred in
-            # key_downloads
-            for server_name in server_to_request_ids.keys():
-                self.key_downloads[server_name] = defer.Deferred()
-                logger.debug("Got key lookup lock on %s", server_name)
-
-            # When we've finished fetching all the keys for a given server_name,
-            # drop the lock by resolving the deferred in key_downloads.
-            def drop_server_lock(server_name):
-                d = self.key_downloads.pop(server_name)
-                d.callback(None)
-
-            def lookup_done(res, verify_request):
-                server_name = verify_request.server_name
-                server_requests = server_to_request_ids[server_name]
-                server_requests.remove(id(verify_request))
-
-                # if there are no more requests for this server, we can drop the lock.
-                if not server_requests:
-                    logger.debug("Releasing key lookup lock on %s", server_name)
-                    drop_server_lock(server_name)
+        if not verify_request.key_ids:
+            raise SynapseError(
+                400,
+                f"Not signed by {verify_request.server_name}",
+                Codes.UNAUTHORIZED,
+            )
 
-                return res
+        # Add the keys we need to verify to the queue for retrieval. We queue
+        # up requests for the same server so we don't end up with many in flight
+        # requests for the same keys.
+        key_request = verify_request.to_fetch_key_request()
+        found_keys_by_server = await self._server_queue.add_to_queue(
+            key_request, key=verify_request.server_name
+        )
 
-            for verify_request in verify_requests:
-                verify_request.key_ready.addBoth(lookup_done, verify_request)
+        # Since we batch up requests the returned set of keys may contain keys
+        # from other servers, so we pull out only the ones we care about.s
+        found_keys = found_keys_by_server.get(verify_request.server_name, {})
 
-            # Actually start fetching keys.
-            self._get_server_verify_keys(verify_requests)
-        except Exception:
-            logger.exception("Error starting key lookups")
+        # Verify each signature we got valid keys for, raising if we can't
+        # verify any of them.
+        verified = False
+        for key_id in verify_request.key_ids:
+            key_result = found_keys.get(key_id)
+            if not key_result:
+                continue
 
-    async def wait_for_previous_lookups(self, server_names: Iterable[str]) -> None:
-        """Waits for any previous key lookups for the given servers to finish.
+            if key_result.valid_until_ts < verify_request.minimum_valid_until_ts:
+                continue
 
-        Args:
-            server_names: list of servers which we want to look up
+            verify_key = key_result.verify_key
+            json_object = verify_request.get_json_object()
+            try:
+                verify_signed_json(
+                    json_object,
+                    verify_request.server_name,
+                    verify_key,
+                )
+                verified = True
+            except SignatureVerifyException as e:
+                logger.debug(
+                    "Error verifying signature for %s:%s:%s with key %s: %s",
+                    verify_request.server_name,
+                    verify_key.alg,
+                    verify_key.version,
+                    encode_verify_key_base64(verify_key),
+                    str(e),
+                )
+                raise SynapseError(
+                    401,
+                    "Invalid signature for server %s with key %s:%s: %s"
+                    % (
+                        verify_request.server_name,
+                        verify_key.alg,
+                        verify_key.version,
+                        str(e),
+                    ),
+                    Codes.UNAUTHORIZED,
+                )
 
-        Returns:
-            Resolves once all key lookups for the given servers have
-                completed. Follows the synapse rules of logcontext preservation.
-        """
-        loop_count = 1
-        while True:
-            wait_on = [
-                (server_name, self.key_downloads[server_name])
-                for server_name in server_names
-                if server_name in self.key_downloads
-            ]
-            if not wait_on:
-                break
-            logger.info(
-                "Waiting for existing lookups for %s to complete [loop %i]",
-                [w[0] for w in wait_on],
-                loop_count,
+        if not verified:
+            raise SynapseError(
+                401,
+                f"Failed to find any key to satisfy: {key_request}",
+                Codes.UNAUTHORIZED,
             )
-            with PreserveLoggingContext():
-                await defer.DeferredList((w[1] for w in wait_on))
 
-            loop_count += 1
+    async def _inner_fetch_key_requests(
+        self, requests: List[_FetchKeyRequest]
+    ) -> Dict[str, Dict[str, FetchKeyResult]]:
+        """Processing function for the queue of `_FetchKeyRequest`."""
+
+        logger.debug("Starting fetch for %s", requests)
+
+        # First we need to deduplicate requests for the same key. We do this by
+        # taking the *maximum* requested `minimum_valid_until_ts` for each pair
+        # of server name/key ID.
+        server_to_key_to_ts: Dict[str, Dict[str, int]] = {}
+        for request in requests:
+            by_server = server_to_key_to_ts.setdefault(request.server_name, {})
+            for key_id in request.key_ids:
+                existing_ts = by_server.get(key_id, 0)
+                by_server[key_id] = max(request.minimum_valid_until_ts, existing_ts)
+
+        deduped_requests = [
+            _FetchKeyRequest(server_name, minimum_valid_ts, [key_id])
+            for server_name, by_server in server_to_key_to_ts.items()
+            for key_id, minimum_valid_ts in by_server.items()
+        ]
+
+        logger.debug("Deduplicated key requests to %s", deduped_requests)
+
+        # For each key we call `_inner_verify_request` which will handle
+        # fetching each key. Note these shouldn't throw if we fail to contact
+        # other servers etc.
+        results_per_request = await yieldable_gather_results(
+            self._inner_fetch_key_request,
+            deduped_requests,
+        )
 
-    def _get_server_verify_keys(self, verify_requests: List[VerifyJsonRequest]) -> None:
-        """Tries to find at least one key for each verify request
+        # We now convert the returned list of results into a map from server
+        # name to key ID to FetchKeyResult, to return.
+        to_return: Dict[str, Dict[str, FetchKeyResult]] = {}
+        for (request, results) in zip(deduped_requests, results_per_request):
+            to_return_by_server = to_return.setdefault(request.server_name, {})
+            for key_id, key_result in results.items():
+                existing = to_return_by_server.get(key_id)
+                if not existing or existing.valid_until_ts < key_result.valid_until_ts:
+                    to_return_by_server[key_id] = key_result
 
-        For each verify_request, verify_request.key_ready is called back with
-        params (server_name, key_id, VerifyKey) if a key is found, or errbacked
-        with a SynapseError if none of the keys are found.
+        return to_return
 
-        Args:
-            verify_requests: list of verify requests
+    async def _inner_fetch_key_request(
+        self, verify_request: _FetchKeyRequest
+    ) -> Dict[str, FetchKeyResult]:
+        """Attempt to fetch the given key by calling each key fetcher one by
+        one.
         """
+        logger.debug("Starting fetch for %s", verify_request)
 
-        remaining_requests = {rq for rq in verify_requests if not rq.key_ready.called}
+        found_keys: Dict[str, FetchKeyResult] = {}
+        missing_key_ids = set(verify_request.key_ids)
 
-        async def do_iterations():
-            try:
-                with Measure(self.clock, "get_server_verify_keys"):
-                    for f in self._key_fetchers:
-                        if not remaining_requests:
-                            return
-                        await self._attempt_key_fetches_with_fetcher(
-                            f, remaining_requests
-                        )
-
-                    # look for any requests which weren't satisfied
-                    while remaining_requests:
-                        verify_request = remaining_requests.pop()
-                        rq_str = (
-                            "VerifyJsonRequest(server=%s, key_ids=%s, min_valid=%i)"
-                            % (
-                                verify_request.server_name,
-                                verify_request.key_ids,
-                                verify_request.minimum_valid_until_ts,
-                            )
-                        )
-
-                        # If we run the errback immediately, it may cancel our
-                        # loggingcontext while we are still in it, so instead we
-                        # schedule it for the next time round the reactor.
-                        #
-                        # (this also ensures that we don't get a stack overflow if we
-                        # has a massive queue of lookups waiting for this server).
-                        self.clock.call_later(
-                            0,
-                            verify_request.key_ready.errback,
-                            SynapseError(
-                                401,
-                                "Failed to find any key to satisfy %s" % (rq_str,),
-                                Codes.UNAUTHORIZED,
-                            ),
-                        )
-            except Exception as err:
-                # we don't really expect to get here, because any errors should already
-                # have been caught and logged. But if we do, let's log the error and make
-                # sure that all of the deferreds are resolved.
-                logger.error("Unexpected error in _get_server_verify_keys: %s", err)
-                with PreserveLoggingContext():
-                    for verify_request in remaining_requests:
-                        if not verify_request.key_ready.called:
-                            verify_request.key_ready.errback(err)
-
-        run_in_background(do_iterations)
-
-    async def _attempt_key_fetches_with_fetcher(
-        self, fetcher: "KeyFetcher", remaining_requests: Set[VerifyJsonRequest]
-    ):
-        """Use a key fetcher to attempt to satisfy some key requests
+        for fetcher in self._key_fetchers:
+            if not missing_key_ids:
+                break
 
-        Args:
-            fetcher: fetcher to use to fetch the keys
-            remaining_requests: outstanding key requests.
-                Any successfully-completed requests will be removed from the list.
-        """
-        # The keys to fetch.
-        # server_name -> key_id -> min_valid_ts
-        missing_keys = defaultdict(dict)  # type: Dict[str, Dict[str, int]]
-
-        for verify_request in remaining_requests:
-            # any completed requests should already have been removed
-            assert not verify_request.key_ready.called
-            keys_for_server = missing_keys[verify_request.server_name]
-
-            for key_id in verify_request.key_ids:
-                # If we have several requests for the same key, then we only need to
-                # request that key once, but we should do so with the greatest
-                # min_valid_until_ts of the requests, so that we can satisfy all of
-                # the requests.
-                keys_for_server[key_id] = max(
-                    keys_for_server.get(key_id, -1),
-                    verify_request.minimum_valid_until_ts,
-                )
+            logger.debug("Getting keys from %s for %s", fetcher, verify_request)
+            keys = await fetcher.get_keys(
+                verify_request.server_name,
+                list(missing_key_ids),
+                verify_request.minimum_valid_until_ts,
+            )
 
-        results = await fetcher.get_keys(missing_keys)
+            for key_id, key in keys.items():
+                if not key:
+                    continue
 
-        completed = []
-        for verify_request in remaining_requests:
-            server_name = verify_request.server_name
+                # If we already have a result for the given key ID we keep the
+                # one with the highest `valid_until_ts`.
+                existing_key = found_keys.get(key_id)
+                if existing_key:
+                    if key.valid_until_ts <= existing_key.valid_until_ts:
+                        continue
 
-            # see if any of the keys we got this time are sufficient to
-            # complete this VerifyJsonRequest.
-            result_keys = results.get(server_name, {})
-            for key_id in verify_request.key_ids:
-                fetch_key_result = result_keys.get(key_id)
-                if not fetch_key_result:
-                    # we didn't get a result for this key
-                    continue
+                # We always store the returned key even if it doesn't the
+                # `minimum_valid_until_ts` requirement, as some verification
+                # requests may still be able to be satisfied by it.
+                #
+                # We still keep looking for the key from other fetchers in that
+                # case though.
+                found_keys[key_id] = key
 
-                if (
-                    fetch_key_result.valid_until_ts
-                    < verify_request.minimum_valid_until_ts
-                ):
-                    # key was not valid at this point
+                if key.valid_until_ts < verify_request.minimum_valid_until_ts:
                     continue
 
-                # we have a valid key for this request. If we run the callback
-                # immediately, it may cancel our loggingcontext while we are still in
-                # it, so instead we schedule it for the next time round the reactor.
-                #
-                # (this also ensures that we don't get a stack overflow if we had
-                # a massive queue of lookups waiting for this server).
-                logger.debug(
-                    "Found key %s:%s for %s",
-                    server_name,
-                    key_id,
-                    verify_request.request_name,
-                )
-                self.clock.call_later(
-                    0,
-                    verify_request.key_ready.callback,
-                    (server_name, key_id, fetch_key_result.verify_key),
-                )
-                completed.append(verify_request)
-                break
+                missing_key_ids.discard(key_id)
 
-        remaining_requests.difference_update(completed)
+        return found_keys
 
 
 class KeyFetcher(metaclass=abc.ABCMeta):
-    @abc.abstractmethod
+    def __init__(self, hs: "HomeServer"):
+        self._queue = BatchingQueue(
+            self.__class__.__name__, hs.get_clock(), self._fetch_keys
+        )
+
     async def get_keys(
-        self, keys_to_fetch: Dict[str, Dict[str, int]]
-    ) -> Dict[str, Dict[str, FetchKeyResult]]:
-        """
-        Args:
-            keys_to_fetch:
-                the keys to be fetched. server_name -> key_id -> min_valid_ts
+        self, server_name: str, key_ids: List[str], minimum_valid_until_ts: int
+    ) -> Dict[str, FetchKeyResult]:
+        results = await self._queue.add_to_queue(
+            _FetchKeyRequest(
+                server_name=server_name,
+                key_ids=key_ids,
+                minimum_valid_until_ts=minimum_valid_until_ts,
+            )
+        )
+        return results.get(server_name, {})
 
-        Returns:
-            Map from server_name -> key_id -> FetchKeyResult
-        """
-        raise NotImplementedError
+    @abc.abstractmethod
+    async def _fetch_keys(
+        self, keys_to_fetch: List[_FetchKeyRequest]
+    ) -> Dict[str, Dict[str, FetchKeyResult]]:
+        pass
 
 
 class StoreKeyFetcher(KeyFetcher):
     """KeyFetcher impl which fetches keys from our data store"""
 
     def __init__(self, hs: "HomeServer"):
-        self.store = hs.get_datastore()
+        super().__init__(hs)
 
-    async def get_keys(
-        self, keys_to_fetch: Dict[str, Dict[str, int]]
-    ) -> Dict[str, Dict[str, FetchKeyResult]]:
-        """see KeyFetcher.get_keys"""
+        self.store = hs.get_datastore()
 
+    async def _fetch_keys(self, keys_to_fetch: List[_FetchKeyRequest]):
         key_ids_to_fetch = (
-            (server_name, key_id)
-            for server_name, keys_for_server in keys_to_fetch.items()
-            for key_id in keys_for_server.keys()
+            (queue_value.server_name, key_id)
+            for queue_value in keys_to_fetch
+            for key_id in queue_value.key_ids
         )
 
         res = await self.store.get_server_verify_keys(key_ids_to_fetch)
-        keys = {}  # type: Dict[str, Dict[str, FetchKeyResult]]
+        keys: Dict[str, Dict[str, FetchKeyResult]] = {}
         for (server_name, key_id), key in res.items():
             keys.setdefault(server_name, {})[key_id] = key
         return keys
@@ -501,8 +465,10 @@ async def get_keys(
 
 class BaseV2KeyFetcher(KeyFetcher):
     def __init__(self, hs: "HomeServer"):
+        super().__init__(hs)
+
         self.store = hs.get_datastore()
-        self.config = hs.get_config()
+        self.config = hs.config
 
     async def process_v2_response(
         self, from_server: str, response_json: JsonDict, time_added_ms: int
@@ -608,10 +574,10 @@ def __init__(self, hs: "HomeServer"):
         self.client = hs.get_federation_http_client()
         self.key_servers = self.config.key_servers
 
-    async def get_keys(
-        self, keys_to_fetch: Dict[str, Dict[str, int]]
+    async def _fetch_keys(
+        self, keys_to_fetch: List[_FetchKeyRequest]
     ) -> Dict[str, Dict[str, FetchKeyResult]]:
-        """see KeyFetcher.get_keys"""
+        """see KeyFetcher._fetch_keys"""
 
         async def get_key(key_server: TrustedKeyServer) -> Dict:
             try:
@@ -639,7 +605,7 @@ async def get_key(key_server: TrustedKeyServer) -> Dict:
             ).addErrback(unwrapFirstError)
         )
 
-        union_of_keys = {}  # type: Dict[str, Dict[str, FetchKeyResult]]
+        union_of_keys: Dict[str, Dict[str, FetchKeyResult]] = {}
         for result in results:
             for server_name, keys in result.items():
                 union_of_keys.setdefault(server_name, {}).update(keys)
@@ -647,12 +613,12 @@ async def get_key(key_server: TrustedKeyServer) -> Dict:
         return union_of_keys
 
     async def get_server_verify_key_v2_indirect(
-        self, keys_to_fetch: Dict[str, Dict[str, int]], key_server: TrustedKeyServer
+        self, keys_to_fetch: List[_FetchKeyRequest], key_server: TrustedKeyServer
     ) -> Dict[str, Dict[str, FetchKeyResult]]:
         """
         Args:
             keys_to_fetch:
-                the keys to be fetched. server_name -> key_id -> min_valid_ts
+                the keys to be fetched.
 
             key_server: notary server to query for the keys
 
@@ -666,7 +632,7 @@ async def get_server_verify_key_v2_indirect(
         perspective_name = key_server.server_name
         logger.info(
             "Requesting keys %s from notary server %s",
-            keys_to_fetch.items(),
+            keys_to_fetch,
             perspective_name,
         )
 
@@ -676,11 +642,13 @@ async def get_server_verify_key_v2_indirect(
                 path="/_matrix/key/v2/query",
                 data={
                     "server_keys": {
-                        server_name: {
-                            key_id: {"minimum_valid_until_ts": min_valid_ts}
-                            for key_id, min_valid_ts in server_keys.items()
+                        queue_value.server_name: {
+                            key_id: {
+                                "minimum_valid_until_ts": queue_value.minimum_valid_until_ts,
+                            }
+                            for key_id in queue_value.key_ids
                         }
-                        for server_name, server_keys in keys_to_fetch.items()
+                        for queue_value in keys_to_fetch
                     }
                 },
             )
@@ -690,8 +658,8 @@ async def get_server_verify_key_v2_indirect(
         except HttpResponseException as e:
             raise KeyLookupError("Remote server returned an error: %s" % (e,))
 
-        keys = {}  # type: Dict[str, Dict[str, FetchKeyResult]]
-        added_keys = []  # type: List[Tuple[str, str, FetchKeyResult]]
+        keys: Dict[str, Dict[str, FetchKeyResult]] = {}
+        added_keys: List[Tuple[str, str, FetchKeyResult]] = []
 
         time_now_ms = self.clock.time_msec()
 
@@ -781,7 +749,20 @@ def __init__(self, hs: "HomeServer"):
         self.client = hs.get_federation_http_client()
 
     async def get_keys(
-        self, keys_to_fetch: Dict[str, Dict[str, int]]
+        self, server_name: str, key_ids: List[str], minimum_valid_until_ts: int
+    ) -> Dict[str, FetchKeyResult]:
+        results = await self._queue.add_to_queue(
+            _FetchKeyRequest(
+                server_name=server_name,
+                key_ids=key_ids,
+                minimum_valid_until_ts=minimum_valid_until_ts,
+            ),
+            key=server_name,
+        )
+        return results.get(server_name, {})
+
+    async def _fetch_keys(
+        self, keys_to_fetch: List[_FetchKeyRequest]
     ) -> Dict[str, Dict[str, FetchKeyResult]]:
         """
         Args:
@@ -794,8 +775,10 @@ async def get_keys(
 
         results = {}
 
-        async def get_key(key_to_fetch_item: Tuple[str, Dict[str, int]]) -> None:
-            server_name, key_ids = key_to_fetch_item
+        async def get_key(key_to_fetch_item: _FetchKeyRequest) -> None:
+            server_name = key_to_fetch_item.server_name
+            key_ids = key_to_fetch_item.key_ids
+
             try:
                 keys = await self.get_server_verify_key_v2_direct(server_name, key_ids)
                 results[server_name] = keys
@@ -806,7 +789,7 @@ async def get_key(key_to_fetch_item: Tuple[str, Dict[str, int]]) -> None:
             except Exception:
                 logger.exception("Error getting keys %s from %s", key_ids, server_name)
 
-        await yieldable_gather_results(get_key, keys_to_fetch.items())
+        await yieldable_gather_results(get_key, keys_to_fetch)
         return results
 
     async def get_server_verify_key_v2_direct(
@@ -824,7 +807,7 @@ async def get_server_verify_key_v2_direct(
         Raises:
             KeyLookupError if there was a problem making the lookup
         """
-        keys = {}  # type: Dict[str, FetchKeyResult]
+        keys: Dict[str, FetchKeyResult] = {}
 
         for requested_key_id in key_ids:
             # we may have found this key as a side-effect of asking for another.
@@ -878,37 +861,3 @@ async def get_server_verify_key_v2_direct(
             keys.update(response_keys)
 
         return keys
-
-
-async def _handle_key_deferred(verify_request: VerifyJsonRequest) -> None:
-    """Waits for the key to become available, and then performs a verification
-
-    Args:
-        verify_request:
-
-    Raises:
-        SynapseError if there was a problem performing the verification
-    """
-    server_name = verify_request.server_name
-    with PreserveLoggingContext():
-        _, key_id, verify_key = await verify_request.key_ready
-
-    json_object = verify_request.json_object
-
-    try:
-        verify_signed_json(json_object, server_name, verify_key)
-    except SignatureVerifyException as e:
-        logger.debug(
-            "Error verifying signature for %s:%s:%s with key %s: %s",
-            server_name,
-            verify_key.alg,
-            verify_key.version,
-            encode_verify_key_base64(verify_key),
-            str(e),
-        )
-        raise SynapseError(
-            401,
-            "Invalid signature for server %s with key %s:%s: %s"
-            % (server_name, verify_key.alg, verify_key.version, str(e)),
-            Codes.UNAUTHORIZED,
-        )
diff --git a/synapse/event_auth.py b/synapse/event_auth.py
index 4e20851d7f..c3a0c10499 100644
--- a/synapse/event_auth.py
+++ b/synapse/event_auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -15,14 +14,14 @@
 # limitations under the License.
 
 import logging
-from typing import List, Optional, Set, Tuple
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 from canonicaljson import encode_canonical_json
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import SignatureVerifyException, verify_signed_json
 from unpaddedbase64 import decode_base64
 
-from synapse.api.constants import EventTypes, JoinRules, Membership
+from synapse.api.constants import MAX_PDU_SIZE, EventTypes, JoinRules, Membership
 from synapse.api.errors import AuthError, EventSizeError, SynapseError
 from synapse.api.room_versions import (
     KNOWN_ROOM_VERSIONS,
@@ -30,6 +29,7 @@
     RoomVersion,
 )
 from synapse.events import EventBase
+from synapse.events.builder import EventBuilder
 from synapse.types import StateMap, UserID, get_domain_from_id
 
 logger = logging.getLogger(__name__)
@@ -48,6 +48,9 @@ def check(
         room_version_obj: the version of the room
         event: the event being checked.
         auth_events: the existing room state.
+        do_sig_check: True if it should be verified that the sending server
+            signed the event.
+        do_size_check: True if the size of the event fields should be verified.
 
     Raises:
         AuthError if the checks fail
@@ -103,6 +106,18 @@ def check(
             if not event.signatures.get(event_id_domain):
                 raise AuthError(403, "Event not signed by sending server")
 
+        is_invite_via_allow_rule = (
+            event.type == EventTypes.Member
+            and event.membership == Membership.JOIN
+            and "join_authorised_via_users_server" in event.content
+        )
+        if is_invite_via_allow_rule:
+            authoriser_domain = get_domain_from_id(
+                event.content["join_authorised_via_users_server"]
+            )
+            if not event.signatures.get(authoriser_domain):
+                raise AuthError(403, "Event not signed by authorising server")
+
     # Implementation of https://matrix.org/docs/spec/rooms/v1#authorization-rules
     #
     # 1. If type is m.room.create:
@@ -163,7 +178,7 @@ def check(
 
     # 5. If type is m.room.membership
     if event.type == EventTypes.Member:
-        _is_membership_change_allowed(event, auth_events)
+        _is_membership_change_allowed(room_version_obj, event, auth_events)
         logger.debug("Allowing! %s", event)
         return
 
@@ -174,7 +189,7 @@ def check(
     # https://github.com/vector-im/vector-web/issues/1208 hopefully
     if event.type == EventTypes.ThirdPartyInvite:
         user_level = get_user_power_level(event.user_id, auth_events)
-        invite_level = _get_named_level(auth_events, "invite", 0)
+        invite_level = get_named_level(auth_events, "invite", 0)
 
         if user_level < invite_level:
             raise AuthError(403, "You don't have permission to invite users")
@@ -190,6 +205,13 @@ def check(
     if event.type == EventTypes.Redaction:
         check_redaction(room_version_obj, event, auth_events)
 
+    if (
+        event.type == EventTypes.MSC2716_INSERTION
+        or event.type == EventTypes.MSC2716_CHUNK
+        or event.type == EventTypes.MSC2716_MARKER
+    ):
+        check_historical(room_version_obj, event, auth_events)
+
     logger.debug("Allowing! %s", event)
 
 
@@ -207,7 +229,7 @@ def too_big(field):
         too_big("type")
     if len(event.event_id) > 255:
         too_big("event_id")
-    if len(encode_canonical_json(event.get_pdu_json())) > 65536:
+    if len(encode_canonical_json(event.get_pdu_json())) > MAX_PDU_SIZE:
         too_big("event")
 
 
@@ -221,8 +243,19 @@ def _can_federate(event: EventBase, auth_events: StateMap[EventBase]) -> bool:
 
 
 def _is_membership_change_allowed(
-    event: EventBase, auth_events: StateMap[EventBase]
+    room_version: RoomVersion, event: EventBase, auth_events: StateMap[EventBase]
 ) -> None:
+    """
+    Confirms that the event which changes membership is an allowed change.
+
+    Args:
+        room_version: The version of the room.
+        event: The event to check.
+        auth_events: The current auth events of the room.
+
+    Raises:
+        AuthError if the event is not allowed.
+    """
     membership = event.content["membership"]
 
     # Check if this is the room creator joining:
@@ -248,7 +281,11 @@ def _is_membership_change_allowed(
 
     caller_in_room = caller and caller.membership == Membership.JOIN
     caller_invited = caller and caller.membership == Membership.INVITE
-    caller_knocked = caller and caller.membership == Membership.KNOCK
+    caller_knocked = (
+        caller
+        and room_version.msc2403_knocking
+        and caller.membership == Membership.KNOCK
+    )
 
     # get info about the target
     key = (EventTypes.Member, target_user_id)
@@ -267,14 +304,15 @@ def _is_membership_change_allowed(
     user_level = get_user_power_level(event.user_id, auth_events)
     target_level = get_user_power_level(target_user_id, auth_events)
 
-    # FIXME (erikj): What should we do here as the default?
-    ban_level = _get_named_level(auth_events, "ban", 50)
+    invite_level = get_named_level(auth_events, "invite", 0)
+    ban_level = get_named_level(auth_events, "ban", 50)
 
     logger.debug(
         "_is_membership_change_allowed: %s",
         {
             "caller_in_room": caller_in_room,
             "caller_invited": caller_invited,
+            "caller_knocked": caller_knocked,
             "target_banned": target_banned,
             "target_in_room": target_in_room,
             "membership": membership,
@@ -292,7 +330,9 @@ def _is_membership_change_allowed(
         return
 
     # Require the user to be in the room for membership changes other than join/knock.
-    if Membership.JOIN != membership and Membership.KNOCK != membership:
+    if Membership.JOIN != membership and (
+        RoomVersion.msc2403_knocking and Membership.KNOCK != membership
+    ):
         # If the user has been invited or has knocked, they are allowed to change their
         # membership event to leave
         if (
@@ -315,21 +355,48 @@ def _is_membership_change_allowed(
         elif target_in_room:  # the target is already in the room.
             raise AuthError(403, "%s is already in the room." % target_user_id)
         else:
-            invite_level = _get_named_level(auth_events, "invite", 0)
-
             if user_level < invite_level:
                 raise AuthError(403, "You don't have permission to invite users")
     elif Membership.JOIN == membership:
-        # Joins are valid iff caller == target and they were:
-        # invited: They are accepting the invitation
-        # joined: It's a NOOP
+        # Joins are valid iff caller == target and:
+        # * They are not banned.
+        # * They are accepting a previously sent invitation.
+        # * They are already joined (it's a NOOP).
+        # * The room is public.
+        # * The room is restricted and the user meets the allows rules.
         if event.user_id != target_user_id:
             raise AuthError(403, "Cannot force another user to join.")
         elif target_banned:
             raise AuthError(403, "You are banned from this room")
         elif join_rule == JoinRules.PUBLIC:
             pass
-        elif join_rule in (JoinRules.INVITE, JoinRules.KNOCK):
+        elif room_version.msc3083_join_rules and join_rule == JoinRules.RESTRICTED:
+            # This is the same as public, but the event must contain a reference
+            # to the server who authorised the join. If the event does not contain
+            # the proper content it is rejected.
+            #
+            # Note that if the caller is in the room or invited, then they do
+            # not need to meet the allow rules.
+            if not caller_in_room and not caller_invited:
+                authorising_user = event.content.get("join_authorised_via_users_server")
+
+                if authorising_user is None:
+                    raise AuthError(403, "Join event is missing authorising user.")
+
+                # The authorising user must be in the room.
+                key = (EventTypes.Member, authorising_user)
+                member_event = auth_events.get(key)
+                _check_joined_room(member_event, authorising_user, event.room_id)
+
+                authorising_user_level = get_user_power_level(
+                    authorising_user, auth_events
+                )
+                if authorising_user_level < invite_level:
+                    raise AuthError(403, "Join event authorised by invalid server.")
+
+        elif join_rule == JoinRules.INVITE or (
+            room_version.msc2403_knocking and join_rule == JoinRules.KNOCK
+        ):
             if not caller_in_room and not caller_invited:
                 raise AuthError(403, "You are not invited to this room.")
         else:
@@ -341,14 +408,14 @@ def _is_membership_change_allowed(
         if target_banned and user_level < ban_level:
             raise AuthError(403, "You cannot unban user %s." % (target_user_id,))
         elif target_user_id != event.user_id:
-            kick_level = _get_named_level(auth_events, "kick", 50)
+            kick_level = get_named_level(auth_events, "kick", 50)
 
             if user_level < kick_level or user_level <= target_level:
                 raise AuthError(403, "You cannot kick user %s." % target_user_id)
     elif Membership.BAN == membership:
         if user_level < ban_level or user_level <= target_level:
             raise AuthError(403, "You don't have permission to ban")
-    elif Membership.KNOCK == membership:
+    elif room_version.msc2403_knocking and Membership.KNOCK == membership:
         if join_rule != JoinRules.KNOCK:
             raise AuthError(403, "You don't have permission to knock")
         elif target_user_id != event.user_id:
@@ -417,7 +484,7 @@ def get_send_level(
 
 
 def _can_send_event(event: EventBase, auth_events: StateMap[EventBase]) -> bool:
-    power_levels_event = _get_power_level_event(auth_events)
+    power_levels_event = get_power_level_event(auth_events)
 
     send_level = get_send_level(event.type, event.get("state_key"), power_levels_event)
     user_level = get_user_power_level(event.user_id, auth_events)
@@ -457,7 +524,7 @@ def check_redaction(
     """
     user_level = get_user_power_level(event.user_id, auth_events)
 
-    redact_level = _get_named_level(auth_events, "redact", 50)
+    redact_level = get_named_level(auth_events, "redact", 50)
 
     if user_level >= redact_level:
         return False
@@ -476,6 +543,37 @@ def check_redaction(
     raise AuthError(403, "You don't have permission to redact events")
 
 
+def check_historical(
+    room_version_obj: RoomVersion,
+    event: EventBase,
+    auth_events: StateMap[EventBase],
+) -> None:
+    """Check whether the event sender is allowed to send historical related
+    events like "insertion", "chunk", and "marker".
+
+    Returns:
+        None
+
+    Raises:
+        AuthError if the event sender is not allowed to send historical related events
+        ("insertion", "chunk", and "marker").
+    """
+    # Ignore the auth checks in room versions that do not support historical
+    # events
+    if not room_version_obj.msc2716_historical:
+        return
+
+    user_level = get_user_power_level(event.user_id, auth_events)
+
+    historical_level = get_named_level(auth_events, "historical", 100)
+
+    if user_level < historical_level:
+        raise AuthError(
+            403,
+            'You don\'t have permission to send send historical related events ("insertion", "chunk", and "marker")',
+        )
+
+
 def _check_power_levels(
     room_version_obj: RoomVersion,
     event: EventBase,
@@ -503,7 +601,7 @@ def _check_power_levels(
     user_level = get_user_power_level(event.user_id, auth_events)
 
     # Check other levels:
-    levels_to_check = [
+    levels_to_check: List[Tuple[str, Optional[str]]] = [
         ("users_default", None),
         ("events_default", None),
         ("state_default", None),
@@ -511,7 +609,7 @@ def _check_power_levels(
         ("redact", None),
         ("kick", None),
         ("invite", None),
-    ]  # type: List[Tuple[str, Optional[str]]]
+    ]
 
     old_list = current_state.content.get("users", {})
     for user in set(list(old_list) + list(user_list)):
@@ -541,12 +639,12 @@ def _check_power_levels(
             new_loc = new_loc.get(dir, {})
 
         if level_to_check in old_loc:
-            old_level = int(old_loc[level_to_check])  # type: Optional[int]
+            old_level: Optional[int] = int(old_loc[level_to_check])
         else:
             old_level = None
 
         if level_to_check in new_loc:
-            new_level = int(new_loc[level_to_check])  # type: Optional[int]
+            new_level: Optional[int] = int(new_loc[level_to_check])
         else:
             new_level = None
 
@@ -572,7 +670,7 @@ def _check_power_levels(
             )
 
 
-def _get_power_level_event(auth_events: StateMap[EventBase]) -> Optional[EventBase]:
+def get_power_level_event(auth_events: StateMap[EventBase]) -> Optional[EventBase]:
     return auth_events.get((EventTypes.PowerLevels, ""))
 
 
@@ -588,10 +686,10 @@ def get_user_power_level(user_id: str, auth_events: StateMap[EventBase]) -> int:
     Returns:
         the user's power level in this room.
     """
-    power_level_event = _get_power_level_event(auth_events)
+    power_level_event = get_power_level_event(auth_events)
     if power_level_event:
         level = power_level_event.content.get("users", {}).get(user_id)
-        if not level:
+        if level is None:
             level = power_level_event.content.get("users_default", 0)
 
         if level is None:
@@ -612,8 +710,8 @@ def get_user_power_level(user_id: str, auth_events: StateMap[EventBase]) -> int:
             return 0
 
 
-def _get_named_level(auth_events: StateMap[EventBase], name: str, default: int) -> int:
-    power_level_event = _get_power_level_event(auth_events)
+def get_named_level(auth_events: StateMap[EventBase], name: str, default: int) -> int:
+    power_level_event = get_power_level_event(auth_events)
 
     if not power_level_event:
         return default
@@ -671,7 +769,7 @@ def _verify_third_party_invite(event: EventBase, auth_events: StateMap[EventBase
         public_key = public_key_object["public_key"]
         try:
             for server, signature_block in signed["signatures"].items():
-                for key_name, encoded_signature in signature_block.items():
+                for key_name in signature_block.keys():
                     if not key_name.startswith("ed25519:"):
                         continue
                     verify_key = decode_verify_key_bytes(
@@ -689,7 +787,7 @@ def _verify_third_party_invite(event: EventBase, auth_events: StateMap[EventBase
     return False
 
 
-def get_public_keys(invite_event):
+def get_public_keys(invite_event: EventBase) -> List[Dict[str, Any]]:
     public_keys = []
     if "public_key" in invite_event.content:
         o = {"public_key": invite_event.content["public_key"]}
@@ -700,7 +798,9 @@ def get_public_keys(invite_event):
     return public_keys
 
 
-def auth_types_for_event(event: EventBase) -> Set[Tuple[str, str]]:
+def auth_types_for_event(
+    room_version: RoomVersion, event: Union[EventBase, EventBuilder]
+) -> Set[Tuple[str, str]]:
     """Given an event, return a list of (EventType, StateKey) that may be
     needed to auth the event. The returned list may be a superset of what
     would actually be required depending on the full state of the room.
@@ -732,4 +832,12 @@ def auth_types_for_event(event: EventBase) -> Set[Tuple[str, str]]:
                 )
                 auth_types.add(key)
 
+        if room_version.msc3083_join_rules and membership == Membership.JOIN:
+            if "join_authorised_via_users_server" in event.content:
+                key = (
+                    EventTypes.Member,
+                    event.content["join_authorised_via_users_server"],
+                )
+                auth_types.add(key)
+
     return auth_types
diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 8f6b955d17..a730c1719a 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
@@ -106,27 +105,28 @@ def __init__(self, internal_metadata_dict: JsonDict):
         self._dict = dict(internal_metadata_dict)
 
         # the stream ordering of this event. None, until it has been persisted.
-        self.stream_ordering = None  # type: Optional[int]
+        self.stream_ordering: Optional[int] = None
 
         # whether this event is an outlier (ie, whether we have the state at that point
         # in the DAG)
         self.outlier = False
 
-    out_of_band_membership = DictProperty("out_of_band_membership")  # type: bool
-    send_on_behalf_of = DictProperty("send_on_behalf_of")  # type: str
-    recheck_redaction = DictProperty("recheck_redaction")  # type: bool
-    soft_failed = DictProperty("soft_failed")  # type: bool
-    proactively_send = DictProperty("proactively_send")  # type: bool
-    redacted = DictProperty("redacted")  # type: bool
-    txn_id = DictProperty("txn_id")  # type: str
-    token_id = DictProperty("token_id")  # type: str
+    out_of_band_membership: bool = DictProperty("out_of_band_membership")
+    send_on_behalf_of: str = DictProperty("send_on_behalf_of")
+    recheck_redaction: bool = DictProperty("recheck_redaction")
+    soft_failed: bool = DictProperty("soft_failed")
+    proactively_send: bool = DictProperty("proactively_send")
+    redacted: bool = DictProperty("redacted")
+    txn_id: str = DictProperty("txn_id")
+    token_id: int = DictProperty("token_id")
+    historical: bool = DictProperty("historical")
 
     # XXX: These are set by StreamWorkerStore._set_before_and_after.
     # I'm pretty sure that these are never persisted to the database, so shouldn't
     # be here
-    before = DictProperty("before")  # type: RoomStreamToken
-    after = DictProperty("after")  # type: RoomStreamToken
-    order = DictProperty("order")  # type: Tuple[int, int]
+    before: RoomStreamToken = DictProperty("before")
+    after: RoomStreamToken = DictProperty("after")
+    order: Tuple[int, int] = DictProperty("order")
 
     def get_dict(self) -> JsonDict:
         return dict(self._dict)
@@ -205,6 +205,14 @@ def is_redacted(self):
         """
         return self._dict.get("redacted", False)
 
+    def is_historical(self) -> bool:
+        """Whether this is a historical message.
+        This is used by the batchsend historical message endpoint and
+        is needed to and mark the event as backfilled and skip some checks
+        like push notifications.
+        """
+        return self._dict.get("historical", False)
+
 
 class EventBase(metaclass=abc.ABCMeta):
     @property
@@ -283,6 +291,20 @@ def get_pdu_json(self, time_now=None) -> JsonDict:
 
         return pdu_json
 
+    def get_templated_pdu_json(self) -> JsonDict:
+        """
+        Return a JSON object suitable for a templated event, as used in the
+        make_{join,leave,knock} workflow.
+        """
+        # By using _dict directly we don't pull in signatures/unsigned.
+        template_json = dict(self._dict)
+        # The hashes (similar to the signature) need to be recalculated by the
+        # joining/leaving/knocking server after (potentially) modifying the
+        # event.
+        template_json.pop("hashes")
+
+        return template_json
+
     def __set__(self, instance, value):
         raise AttributeError("Unrecognized attribute %s" % (instance,))
 
@@ -330,9 +352,11 @@ def __init__(
         self,
         event_dict: JsonDict,
         room_version: RoomVersion,
-        internal_metadata_dict: JsonDict = {},
+        internal_metadata_dict: Optional[JsonDict] = None,
         rejected_reason: Optional[str] = None,
     ):
+        internal_metadata_dict = internal_metadata_dict or {}
+
         event_dict = dict(event_dict)
 
         # Signatures is a dict of dicts, and this is faster than doing a
@@ -372,10 +396,11 @@ def __str__(self):
         return self.__repr__()
 
     def __repr__(self):
-        return "<FrozenEvent event_id=%r, type=%r, state_key=%r>" % (
+        return "<FrozenEvent event_id=%r, type=%r, state_key=%r, outlier=%s>" % (
             self.get("event_id", None),
             self.get("type", None),
             self.get("state_key", None),
+            self.internal_metadata.is_outlier(),
         )
 
 
@@ -386,9 +411,11 @@ def __init__(
         self,
         event_dict: JsonDict,
         room_version: RoomVersion,
-        internal_metadata_dict: JsonDict = {},
+        internal_metadata_dict: Optional[JsonDict] = None,
         rejected_reason: Optional[str] = None,
     ):
+        internal_metadata_dict = internal_metadata_dict or {}
+
         event_dict = dict(event_dict)
 
         # Signatures is a dict of dicts, and this is faster than doing a
@@ -507,9 +534,11 @@ def _event_type_from_format_version(format_version: int) -> Type[EventBase]:
 def make_event_from_dict(
     event_dict: JsonDict,
     room_version: RoomVersion = RoomVersions.V1,
-    internal_metadata_dict: JsonDict = {},
+    internal_metadata_dict: Optional[JsonDict] = None,
     rejected_reason: Optional[str] = None,
 ) -> EventBase:
     """Construct an EventBase from the given event dict"""
     event_type = _event_type_from_format_version(room_version.event_format)
-    return event_type(event_dict, room_version, internal_metadata_dict, rejected_reason)
+    return event_type(
+        event_dict, room_version, internal_metadata_dict or {}, rejected_reason
+    )
diff --git a/synapse/events/builder.py b/synapse/events/builder.py
index c1c0426f6e..87e2bb123b 100644
--- a/synapse/events/builder.py
+++ b/synapse/events/builder.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,12 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, Dict, List, Optional, Tuple, Union
+import logging
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
 import attr
 from nacl.signing import SigningKey
 
-from synapse.api.auth import Auth
 from synapse.api.constants import MAX_DEPTH
 from synapse.api.errors import UnsupportedRoomVersionError
 from synapse.api.room_versions import (
@@ -34,8 +33,14 @@
 from synapse.util import Clock
 from synapse.util.stringutils import random_string
 
+if TYPE_CHECKING:
+    from synapse.handlers.event_auth import EventAuthHandler
+    from synapse.server import HomeServer
 
-@attr.s(slots=True, cmp=False, frozen=True)
+logger = logging.getLogger(__name__)
+
+
+@attr.s(slots=True, cmp=False, frozen=True, auto_attribs=True)
 class EventBuilder:
     """A format independent event builder used to build up the event content
     before signing the event.
@@ -60,31 +65,30 @@ class EventBuilder:
         _signing_key: The signing key to use to sign the event as the server
     """
 
-    _state = attr.ib(type=StateHandler)
-    _auth = attr.ib(type=Auth)
-    _store = attr.ib(type=DataStore)
-    _clock = attr.ib(type=Clock)
-    _hostname = attr.ib(type=str)
-    _signing_key = attr.ib(type=SigningKey)
+    _state: StateHandler
+    _event_auth_handler: "EventAuthHandler"
+    _store: DataStore
+    _clock: Clock
+    _hostname: str
+    _signing_key: SigningKey
 
-    room_version = attr.ib(type=RoomVersion)
+    room_version: RoomVersion
 
-    room_id = attr.ib(type=str)
-    type = attr.ib(type=str)
-    sender = attr.ib(type=str)
+    room_id: str
+    type: str
+    sender: str
 
-    content = attr.ib(default=attr.Factory(dict), type=JsonDict)
-    unsigned = attr.ib(default=attr.Factory(dict), type=JsonDict)
+    content: JsonDict = attr.Factory(dict)
+    unsigned: JsonDict = attr.Factory(dict)
 
     # These only exist on a subset of events, so they raise AttributeError if
     # someone tries to get them when they don't exist.
-    _state_key = attr.ib(default=None, type=Optional[str])
-    _redacts = attr.ib(default=None, type=Optional[str])
-    _origin_server_ts = attr.ib(default=None, type=Optional[int])
+    _state_key: Optional[str] = None
+    _redacts: Optional[str] = None
+    _origin_server_ts: Optional[int] = None
 
-    internal_metadata = attr.ib(
-        default=attr.Factory(lambda: _EventInternalMetadata({})),
-        type=_EventInternalMetadata,
+    internal_metadata: _EventInternalMetadata = attr.Factory(
+        lambda: _EventInternalMetadata({})
     )
 
     @property
@@ -101,6 +105,7 @@ async def build(
         self,
         prev_event_ids: List[str],
         auth_event_ids: Optional[List[str]],
+        depth: Optional[int] = None,
     ) -> EventBase:
         """Transform into a fully signed and hashed event
 
@@ -109,6 +114,9 @@ async def build(
             auth_event_ids: The event IDs to use as the auth events.
                 Should normally be set to None, which will cause them to be calculated
                 based on the room state at the prev_events.
+            depth: Override the depth used to order the event in the DAG.
+                Should normally be set to None, which will cause the depth to be calculated
+                based on the prev_events.
 
         Returns:
             The signed and hashed event.
@@ -117,30 +125,38 @@ async def build(
             state_ids = await self._state.get_current_state_ids(
                 self.room_id, prev_event_ids
             )
-            auth_event_ids = self._auth.compute_auth_events(self, state_ids)
+            auth_event_ids = self._event_auth_handler.compute_auth_events(
+                self, state_ids
+            )
 
         format_version = self.room_version.event_format
         if format_version == EventFormatVersions.V1:
             # The types of auth/prev events changes between event versions.
-            auth_events = await self._store.add_event_hashes(
-                auth_event_ids
-            )  # type: Union[List[str], List[Tuple[str, Dict[str, str]]]]
-            prev_events = await self._store.add_event_hashes(
-                prev_event_ids
-            )  # type: Union[List[str], List[Tuple[str, Dict[str, str]]]]
+            auth_events: Union[
+                List[str], List[Tuple[str, Dict[str, str]]]
+            ] = await self._store.add_event_hashes(auth_event_ids)
+            prev_events: Union[
+                List[str], List[Tuple[str, Dict[str, str]]]
+            ] = await self._store.add_event_hashes(prev_event_ids)
         else:
             auth_events = auth_event_ids
             prev_events = prev_event_ids
 
-        old_depth = await self._store.get_max_depth_of(prev_event_ids)
-        depth = old_depth + 1
+        # Otherwise, progress the depth as normal
+        if depth is None:
+            (
+                _,
+                most_recent_prev_event_depth,
+            ) = await self._store.get_max_depth_of(prev_event_ids)
+
+            depth = most_recent_prev_event_depth + 1
 
         # we cap depth of generated events, to ensure that they are not
         # rejected by other servers (and so that they can be persisted in
         # the db)
         depth = min(depth, MAX_DEPTH)
 
-        event_dict = {
+        event_dict: Dict[str, Any] = {
             "auth_events": auth_events,
             "prev_events": prev_events,
             "type": self.type,
@@ -150,7 +166,7 @@ async def build(
             "unsigned": self.unsigned,
             "depth": depth,
             "prev_state": [],
-        }  # type: Dict[str, Any]
+        }
 
         if self.is_state():
             event_dict["state_key"] = self._state_key
@@ -172,24 +188,23 @@ async def build(
 
 
 class EventBuilderFactory:
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.clock = hs.get_clock()
         self.hostname = hs.hostname
         self.signing_key = hs.signing_key
 
         self.store = hs.get_datastore()
         self.state = hs.get_state_handler()
-        self.auth = hs.get_auth()
+        self._event_auth_handler = hs.get_event_auth_handler()
 
-    def new(self, room_version, key_values):
+    def new(self, room_version: str, key_values: dict) -> EventBuilder:
         """Generate an event builder appropriate for the given room version
 
         Deprecated: use for_room_version with a RoomVersion object instead
 
         Args:
-            room_version (str): Version of the room that we're creating an event builder
-                for
-            key_values (dict): Fields used as the basis of the new event
+            room_version: Version of the room that we're creating an event builder for
+            key_values: Fields used as the basis of the new event
 
         Returns:
             EventBuilder
@@ -200,13 +215,15 @@ def new(self, room_version, key_values):
             raise UnsupportedRoomVersionError()
         return self.for_room_version(v, key_values)
 
-    def for_room_version(self, room_version, key_values):
+    def for_room_version(
+        self, room_version: RoomVersion, key_values: dict
+    ) -> EventBuilder:
         """Generate an event builder appropriate for the given room version
 
         Args:
-            room_version (synapse.api.room_versions.RoomVersion):
+            room_version:
                 Version of the room that we're creating an event builder for
-            key_values (dict): Fields used as the basis of the new event
+            key_values: Fields used as the basis of the new event
 
         Returns:
             EventBuilder
@@ -214,7 +231,7 @@ def for_room_version(self, room_version, key_values):
         return EventBuilder(
             store=self.store,
             state=self.state,
-            auth=self.auth,
+            event_auth_handler=self._event_auth_handler,
             clock=self.clock,
             hostname=self.hostname,
             signing_key=self.signing_key,
@@ -274,15 +291,15 @@ def create_local_event_from_event_dict(
 _event_id_counter = 0
 
 
-def _create_event_id(clock, hostname):
+def _create_event_id(clock: Clock, hostname: str) -> str:
     """Create a new event ID
 
     Args:
-        clock (Clock)
-        hostname (str): The server name for the event ID
+        clock
+        hostname: The server name for the event ID
 
     Returns:
-        str
+        The new event ID
     """
 
     global _event_id_counter
diff --git a/synapse/events/presence_router.py b/synapse/events/presence_router.py
new file mode 100644
index 0000000000..6c37c8a7a4
--- /dev/null
+++ b/synapse/events/presence_router.py
@@ -0,0 +1,103 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Dict, Iterable, Set, Union
+
+from synapse.api.presence import UserPresenceState
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+
+class PresenceRouter:
+    """
+    A module that the homeserver will call upon to help route user presence updates to
+    additional destinations. If a custom presence router is configured, calls will be
+    passed to that instead.
+    """
+
+    ALL_USERS = "ALL"
+
+    def __init__(self, hs: "HomeServer"):
+        self.custom_presence_router = None
+
+        # Check whether a custom presence router module has been configured
+        if hs.config.presence_router_module_class:
+            # Initialise the module
+            self.custom_presence_router = hs.config.presence_router_module_class(
+                config=hs.config.presence_router_config, module_api=hs.get_module_api()
+            )
+
+            # Ensure the module has implemented the required methods
+            required_methods = ["get_users_for_states", "get_interested_users"]
+            for method_name in required_methods:
+                if not hasattr(self.custom_presence_router, method_name):
+                    raise Exception(
+                        "PresenceRouter module '%s' must implement all required methods: %s"
+                        % (
+                            hs.config.presence_router_module_class.__name__,
+                            ", ".join(required_methods),
+                        )
+                    )
+
+    async def get_users_for_states(
+        self,
+        state_updates: Iterable[UserPresenceState],
+    ) -> Dict[str, Set[UserPresenceState]]:
+        """
+        Given an iterable of user presence updates, determine where each one
+        needs to go.
+
+        Args:
+            state_updates: An iterable of user presence state updates.
+
+        Returns:
+          A dictionary of user_id -> set of UserPresenceState, indicating which
+          presence updates each user should receive.
+        """
+        if self.custom_presence_router is not None:
+            # Ask the custom module
+            return await self.custom_presence_router.get_users_for_states(
+                state_updates=state_updates
+            )
+
+        # Don't include any extra destinations for presence updates
+        return {}
+
+    async def get_interested_users(self, user_id: str) -> Union[Set[str], ALL_USERS]:
+        """
+        Retrieve a list of users that `user_id` is interested in receiving the
+        presence of. This will be in addition to those they share a room with.
+        Optionally, the object PresenceRouter.ALL_USERS can be returned to indicate
+        that this user should receive all incoming local and remote presence updates.
+
+        Note that this method will only be called for local users, but can return users
+        that are local or remote.
+
+        Args:
+            user_id: A user requesting presence updates.
+
+        Returns:
+            A set of user IDs to return presence updates for, or ALL_USERS to return all
+            known updates.
+        """
+        if self.custom_presence_router is not None:
+            # Ask the custom module for interested users
+            return await self.custom_presence_router.get_interested_users(
+                user_id=user_id
+            )
+
+        # A custom presence router is not defined.
+        # Don't report any additional interested users
+        return set()
diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index 7295df74fe..f8d898c3b1 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py
index b5a9c71ee6..67d66b5a1f 100644
--- a/synapse/events/spamcheck.py
+++ b/synapse/events/spamcheck.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
@@ -16,12 +15,23 @@
 
 import inspect
 import logging
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    Collection,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from synapse.rest.media.v1._base import FileInfo
 from synapse.rest.media.v1.media_storage import ReadableFileWrapper
 from synapse.spam_checker_api import RegistrationBehaviour
-from synapse.types import Collection
+from synapse.types import RoomAlias
 from synapse.util.async_helpers import maybe_awaitable
 
 if TYPE_CHECKING:
@@ -30,20 +40,201 @@
 
 logger = logging.getLogger(__name__)
 
+CHECK_EVENT_FOR_SPAM_CALLBACK = Callable[
+    ["synapse.events.EventBase"],
+    Awaitable[Union[bool, str]],
+]
+# FIXME: Callback signature differs from mainline
+USER_MAY_INVITE_CALLBACK = Callable[
+    [str, Optional[str], Optional[dict], str, bool, bool], Awaitable[bool]
+]
+# FIXME: Callback signature differs from mainline
+USER_MAY_CREATE_ROOM_CALLBACK = Callable[
+    [str, List[str], List[dict], bool], Awaitable[bool]
+]
+USER_MAY_CREATE_ROOM_ALIAS_CALLBACK = Callable[[str, RoomAlias], Awaitable[bool]]
+USER_MAY_PUBLISH_ROOM_CALLBACK = Callable[[str, str], Awaitable[bool]]
+CHECK_USERNAME_FOR_SPAM_CALLBACK = Callable[[Dict[str, str]], Awaitable[bool]]
+LEGACY_CHECK_REGISTRATION_FOR_SPAM_CALLBACK = Callable[
+    [
+        Optional[dict],
+        Optional[str],
+        Collection[Tuple[str, str]],
+    ],
+    Awaitable[RegistrationBehaviour],
+]
+CHECK_REGISTRATION_FOR_SPAM_CALLBACK = Callable[
+    [
+        Optional[dict],
+        Optional[str],
+        Collection[Tuple[str, str]],
+        Optional[str],
+    ],
+    Awaitable[RegistrationBehaviour],
+]
+CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK = Callable[
+    [ReadableFileWrapper, FileInfo],
+    Awaitable[bool],
+]
+# FIXME: This callback only exists on the DINUM fork and not in mainline.
+USER_MAY_JOIN_ROOM_CALLBACK = Callable[[str, str, bool], Awaitable[bool]]
+
+
+def load_legacy_spam_checkers(hs: "synapse.server.HomeServer"):
+    """Wrapper that loads spam checkers configured using the old configuration, and
+    registers the spam checker hooks they implement.
+    """
+    spam_checkers: List[Any] = []
+    api = hs.get_module_api()
+    for module, config in hs.config.spam_checkers:
+        # Older spam checkers don't accept the `api` argument, so we
+        # try and detect support.
+        spam_args = inspect.getfullargspec(module)
+        if "api" in spam_args.args:
+            spam_checkers.append(module(config=config, api=api))
+        else:
+            spam_checkers.append(module(config=config))
+
+    # The known spam checker hooks. If a spam checker module implements a method
+    # which name appears in this set, we'll want to register it.
+    spam_checker_methods = {
+        "check_event_for_spam",
+        "user_may_invite",
+        "user_may_create_room",
+        "user_may_create_room_alias",
+        "user_may_publish_room",
+        "check_username_for_spam",
+        "check_registration_for_spam",
+        "check_media_file_for_spam",
+        "user_may_join_room",
+    }
+
+    for spam_checker in spam_checkers:
+        # Methods on legacy spam checkers might not be async, so we wrap them around a
+        # wrapper that will call maybe_awaitable on the result.
+        def async_wrapper(f: Optional[Callable]) -> Optional[Callable[..., Awaitable]]:
+            # f might be None if the callback isn't implemented by the module. In this
+            # case we don't want to register a callback at all so we return None.
+            if f is None:
+                return None
+
+            wrapped_func = f
+
+            if f.__name__ == "check_registration_for_spam":
+                checker_args = inspect.signature(f)
+                if len(checker_args.parameters) == 3:
+                    # Backwards compatibility; some modules might implement a hook that
+                    # doesn't expect a 4th argument. In this case, wrap it in a function
+                    # that gives it only 3 arguments and drops the auth_provider_id on
+                    # the floor.
+                    def wrapper(
+                        email_threepid: Optional[dict],
+                        username: Optional[str],
+                        request_info: Collection[Tuple[str, str]],
+                        auth_provider_id: Optional[str],
+                    ) -> Union[Awaitable[RegistrationBehaviour], RegistrationBehaviour]:
+                        # We've already made sure f is not None above, but mypy doesn't
+                        # do well across function boundaries so we need to tell it f is
+                        # definitely not None.
+                        assert f is not None
+
+                        return f(
+                            email_threepid,
+                            username,
+                            request_info,
+                        )
+
+                    wrapped_func = wrapper
+                elif len(checker_args.parameters) != 4:
+                    raise RuntimeError(
+                        "Bad signature for callback check_registration_for_spam",
+                    )
+
+            def run(*args, **kwargs):
+                # mypy doesn't do well across function boundaries so we need to tell it
+                # wrapped_func is definitely not None.
+                assert wrapped_func is not None
+
+                return maybe_awaitable(wrapped_func(*args, **kwargs))
+
+            return run
+
+        # Register the hooks through the module API.
+        hooks = {
+            hook: async_wrapper(getattr(spam_checker, hook, None))
+            for hook in spam_checker_methods
+        }
+
+        api.register_spam_checker_callbacks(**hooks)
+
 
 class SpamChecker:
-    def __init__(self, hs: "synapse.server.HomeServer"):
-        self.spam_checkers = []  # type: List[Any]
-        api = hs.get_module_api()
-
-        for module, config in hs.config.spam_checkers:
-            # Older spam checkers don't accept the `api` argument, so we
-            # try and detect support.
-            spam_args = inspect.getfullargspec(module)
-            if "api" in spam_args.args:
-                self.spam_checkers.append(module(config=config, api=api))
-            else:
-                self.spam_checkers.append(module(config=config))
+    def __init__(self):
+        self._check_event_for_spam_callbacks: List[CHECK_EVENT_FOR_SPAM_CALLBACK] = []
+        self._user_may_invite_callbacks: List[USER_MAY_INVITE_CALLBACK] = []
+        self._user_may_create_room_callbacks: List[USER_MAY_CREATE_ROOM_CALLBACK] = []
+        self._user_may_create_room_alias_callbacks: List[
+            USER_MAY_CREATE_ROOM_ALIAS_CALLBACK
+        ] = []
+        self._user_may_publish_room_callbacks: List[USER_MAY_PUBLISH_ROOM_CALLBACK] = []
+        self._check_username_for_spam_callbacks: List[
+            CHECK_USERNAME_FOR_SPAM_CALLBACK
+        ] = []
+        self._check_registration_for_spam_callbacks: List[
+            CHECK_REGISTRATION_FOR_SPAM_CALLBACK
+        ] = []
+        self._check_media_file_for_spam_callbacks: List[
+            CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK
+        ] = []
+        self._user_may_join_room_callbacks: List[USER_MAY_JOIN_ROOM_CALLBACK] = []
+
+    def register_callbacks(
+        self,
+        check_event_for_spam: Optional[CHECK_EVENT_FOR_SPAM_CALLBACK] = None,
+        user_may_invite: Optional[USER_MAY_INVITE_CALLBACK] = None,
+        user_may_create_room: Optional[USER_MAY_CREATE_ROOM_CALLBACK] = None,
+        user_may_create_room_alias: Optional[
+            USER_MAY_CREATE_ROOM_ALIAS_CALLBACK
+        ] = None,
+        user_may_publish_room: Optional[USER_MAY_PUBLISH_ROOM_CALLBACK] = None,
+        check_username_for_spam: Optional[CHECK_USERNAME_FOR_SPAM_CALLBACK] = None,
+        check_registration_for_spam: Optional[
+            CHECK_REGISTRATION_FOR_SPAM_CALLBACK
+        ] = None,
+        check_media_file_for_spam: Optional[CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK] = None,
+        user_may_join_room: Optional[USER_MAY_JOIN_ROOM_CALLBACK] = None,
+    ):
+        """Register callbacks from module for each hook."""
+        if check_event_for_spam is not None:
+            self._check_event_for_spam_callbacks.append(check_event_for_spam)
+
+        if user_may_invite is not None:
+            self._user_may_invite_callbacks.append(user_may_invite)
+
+        if user_may_create_room is not None:
+            self._user_may_create_room_callbacks.append(user_may_create_room)
+
+        if user_may_create_room_alias is not None:
+            self._user_may_create_room_alias_callbacks.append(
+                user_may_create_room_alias,
+            )
+
+        if user_may_publish_room is not None:
+            self._user_may_publish_room_callbacks.append(user_may_publish_room)
+
+        if check_username_for_spam is not None:
+            self._check_username_for_spam_callbacks.append(check_username_for_spam)
+
+        if check_registration_for_spam is not None:
+            self._check_registration_for_spam_callbacks.append(
+                check_registration_for_spam,
+            )
+
+        if check_media_file_for_spam is not None:
+            self._check_media_file_for_spam_callbacks.append(check_media_file_for_spam)
+
+        if user_may_join_room is not None:
+            self._user_may_join_room_callbacks.append(user_may_join_room)
 
     async def check_event_for_spam(
         self, event: "synapse.events.EventBase"
@@ -61,9 +252,10 @@ async def check_event_for_spam(
             True or a string if the event is spammy. If a string is returned it
             will be used as the error message returned to the user.
         """
-        for spam_checker in self.spam_checkers:
-            if await maybe_awaitable(spam_checker.check_event_for_spam(event)):
-                return True
+        for callback in self._check_event_for_spam_callbacks:
+            res: Union[bool, str] = await callback(event)
+            if res:
+                return res
 
         return False
 
@@ -98,17 +290,15 @@ async def user_may_invite(
         Returns:
             True if the user may send an invite, otherwise False
         """
-        for spam_checker in self.spam_checkers:
+        for callback in self._user_may_invite_callbacks:
             if (
-                await maybe_awaitable(
-                    spam_checker.user_may_invite(
-                        inviter_userid,
-                        invitee_userid,
-                        third_party_invite,
-                        room_id,
-                        new_room,
-                        published_room,
-                    )
+                await callback(
+                    inviter_userid,
+                    invitee_userid,
+                    third_party_invite,
+                    room_id,
+                    new_room,
+                    published_room,
                 )
                 is False
             ):
@@ -139,20 +329,18 @@ async def user_may_create_room(
         Returns:
             True if the user may create a room, otherwise False
         """
-        for spam_checker in self.spam_checkers:
+        for callback in self._user_may_create_room_callbacks:
             if (
-                await maybe_awaitable(
-                    spam_checker.user_may_create_room(
-                        userid, invite_list, third_party_invite_list, cloning
-                    )
-                )
+                await callback(userid, invite_list, third_party_invite_list, cloning)
                 is False
             ):
                 return False
 
         return True
 
-    async def user_may_create_room_alias(self, userid: str, room_alias: str) -> bool:
+    async def user_may_create_room_alias(
+        self, userid: str, room_alias: RoomAlias
+    ) -> bool:
         """Checks if a given user may create a room alias
 
         If this method returns false, the association request will be rejected.
@@ -164,13 +352,8 @@ async def user_may_create_room_alias(self, userid: str, room_alias: str) -> bool
         Returns:
             True if the user may create a room alias, otherwise False
         """
-        for spam_checker in self.spam_checkers:
-            if (
-                await maybe_awaitable(
-                    spam_checker.user_may_create_room_alias(userid, room_alias)
-                )
-                is False
-            ):
+        for callback in self._user_may_create_room_alias_callbacks:
+            if await callback(userid, room_alias) is False:
                 return False
 
         return True
@@ -187,18 +370,13 @@ async def user_may_publish_room(self, userid: str, room_id: str) -> bool:
         Returns:
             True if the user may publish the room, otherwise False
         """
-        for spam_checker in self.spam_checkers:
-            if (
-                await maybe_awaitable(
-                    spam_checker.user_may_publish_room(userid, room_id)
-                )
-                is False
-            ):
+        for callback in self._user_may_publish_room_callbacks:
+            if await callback(userid, room_id) is False:
                 return False
 
         return True
 
-    def user_may_join_room(self, userid: str, room_id: str, is_invited: bool):
+    async def user_may_join_room(self, userid: str, room_id: str, is_invited: bool):
         """Checks if a given users is allowed to join a room.
 
         Not called when a user creates a room.
@@ -211,8 +389,8 @@ def user_may_join_room(self, userid: str, room_id: str, is_invited: bool):
         Returns:
             bool: Whether the user may join the room
         """
-        for spam_checker in self.spam_checkers:
-            if spam_checker.user_may_join_room(userid, room_id, is_invited) is False:
+        for callback in self._user_may_join_room_callbacks:
+            if await callback(userid, room_id, is_invited) is False:
                 return False
 
         return True
@@ -232,15 +410,11 @@ async def check_username_for_spam(self, user_profile: Dict[str, str]) -> bool:
         Returns:
             True if the user is spammy.
         """
-        for spam_checker in self.spam_checkers:
-            # For backwards compatibility, only run if the method exists on the
-            # spam checker
-            checker = getattr(spam_checker, "check_username_for_spam", None)
-            if checker:
-                # Make a copy of the user profile object to ensure the spam checker
-                # cannot modify it.
-                if await maybe_awaitable(checker(user_profile.copy())):
-                    return True
+        for callback in self._check_username_for_spam_callbacks:
+            # Make a copy of the user profile object to ensure the spam checker cannot
+            # modify it.
+            if await callback(user_profile.copy()):
+                return True
 
         return False
 
@@ -266,33 +440,13 @@ async def check_registration_for_spam(
             Enum for how the request should be handled
         """
 
-        for spam_checker in self.spam_checkers:
-            # For backwards compatibility, only run if the method exists on the
-            # spam checker
-            checker = getattr(spam_checker, "check_registration_for_spam", None)
-            if checker:
-                # Provide auth_provider_id if the function supports it
-                checker_args = inspect.signature(checker)
-                if len(checker_args.parameters) == 4:
-                    d = checker(
-                        email_threepid,
-                        username,
-                        request_info,
-                        auth_provider_id,
-                    )
-                elif len(checker_args.parameters) == 3:
-                    d = checker(email_threepid, username, request_info)
-                else:
-                    logger.error(
-                        "Invalid signature for %s.check_registration_for_spam. Denying registration",
-                        spam_checker.__module__,
-                    )
-                    return RegistrationBehaviour.DENY
-
-                behaviour = await maybe_awaitable(d)
-                assert isinstance(behaviour, RegistrationBehaviour)
-                if behaviour != RegistrationBehaviour.ALLOW:
-                    return behaviour
+        for callback in self._check_registration_for_spam_callbacks:
+            behaviour = await (
+                callback(email_threepid, username, request_info, auth_provider_id)
+            )
+            assert isinstance(behaviour, RegistrationBehaviour)
+            if behaviour != RegistrationBehaviour.ALLOW:
+                return behaviour
 
         return RegistrationBehaviour.ALLOW
 
@@ -330,13 +484,9 @@ async def check_media_file_for_spam(
             allowed.
         """
 
-        for spam_checker in self.spam_checkers:
-            # For backwards compatibility, only run if the method exists on the
-            # spam checker
-            checker = getattr(spam_checker, "check_media_file_for_spam", None)
-            if checker:
-                spam = await maybe_awaitable(checker(file_wrapper, file_info))
-                if spam:
-                    return True
+        for callback in self._check_media_file_for_spam_callbacks:
+            spam = await callback(file_wrapper, file_info)
+            if spam:
+                return True
 
         return False
diff --git a/synapse/events/third_party_rules.py b/synapse/events/third_party_rules.py
index 9767d23940..7a6eb3e516 100644
--- a/synapse/events/third_party_rules.py
+++ b/synapse/events/third_party_rules.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,16 +11,124 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
+from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional, Tuple
 
-from typing import TYPE_CHECKING, Union
-
+from synapse.api.errors import SynapseError
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
 from synapse.types import Requester, StateMap
+from synapse.util.async_helpers import maybe_awaitable
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
 
+logger = logging.getLogger(__name__)
+
+
+CHECK_EVENT_ALLOWED_CALLBACK = Callable[
+    [EventBase, StateMap[EventBase]], Awaitable[Tuple[bool, Optional[dict]]]
+]
+ON_CREATE_ROOM_CALLBACK = Callable[[Requester, dict, bool], Awaitable]
+CHECK_THREEPID_CAN_BE_INVITED_CALLBACK = Callable[
+    [str, str, StateMap[EventBase]], Awaitable[bool]
+]
+CHECK_VISIBILITY_CAN_BE_MODIFIED_CALLBACK = Callable[
+    [str, StateMap[EventBase], str], Awaitable[bool]
+]
+
+
+def load_legacy_third_party_event_rules(hs: "HomeServer"):
+    """Wrapper that loads a third party event rules module configured using the old
+    configuration, and registers the hooks they implement.
+    """
+    if hs.config.third_party_event_rules is None:
+        return
+
+    module, config = hs.config.third_party_event_rules
+
+    api = hs.get_module_api()
+    third_party_rules = module(config=config, module_api=api)
+
+    # The known hooks. If a module implements a method which name appears in this set,
+    # we'll want to register it.
+    third_party_event_rules_methods = {
+        "check_event_allowed",
+        "on_create_room",
+        "check_threepid_can_be_invited",
+        "check_visibility_can_be_modified",
+    }
+
+    def async_wrapper(f: Optional[Callable]) -> Optional[Callable[..., Awaitable]]:
+        # f might be None if the callback isn't implemented by the module. In this
+        # case we don't want to register a callback at all so we return None.
+        if f is None:
+            return None
+
+        # We return a separate wrapper for these methods because, in order to wrap them
+        # correctly, we need to await its result. Therefore it doesn't make a lot of
+        # sense to make it go through the run() wrapper.
+        if f.__name__ == "check_event_allowed":
+
+            # We need to wrap check_event_allowed because its old form would return either
+            # a boolean or a dict, but now we want to return the dict separately from the
+            # boolean.
+            async def wrap_check_event_allowed(
+                event: EventBase,
+                state_events: StateMap[EventBase],
+            ) -> Tuple[bool, Optional[dict]]:
+                # We've already made sure f is not None above, but mypy doesn't do well
+                # across function boundaries so we need to tell it f is definitely not
+                # None.
+                assert f is not None
+
+                res = await f(event, state_events)
+                if isinstance(res, dict):
+                    return True, res
+                else:
+                    return res, None
+
+            return wrap_check_event_allowed
+
+        if f.__name__ == "on_create_room":
+
+            # We need to wrap on_create_room because its old form would return a boolean
+            # if the room creation is denied, but now we just want it to raise an
+            # exception.
+            async def wrap_on_create_room(
+                requester: Requester, config: dict, is_requester_admin: bool
+            ) -> None:
+                # We've already made sure f is not None above, but mypy doesn't do well
+                # across function boundaries so we need to tell it f is definitely not
+                # None.
+                assert f is not None
+
+                res = await f(requester, config, is_requester_admin)
+                if res is False:
+                    raise SynapseError(
+                        403,
+                        "Room creation forbidden with these parameters",
+                    )
+
+            return wrap_on_create_room
+
+        def run(*args, **kwargs):
+            # mypy doesn't do well across function boundaries so we need to tell it
+            # f is definitely not None.
+            assert f is not None
+
+            return maybe_awaitable(f(*args, **kwargs))
+
+        return run
+
+    # Register the hooks through the module API.
+    hooks = {
+        hook: async_wrapper(getattr(third_party_rules, hook, None))
+        for hook in third_party_event_rules_methods
+    }
+
+    api.register_third_party_rules_callbacks(**hooks)
+
 
 class ThirdPartyEventRules:
     """Allows server admins to provide a Python module implementing an extra
@@ -36,36 +143,65 @@ def __init__(self, hs: "HomeServer"):
 
         self.store = hs.get_datastore()
 
-        module = None
-        config = None
-        if hs.config.third_party_event_rules:
-            module, config = hs.config.third_party_event_rules
+        self._check_event_allowed_callbacks: List[CHECK_EVENT_ALLOWED_CALLBACK] = []
+        self._on_create_room_callbacks: List[ON_CREATE_ROOM_CALLBACK] = []
+        self._check_threepid_can_be_invited_callbacks: List[
+            CHECK_THREEPID_CAN_BE_INVITED_CALLBACK
+        ] = []
+        self._check_visibility_can_be_modified_callbacks: List[
+            CHECK_VISIBILITY_CAN_BE_MODIFIED_CALLBACK
+        ] = []
+
+    def register_third_party_rules_callbacks(
+        self,
+        check_event_allowed: Optional[CHECK_EVENT_ALLOWED_CALLBACK] = None,
+        on_create_room: Optional[ON_CREATE_ROOM_CALLBACK] = None,
+        check_threepid_can_be_invited: Optional[
+            CHECK_THREEPID_CAN_BE_INVITED_CALLBACK
+        ] = None,
+        check_visibility_can_be_modified: Optional[
+            CHECK_VISIBILITY_CAN_BE_MODIFIED_CALLBACK
+        ] = None,
+    ):
+        """Register callbacks from modules for each hook."""
+        if check_event_allowed is not None:
+            self._check_event_allowed_callbacks.append(check_event_allowed)
+
+        if on_create_room is not None:
+            self._on_create_room_callbacks.append(on_create_room)
+
+        if check_threepid_can_be_invited is not None:
+            self._check_threepid_can_be_invited_callbacks.append(
+                check_threepid_can_be_invited,
+            )
 
-        if module is not None:
-            self.third_party_rules = module(
-                config=config,
-                module_api=hs.get_module_api(),
+        if check_visibility_can_be_modified is not None:
+            self._check_visibility_can_be_modified_callbacks.append(
+                check_visibility_can_be_modified,
             )
 
     async def check_event_allowed(
         self, event: EventBase, context: EventContext
-    ) -> Union[bool, dict]:
+    ) -> Tuple[bool, Optional[dict]]:
         """Check if a provided event should be allowed in the given context.
 
         The module can return:
             * True: the event is allowed.
             * False: the event is not allowed, and should be rejected with M_FORBIDDEN.
-            * a dict: replacement event data.
+
+        If the event is allowed, the module can also return a dictionary to use as a
+        replacement for the event.
 
         Args:
             event: The event to be checked.
             context: The context of the event.
 
         Returns:
-            The result from the ThirdPartyRules module, as above
+            The result from the ThirdPartyRules module, as above.
         """
-        if self.third_party_rules is None:
-            return True
+        # Bail out early without hitting the store if we don't have any callbacks to run.
+        if len(self._check_event_allowed_callbacks) == 0:
+            return True, None
 
         prev_state_ids = await context.get_prev_state_ids()
 
@@ -78,29 +214,46 @@ async def check_event_allowed(
         # the hashes and signatures.
         event.freeze()
 
-        return await self.third_party_rules.check_event_allowed(event, state_events)
+        for callback in self._check_event_allowed_callbacks:
+            try:
+                res, replacement_data = await callback(event, state_events)
+            except Exception as e:
+                logger.warning("Failed to run module API callback %s: %s", callback, e)
+                continue
+
+            # Return if the event shouldn't be allowed or if the module came up with a
+            # replacement dict for the event.
+            if res is False:
+                return res, None
+            elif isinstance(replacement_data, dict):
+                return True, replacement_data
+
+        return True, None
 
     async def on_create_room(
         self, requester: Requester, config: dict, is_requester_admin: bool
-    ) -> bool:
-        """Intercept requests to create room to allow, deny or update the
-        request config.
+    ) -> None:
+        """Intercept requests to create room to maybe deny it (via an exception) or
+        update the request config.
 
         Args:
             requester
             config: The creation config from the client.
             is_requester_admin: If the requester is an admin
-
-        Returns:
-            Whether room creation is allowed or denied.
         """
-
-        if self.third_party_rules is None:
-            return True
-
-        return await self.third_party_rules.on_create_room(
-            requester, config, is_requester_admin
-        )
+        for callback in self._on_create_room_callbacks:
+            try:
+                await callback(requester, config, is_requester_admin)
+            except Exception as e:
+                # Don't silence the errors raised by this callback since we expect it to
+                # raise an exception to deny the creation of the room; instead make sure
+                # it's a SynapseError we can send to clients.
+                if not isinstance(e, SynapseError):
+                    e = SynapseError(
+                        403, "Room creation forbidden with these parameters"
+                    )
+
+                raise e
 
     async def check_threepid_can_be_invited(
         self, medium: str, address: str, room_id: str
@@ -115,15 +268,20 @@ async def check_threepid_can_be_invited(
         Returns:
             True if the 3PID can be invited, False if not.
         """
-
-        if self.third_party_rules is None:
+        # Bail out early without hitting the store if we don't have any callbacks to run.
+        if len(self._check_threepid_can_be_invited_callbacks) == 0:
             return True
 
         state_events = await self._get_state_map_for_room(room_id)
 
-        return await self.third_party_rules.check_threepid_can_be_invited(
-            medium, address, state_events
-        )
+        for callback in self._check_threepid_can_be_invited_callbacks:
+            try:
+                if await callback(medium, address, state_events) is False:
+                    return False
+            except Exception as e:
+                logger.warning("Failed to run module API callback %s: %s", callback, e)
+
+        return True
 
     async def check_visibility_can_be_modified(
         self, room_id: str, new_visibility: str
@@ -138,18 +296,20 @@ async def check_visibility_can_be_modified(
         Returns:
             True if the room's visibility can be modified, False if not.
         """
-        if self.third_party_rules is None:
-            return True
-
-        check_func = getattr(
-            self.third_party_rules, "check_visibility_can_be_modified", None
-        )
-        if not check_func or not callable(check_func):
+        # Bail out early without hitting the store if we don't have any callback
+        if len(self._check_visibility_can_be_modified_callbacks) == 0:
             return True
 
         state_events = await self._get_state_map_for_room(room_id)
 
-        return await check_func(room_id, state_events, new_visibility)
+        for callback in self._check_visibility_can_be_modified_callbacks:
+            try:
+                if await callback(room_id, state_events, new_visibility) is False:
+                    return False
+            except Exception as e:
+                logger.warning("Failed to run module API callback %s: %s", callback, e)
+
+        return True
 
     async def _get_state_map_for_room(self, room_id: str) -> StateMap[EventBase]:
         """Given a room ID, return the state events of that room.
diff --git a/synapse/events/utils.py b/synapse/events/utils.py
index 7050f0c885..b6da2f60af 100644
--- a/synapse/events/utils.py
+++ b/synapse/events/utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +17,7 @@
 
 from frozendict import frozendict
 
-from synapse.api.constants import EventTypes, RelationTypes
+from synapse.api.constants import EventContentFields, EventTypes, RelationTypes
 from synapse.api.errors import Codes, SynapseError
 from synapse.api.room_versions import RoomVersion
 from synapse.util.async_helpers import yieldable_gather_results
@@ -110,6 +109,8 @@ def add_fields(*fields):
         add_fields("creator")
     elif event_type == EventTypes.JoinRules:
         add_fields("join_rule")
+        if room_version.msc3083_join_rules:
+            add_fields("allow")
     elif event_type == EventTypes.PowerLevels:
         add_fields(
             "users",
@@ -125,12 +126,21 @@ def add_fields(*fields):
         if room_version.msc2176_redaction_rules:
             add_fields("invite")
 
+        if room_version.msc2716_historical:
+            add_fields("historical")
+
     elif event_type == EventTypes.Aliases and room_version.special_case_aliases_auth:
         add_fields("aliases")
     elif event_type == EventTypes.RoomHistoryVisibility:
         add_fields("history_visibility")
     elif event_type == EventTypes.Redaction and room_version.msc2176_redaction_rules:
         add_fields("redacts")
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_INSERTION:
+        add_fields(EventContentFields.MSC2716_NEXT_CHUNK_ID)
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_CHUNK:
+        add_fields(EventContentFields.MSC2716_CHUNK_ID)
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_MARKER:
+        add_fields(EventContentFields.MSC2716_MARKER_INSERTION)
 
     allowed_fields = {k: v for k, v in event_dict.items() if k in allowed_keys}
 
diff --git a/synapse/events/validator.py b/synapse/events/validator.py
index f8f3b1a31e..fa6987d7cb 100644
--- a/synapse/events/validator.py
+++ b/synapse/events/validator.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/federation/__init__.py b/synapse/federation/__init__.py
index f5f0bdfca3..46300cba25 100644
--- a/synapse/federation/__init__.py
+++ b/synapse/federation/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index 383737520a..024e440ff4 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -15,11 +14,6 @@
 # limitations under the License.
 import logging
 from collections import namedtuple
-from typing import Iterable, List
-
-from twisted.internet import defer
-from twisted.internet.defer import Deferred, DeferredList
-from twisted.python.failure import Failure
 
 from synapse.api.constants import MAX_DEPTH, EventTypes, Membership
 from synapse.api.errors import Codes, SynapseError
@@ -29,11 +23,6 @@
 from synapse.events import EventBase, make_event_from_dict
 from synapse.events.utils import prune_event, validate_canonicaljson
 from synapse.http.servlet import assert_params_in_dict
-from synapse.logging.context import (
-    PreserveLoggingContext,
-    current_context,
-    make_deferred_yieldable,
-)
 from synapse.types import JsonDict, get_domain_from_id
 
 logger = logging.getLogger(__name__)
@@ -49,116 +38,82 @@ def __init__(self, hs):
         self.store = hs.get_datastore()
         self._clock = hs.get_clock()
 
-    def _check_sigs_and_hash(
+    async def _check_sigs_and_hash(
         self, room_version: RoomVersion, pdu: EventBase
-    ) -> Deferred:
-        return make_deferred_yieldable(
-            self._check_sigs_and_hashes(room_version, [pdu])[0]
-        )
-
-    def _check_sigs_and_hashes(
-        self, room_version: RoomVersion, pdus: List[EventBase]
-    ) -> List[Deferred]:
-        """Checks that each of the received events is correctly signed by the
-        sending server.
+    ) -> EventBase:
+        """Checks that event is correctly signed by the sending server.
 
         Args:
-            room_version: The room version of the PDUs
-            pdus: the events to be checked
+            room_version: The room version of the PDU
+            pdu: the event to be checked
 
         Returns:
-            For each input event, a deferred which:
-              * returns the original event if the checks pass
-              * returns a redacted version of the event (if the signature
+              * the original event if the checks pass
+              * a redacted version of the event (if the signature
                 matched but the hash did not)
-              * throws a SynapseError if the signature check failed.
-            The deferreds run their callbacks in the sentinel
-        """
-        deferreds = _check_sigs_on_pdus(self.keyring, room_version, pdus)
-
-        ctx = current_context()
-
-        @defer.inlineCallbacks
-        def callback(_, pdu: EventBase):
-            with PreserveLoggingContext(ctx):
-                if not check_event_content_hash(pdu):
-                    # let's try to distinguish between failures because the event was
-                    # redacted (which are somewhat expected) vs actual ball-tampering
-                    # incidents.
-                    #
-                    # This is just a heuristic, so we just assume that if the keys are
-                    # about the same between the redacted and received events, then the
-                    # received event was probably a redacted copy (but we then use our
-                    # *actual* redacted copy to be on the safe side.)
-                    redacted_event = prune_event(pdu)
-                    if set(redacted_event.keys()) == set(pdu.keys()) and set(
-                        redacted_event.content.keys()
-                    ) == set(pdu.content.keys()):
-                        logger.info(
-                            "Event %s seems to have been redacted; using our redacted "
-                            "copy",
-                            pdu.event_id,
-                        )
-                    else:
-                        logger.warning(
-                            "Event %s content has been tampered, redacting",
-                            pdu.event_id,
-                        )
-                    return redacted_event
-
-                result = yield defer.ensureDeferred(
-                    self.spam_checker.check_event_for_spam(pdu)
+              * throws a SynapseError if the signature check failed."""
+        try:
+            await _check_sigs_on_pdu(self.keyring, room_version, pdu)
+        except SynapseError as e:
+            logger.warning(
+                "Signature check failed for %s: %s",
+                pdu.event_id,
+                e,
+            )
+            raise
+
+        if not check_event_content_hash(pdu):
+            # let's try to distinguish between failures because the event was
+            # redacted (which are somewhat expected) vs actual ball-tampering
+            # incidents.
+            #
+            # This is just a heuristic, so we just assume that if the keys are
+            # about the same between the redacted and received events, then the
+            # received event was probably a redacted copy (but we then use our
+            # *actual* redacted copy to be on the safe side.)
+            redacted_event = prune_event(pdu)
+            if set(redacted_event.keys()) == set(pdu.keys()) and set(
+                redacted_event.content.keys()
+            ) == set(pdu.content.keys()):
+                logger.info(
+                    "Event %s seems to have been redacted; using our redacted copy",
+                    pdu.event_id,
                 )
-
-                if result:
-                    logger.warning(
-                        "Event contains spam, redacting %s: %s",
-                        pdu.event_id,
-                        pdu.get_pdu_json(),
-                    )
-                    return prune_event(pdu)
-
-                return pdu
-
-        def errback(failure: Failure, pdu: EventBase):
-            failure.trap(SynapseError)
-            with PreserveLoggingContext(ctx):
+            else:
                 logger.warning(
-                    "Signature check failed for %s: %s",
+                    "Event %s content has been tampered, redacting",
                     pdu.event_id,
-                    failure.getErrorMessage(),
                 )
-            return failure
+            return redacted_event
 
-        for deferred, pdu in zip(deferreds, pdus):
-            deferred.addCallbacks(
-                callback, errback, callbackArgs=[pdu], errbackArgs=[pdu]
-            )
+        result = await self.spam_checker.check_event_for_spam(pdu)
 
-        return deferreds
+        if result:
+            logger.warning("Event contains spam, soft-failing %s", pdu.event_id)
+            # we redact (to save disk space) as well as soft-failing (to stop
+            # using the event in prev_events).
+            redacted_event = prune_event(pdu)
+            redacted_event.internal_metadata.soft_failed = True
+            return redacted_event
 
+        return pdu
 
-class PduToCheckSig(
-    namedtuple(
-        "PduToCheckSig", ["pdu", "redacted_pdu_json", "sender_domain", "deferreds"]
-    )
-):
+
+class PduToCheckSig(namedtuple("PduToCheckSig", ["pdu", "sender_domain", "deferreds"])):
     pass
 
 
-def _check_sigs_on_pdus(
-    keyring: Keyring, room_version: RoomVersion, pdus: Iterable[EventBase]
-) -> List[Deferred]:
+async def _check_sigs_on_pdu(
+    keyring: Keyring, room_version: RoomVersion, pdu: EventBase
+) -> None:
     """Check that the given events are correctly signed
 
+    Raise a SynapseError if the event wasn't correctly signed.
+
     Args:
         keyring: keyring object to do the checks
         room_version: the room version of the PDUs
         pdus: the events to be checked
-
-    Returns:
-        A Deferred for each event in pdus, which will either succeed if
-           the signatures are valid, or fail (with a SynapseError) if not.
     """
 
     # we want to check that the event is signed by:
@@ -182,93 +137,75 @@ def _check_sigs_on_pdus(
     # let's start by getting the domain for each pdu, and flattening the event back
     # to JSON.
 
-    pdus_to_check = [
-        PduToCheckSig(
-            pdu=p,
-            redacted_pdu_json=prune_event(p).get_pdu_json(),
-            sender_domain=get_domain_from_id(p.sender),
-            deferreds=[],
-        )
-        for p in pdus
-    ]
-
     # First we check that the sender event is signed by the sender's domain
     # (except if its a 3pid invite, in which case it may be sent by any server)
-    pdus_to_check_sender = [p for p in pdus_to_check if not _is_invite_via_3pid(p.pdu)]
-
-    more_deferreds = keyring.verify_json_objects_for_server(
-        [
-            (
-                p.sender_domain,
-                p.redacted_pdu_json,
-                p.pdu.origin_server_ts if room_version.enforce_key_validity else 0,
-                p.pdu.event_id,
+    if not _is_invite_via_3pid(pdu):
+        try:
+            await keyring.verify_event_for_server(
+                get_domain_from_id(pdu.sender),
+                pdu,
+                pdu.origin_server_ts if room_version.enforce_key_validity else 0,
             )
-            for p in pdus_to_check_sender
-        ]
-    )
-
-    def sender_err(e, pdu_to_check):
-        errmsg = "event id %s: unable to verify signature for sender %s: %s" % (
-            pdu_to_check.pdu.event_id,
-            pdu_to_check.sender_domain,
-            e.getErrorMessage(),
-        )
-        raise SynapseError(403, errmsg, Codes.FORBIDDEN)
-
-    for p, d in zip(pdus_to_check_sender, more_deferreds):
-        d.addErrback(sender_err, p)
-        p.deferreds.append(d)
+        except Exception as e:
+            errmsg = "event id %s: unable to verify signature for sender %s: %s" % (
+                pdu.event_id,
+                get_domain_from_id(pdu.sender),
+                e,
+            )
+            raise SynapseError(403, errmsg, Codes.FORBIDDEN)
 
     # now let's look for events where the sender's domain is different to the
     # event id's domain (normally only the case for joins/leaves), and add additional
     # checks. Only do this if the room version has a concept of event ID domain
     # (ie, the room version uses old-style non-hash event IDs).
-    if room_version.event_format == EventFormatVersions.V1:
-        pdus_to_check_event_id = [
-            p
-            for p in pdus_to_check
-            if p.sender_domain != get_domain_from_id(p.pdu.event_id)
-        ]
-
-        more_deferreds = keyring.verify_json_objects_for_server(
-            [
-                (
-                    get_domain_from_id(p.pdu.event_id),
-                    p.redacted_pdu_json,
-                    p.pdu.origin_server_ts if room_version.enforce_key_validity else 0,
-                    p.pdu.event_id,
+    if room_version.event_format == EventFormatVersions.V1 and get_domain_from_id(
+        pdu.event_id
+    ) != get_domain_from_id(pdu.sender):
+        try:
+            await keyring.verify_event_for_server(
+                get_domain_from_id(pdu.event_id),
+                pdu,
+                pdu.origin_server_ts if room_version.enforce_key_validity else 0,
+            )
+        except Exception as e:
+            errmsg = (
+                "event id %s: unable to verify signature for event id domain %s: %s"
+                % (
+                    pdu.event_id,
+                    get_domain_from_id(pdu.event_id),
+                    e,
                 )
-                for p in pdus_to_check_event_id
-            ]
-        )
+            )
+            raise SynapseError(403, errmsg, Codes.FORBIDDEN)
 
-        def event_err(e, pdu_to_check):
+    # If this is a join event for a restricted room it may have been authorised
+    # via a different server from the sending server. Check those signatures.
+    if (
+        room_version.msc3083_join_rules
+        and pdu.type == EventTypes.Member
+        and pdu.membership == Membership.JOIN
+        and "join_authorised_via_users_server" in pdu.content
+    ):
+        authorising_server = get_domain_from_id(
+            pdu.content["join_authorised_via_users_server"]
+        )
+        try:
+            await keyring.verify_event_for_server(
+                authorising_server,
+                pdu,
+                pdu.origin_server_ts if room_version.enforce_key_validity else 0,
+            )
+        except Exception as e:
             errmsg = (
-                "event id %s: unable to verify signature for event id domain: %s"
-                % (pdu_to_check.pdu.event_id, e.getErrorMessage())
+                "event id %s: unable to verify signature for authorising server %s: %s"
+                % (
+                    pdu.event_id,
+                    authorising_server,
+                    e,
+                )
             )
             raise SynapseError(403, errmsg, Codes.FORBIDDEN)
 
-        for p, d in zip(pdus_to_check_event_id, more_deferreds):
-            d.addErrback(event_err, p)
-            p.deferreds.append(d)
-
-    # replace lists of deferreds with single Deferreds
-    return [_flatten_deferred_list(p.deferreds) for p in pdus_to_check]
-
-
-def _flatten_deferred_list(deferreds: List[Deferred]) -> Deferred:
-    """Given a list of deferreds, either return the single deferred,
-    combine into a DeferredList, or return an already resolved deferred.
-    """
-    if len(deferreds) > 1:
-        return DeferredList(deferreds, fireOnOneErrback=True, consumeErrors=True)
-    elif len(deferreds) == 1:
-        return deferreds[0]
-    else:
-        return defer.succeed(None)
-
 
 def _is_invite_via_3pid(event: EventBase) -> bool:
     return (
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 184096d165..29979414e3 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015, 2016 OpenMarket Ltd
-# Copyrignt 2020 Sorunome
-# Copyrignt 2020 The Matrix.org Foundation C.I.C.
+# Copyright 2015-2021 The Matrix.org Foundation C.I.C.
+# Copyright 2020 Sorunome
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,9 +19,10 @@
 import logging
 from typing import (
     TYPE_CHECKING,
-    Any,
     Awaitable,
     Callable,
+    Collection,
+    Container,
     Dict,
     Iterable,
     List,
@@ -38,9 +37,6 @@
 import attr
 from prometheus_client import Counter
 
-from twisted.internet import defer
-from twisted.internet.defer import Deferred
-
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import (
     CodeMessageException,
@@ -58,10 +54,10 @@
 )
 from synapse.events import EventBase, builder
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
-from synapse.logging.context import make_deferred_yieldable, preserve_fn
+from synapse.federation.transport.client import SendJoinResponse
 from synapse.logging.utils import log_function
 from synapse.types import JsonDict, get_domain_from_id
-from synapse.util import unwrapFirstError
+from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.retryutils import NotRetryingDestination
 
@@ -83,14 +79,22 @@ class InvalidResponseError(RuntimeError):
     we couldn't parse
     """
 
-    pass
+
+@attr.s(slots=True, frozen=True, auto_attribs=True)
+class SendJoinResult:
+    # The event to persist.
+    event: EventBase
+    # A string giving the server the event was sent to.
+    origin: str
+    state: List[EventBase]
+    auth_chain: List[EventBase]
 
 
 class FederationClient(FederationBase):
     def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
-        self.pdu_destination_tried = {}  # type: Dict[str, Dict[str, int]]
+        self.pdu_destination_tried: Dict[str, Dict[str, int]] = {}
         self._clock.looping_call(self._clear_tried_cache, 60 * 1000)
         self.state = hs.get_state_handler()
         self.transport_layer = hs.get_federation_transport_client()
@@ -98,7 +102,7 @@ def __init__(self, hs: "HomeServer"):
         self.hostname = hs.hostname
         self.signing_key = hs.signing_key
 
-        self._get_pdu_cache = ExpiringCache(
+        self._get_pdu_cache: ExpiringCache[str, EventBase] = ExpiringCache(
             cache_name="get_pdu_cache",
             clock=self._clock,
             max_len=1000,
@@ -297,10 +301,10 @@ async def get_pdu(
                     transaction_data,
                 )
 
-                pdu_list = [
+                pdu_list: List[EventBase] = [
                     event_from_pdu_json(p, room_version, outlier=outlier)
                     for p in transaction_data["pdus"]
-                ]  # type: List[EventBase]
+                ]
 
                 if pdu_list and pdu_list[0]:
                     pdu = pdu_list[0]
@@ -362,10 +366,9 @@ async def get_room_state_ids(
     async def _check_sigs_and_hash_and_fetch(
         self,
         origin: str,
-        pdus: List[EventBase],
+        pdus: Collection[EventBase],
         room_version: RoomVersion,
         outlier: bool = False,
-        include_none: bool = False,
     ) -> List[EventBase]:
         """Takes a list of PDUs and checks the signatures and hashes of each
         one. If a PDU fails its signature check then we check if we have it in
@@ -382,57 +385,87 @@ async def _check_sigs_and_hash_and_fetch(
             pdu
             room_version
             outlier: Whether the events are outliers or not
-            include_none: Whether to include None in the returned list
-                for events that have failed their checks
 
         Returns:
             A list of PDUs that have valid signatures and hashes.
         """
-        deferreds = self._check_sigs_and_hashes(room_version, pdus)
 
-        async def handle_check_result(pdu: EventBase, deferred: Deferred):
-            try:
-                res = await make_deferred_yieldable(deferred)
-            except SynapseError:
-                res = None
+        # We limit how many PDUs we check at once, as if we try to do hundreds
+        # of thousands of PDUs at once we see large memory spikes.
 
-            if not res:
-                # Check local db.
-                res = await self.store.get_event(
-                    pdu.event_id, allow_rejected=True, allow_none=True
-                )
+        valid_pdus = []
 
-            pdu_origin = get_domain_from_id(pdu.sender)
-            if not res and pdu_origin != origin:
-                try:
-                    res = await self.get_pdu(
-                        destinations=[pdu_origin],
-                        event_id=pdu.event_id,
-                        room_version=room_version,
-                        outlier=outlier,
-                        timeout=10000,
-                    )
-                except SynapseError:
-                    pass
+        async def _execute(pdu: EventBase) -> None:
+            valid_pdu = await self._check_sigs_and_hash_and_fetch_one(
+                pdu=pdu,
+                origin=origin,
+                outlier=outlier,
+                room_version=room_version,
+            )
 
-            if not res:
-                logger.warning(
-                    "Failed to find copy of %s with valid signature", pdu.event_id
-                )
+            if valid_pdu:
+                valid_pdus.append(valid_pdu)
+
+        await concurrently_execute(_execute, pdus, 10000)
+
+        return valid_pdus
+
+    async def _check_sigs_and_hash_and_fetch_one(
+        self,
+        pdu: EventBase,
+        origin: str,
+        room_version: RoomVersion,
+        outlier: bool = False,
+    ) -> Optional[EventBase]:
+        """Takes a PDU and checks its signatures and hashes. If the PDU fails
+        its signature check then we check if we have it in the database and if
+        not then request if from the originating server of that PDU.
+
+        If then PDU fails its content hash check then it is redacted.
+
+        Args:
+            origin
+            pdu
+            room_version
+            outlier: Whether the events are outliers or not
+            include_none: Whether to include None in the returned list
+                for events that have failed their checks
+
+        Returns:
+            The PDU (possibly redacted) if it has valid signatures and hashes.
+        """
 
-            return res
+        res = None
+        try:
+            res = await self._check_sigs_and_hash(room_version, pdu)
+        except SynapseError:
+            pass
+
+        if not res:
+            # Check local db.
+            res = await self.store.get_event(
+                pdu.event_id, allow_rejected=True, allow_none=True
+            )
 
-        handle = preserve_fn(handle_check_result)
-        deferreds2 = [handle(pdu, deferred) for pdu, deferred in zip(pdus, deferreds)]
+        pdu_origin = get_domain_from_id(pdu.sender)
+        if not res and pdu_origin != origin:
+            try:
+                res = await self.get_pdu(
+                    destinations=[pdu_origin],
+                    event_id=pdu.event_id,
+                    room_version=room_version,
+                    outlier=outlier,
+                    timeout=10000,
+                )
+            except SynapseError:
+                pass
 
-        valid_pdus = await make_deferred_yieldable(
-            defer.gatherResults(deferreds2, consumeErrors=True)
-        ).addErrback(unwrapFirstError)
+        if not res:
+            logger.warning(
+                "Failed to find copy of %s with valid signature", pdu.event_id
+            )
 
-        if include_none:
-            return valid_pdus
-        else:
-            return [p for p in valid_pdus if p]
+        return res
 
     async def get_event_auth(
         self, destination: str, room_id: str, event_id: str
@@ -454,11 +487,34 @@ async def get_event_auth(
 
         return signed_auth
 
+    def _is_unknown_endpoint(
+        self, e: HttpResponseException, synapse_error: Optional[SynapseError] = None
+    ) -> bool:
+        """
+        Returns true if the response was due to an endpoint being unimplemented.
+
+        Args:
+            e: The error response received from the remote server.
+            synapse_error: The above error converted to a SynapseError. This is
+                automatically generated if not provided.
+
+        """
+        if synapse_error is None:
+            synapse_error = e.to_synapse_error()
+        # There is no good way to detect an "unknown" endpoint.
+        #
+        # Dendrite returns a 404 (with no body); synapse returns a 400
+        # with M_UNRECOGNISED.
+        return e.code == 404 or (
+            e.code == 400 and synapse_error.errcode == Codes.UNRECOGNIZED
+        )
+
     async def _try_destination_list(
         self,
         description: str,
         destinations: Iterable[str],
         callback: Callable[[str], Awaitable[T]],
+        failover_errcodes: Optional[Container[str]] = None,
         failover_on_unknown_endpoint: bool = False,
     ) -> T:
         """Try an operation on a series of servers, until it succeeds
@@ -471,14 +527,17 @@ async def _try_destination_list(
             callback:  Function to run for each server. Passed a single
                 argument: the server_name to try.
 
-                If the callback raises a CodeMessageException with a 300/400 code,
-                attempts to perform the operation stop immediately and the exception is
-                reraised.
+                If the callback raises a CodeMessageException with a 300/400 code or
+                an UnsupportedRoomVersionError, attempts to perform the operation
+                stop immediately and the exception is reraised.
 
                 Otherwise, if the callback raises an Exception the error is logged and the
                 next server tried. Normally the stacktrace is logged but this is
                 suppressed if the exception is an InvalidResponseError.
 
+            failover_errcodes: Error codes (specific to this endpoint) which should
+                cause a failover when received as part of an HTTP 400 error.
+
             failover_on_unknown_endpoint: if True, we will try other servers if it looks
                 like a server doesn't support the endpoint. This is typically useful
                 if the endpoint in question is new or experimental.
@@ -490,13 +549,15 @@ async def _try_destination_list(
             SynapseError if the chosen remote server returns a 300/400 code, or
             no servers were reachable.
         """
+        if failover_errcodes is None:
+            failover_errcodes = ()
+
         for destination in destinations:
             if destination == self.server_name:
                 continue
 
             try:
-                res = await callback(destination)
-                return res
+                return await callback(destination)
             except InvalidResponseError as e:
                 logger.warning("Failed to %s via %s: %s", description, destination, e)
             except UnsupportedRoomVersionError:
@@ -505,17 +566,21 @@ async def _try_destination_list(
                 synapse_error = e.to_synapse_error()
                 failover = False
 
+                # Failover should occur:
+                #
+                # * On internal server errors.
+                # * If the destination responds that it cannot complete the request.
+                # * If the destination doesn't implemented the endpoint for some reason.
                 if 500 <= e.code < 600:
                     failover = True
 
-                elif failover_on_unknown_endpoint:
-                    # there is no good way to detect an "unknown" endpoint. Dendrite
-                    # returns a 404 (with no body); synapse returns a 400
-                    # with M_UNRECOGNISED.
-                    if e.code == 404 or (
-                        e.code == 400 and synapse_error.errcode == Codes.UNRECOGNIZED
-                    ):
-                        failover = True
+                elif e.code == 400 and synapse_error.errcode in failover_errcodes:
+                    failover = True
+
+                elif failover_on_unknown_endpoint and self._is_unknown_endpoint(
+                    e, synapse_error
+                ):
+                    failover = True
 
                 if not failover:
                     raise synapse_error from e
@@ -573,11 +638,11 @@ async def make_membership_event(
             UnsupportedRoomVersionError: if remote responds with
                 a room version we don't understand.
 
-            SynapseError: if the chosen remote server returns a 300/400 code.
-
-            RuntimeError: if no servers were reachable.
+            SynapseError: if the chosen remote server returns a 300/400 code, or
+                no servers successfully handle the request.
         """
         valid_memberships = {Membership.JOIN, Membership.LEAVE, Membership.KNOCK}
+
         if membership not in valid_memberships:
             raise RuntimeError(
                 "make_membership_event called with membership='%s', must be one of %s"
@@ -596,6 +661,13 @@ async def send_request(destination: str) -> Tuple[str, EventBase, RoomVersion]:
             if not room_version:
                 raise UnsupportedRoomVersionError()
 
+            if not room_version.msc2403_knocking and membership == Membership.KNOCK:
+                raise SynapseError(
+                    400,
+                    "This room version does not support knocking",
+                    errcode=Codes.FORBIDDEN,
+                )
+
             pdu_dict = ret.get("event", None)
             if not isinstance(pdu_dict, dict):
                 raise InvalidResponseError("Bad 'event' field in response")
@@ -620,13 +692,25 @@ async def send_request(destination: str) -> Tuple[str, EventBase, RoomVersion]:
 
             return destination, ev, room_version
 
+        # MSC3083 defines additional error codes for room joins. Unfortunately
+        # we do not yet know the room version, assume these will only be returned
+        # by valid room versions.
+        failover_errcodes = (
+            (Codes.UNABLE_AUTHORISE_JOIN, Codes.UNABLE_TO_GRANT_JOIN)
+            if membership == Membership.JOIN
+            else None
+        )
+
         return await self._try_destination_list(
-            "make_" + membership, destinations, send_request
+            "make_" + membership,
+            destinations,
+            send_request,
+            failover_errcodes=failover_errcodes,
         )
 
     async def send_join(
         self, destinations: Iterable[str], pdu: EventBase, room_version: RoomVersion
-    ) -> Dict[str, Any]:
+    ) -> SendJoinResult:
         """Sends a join event to one of a list of homeservers.
 
         Doing so will cause the remote server to add the event to the graph,
@@ -640,32 +724,40 @@ async def send_join(
                 did the make_join)
 
         Returns:
-            a dict with members ``origin`` (a string
-            giving the server the event was sent to, ``state`` (?) and
-            ``auth_chain``.
+            The result of the send join request.
 
         Raises:
-            SynapseError: if the chosen remote server returns a 300/400 code.
-
-            RuntimeError: if no servers were reachable.
+            SynapseError: if the chosen remote server returns a 300/400 code, or
+                no servers successfully handle the request.
         """
 
-        async def send_request(destination) -> Dict[str, Any]:
-            content = await self._do_send_join(destination, pdu)
-
-            logger.debug("Got content: %s", content)
-
-            state = [
-                event_from_pdu_json(p, room_version, outlier=True)
-                for p in content.get("state", [])
-            ]
+        async def send_request(destination) -> SendJoinResult:
+            response = await self._do_send_join(room_version, destination, pdu)
+
+            # If an event was returned (and expected to be returned):
+            #
+            # * Ensure it has the same event ID (note that the event ID is a hash
+            #   of the event fields for versions which support MSC3083).
+            # * Ensure the signatures are good.
+            #
+            # Otherwise, fallback to the provided event.
+            if room_version.msc3083_join_rules and response.event:
+                event = response.event
+
+                valid_pdu = await self._check_sigs_and_hash_and_fetch_one(
+                    pdu=event,
+                    origin=destination,
+                    outlier=True,
+                    room_version=room_version,
+                )
 
-            auth_chain = [
-                event_from_pdu_json(p, room_version, outlier=True)
-                for p in content.get("auth_chain", [])
-            ]
+                if valid_pdu is None or event.event_id != pdu.event_id:
+                    raise InvalidResponseError("Returned an invalid join event")
+            else:
+                event = pdu
 
-            pdus = {p.event_id: p for p in itertools.chain(state, auth_chain)}
+            state = response.state
+            auth_chain = response.auth_events
 
             create_event = None
             for e in state:
@@ -676,7 +768,7 @@ async def send_request(destination) -> Dict[str, Any]:
             if create_event is None:
                 # If the state doesn't have a create event then the room is
                 # invalid, and it would fail auth checks anyway.
-                raise SynapseError(400, "No create event in state")
+                raise InvalidResponseError("No create event in state")
 
             # the room version should be sane.
             create_room_version = create_event.content.get(
@@ -690,14 +782,29 @@ async def send_request(destination) -> Dict[str, Any]:
                     % (create_room_version,)
                 )
 
-            valid_pdus = await self._check_sigs_and_hash_and_fetch(
-                destination,
-                list(pdus.values()),
-                outlier=True,
-                room_version=room_version,
+            logger.info(
+                "Processing from send_join %d events", len(state) + len(auth_chain)
             )
 
-            valid_pdus_map = {p.event_id: p for p in valid_pdus}
+            # We now go and check the signatures and hashes for the event. Note
+            # that we limit how many events we process at a time to keep the
+            # memory overhead from exploding.
+            valid_pdus_map: Dict[str, EventBase] = {}
+
+            async def _execute(pdu: EventBase) -> None:
+                valid_pdu = await self._check_sigs_and_hash_and_fetch_one(
+                    pdu=pdu,
+                    origin=destination,
+                    outlier=True,
+                    room_version=room_version,
+                )
+
+                if valid_pdu:
+                    valid_pdus_map[valid_pdu.event_id] = valid_pdu
+
+            await concurrently_execute(
+                _execute, itertools.chain(state, auth_chain), 10000
+            )
 
             # NB: We *need* to copy to ensure that we don't have multiple
             # references being passed on, as that causes... issues.
@@ -730,49 +837,63 @@ async def send_request(destination) -> Dict[str, Any]:
                     % (auth_chain_create_events,)
                 )
 
-            return {
-                "state": signed_state,
-                "auth_chain": signed_auth,
-                "origin": destination,
-            }
+            return SendJoinResult(
+                event=event,
+                state=signed_state,
+                auth_chain=signed_auth,
+                origin=destination,
+            )
 
-        return await self._try_destination_list("send_join", destinations, send_request)
+        # MSC3083 defines additional error codes for room joins.
+        failover_errcodes = None
+        if room_version.msc3083_join_rules:
+            failover_errcodes = (
+                Codes.UNABLE_AUTHORISE_JOIN,
+                Codes.UNABLE_TO_GRANT_JOIN,
+            )
+
+            # If the join is being authorised via allow rules, we need to send
+            # the /send_join back to the same server that was originally used
+            # with /make_join.
+            if "join_authorised_via_users_server" in pdu.content:
+                destinations = [
+                    get_domain_from_id(pdu.content["join_authorised_via_users_server"])
+                ]
+
+        return await self._try_destination_list(
+            "send_join", destinations, send_request, failover_errcodes=failover_errcodes
+        )
 
-    async def _do_send_join(self, destination: str, pdu: EventBase) -> JsonDict:
+    async def _do_send_join(
+        self, room_version: RoomVersion, destination: str, pdu: EventBase
+    ) -> SendJoinResponse:
         time_now = self._clock.time_msec()
 
         try:
             return await self.transport_layer.send_join_v2(
+                room_version=room_version,
                 destination=destination,
                 room_id=pdu.room_id,
                 event_id=pdu.event_id,
                 content=pdu.get_pdu_json(time_now),
             )
         except HttpResponseException as e:
-            if e.code in [400, 404]:
-                err = e.to_synapse_error()
-
-                # If we receive an error response that isn't a generic error, or an
-                # unrecognised endpoint error, we  assume that the remote understands
-                # the v2 invite API and this is a legitimate error.
-                if err.errcode not in [Codes.UNKNOWN, Codes.UNRECOGNIZED]:
-                    raise err
-            else:
-                raise e.to_synapse_error()
+            # If an error is received that is due to an unrecognised endpoint,
+            # fallback to the v1 endpoint. Otherwise consider it a legitmate error
+            # and raise.
+            if not self._is_unknown_endpoint(e):
+                raise
 
         logger.debug("Couldn't send_join with the v2 API, falling back to the v1 API")
 
-        resp = await self.transport_layer.send_join_v1(
+        return await self.transport_layer.send_join_v1(
+            room_version=room_version,
             destination=destination,
             room_id=pdu.room_id,
             event_id=pdu.event_id,
             content=pdu.get_pdu_json(time_now),
         )
 
-        # We expect the v1 API to respond with [200, content], so we only return the
-        # content.
-        return resp[1]
-
     async def send_invite(
         self,
         destination: str,
@@ -805,6 +926,11 @@ async def _do_send_invite(
 
         Returns:
             The event as a dict as returned by the remote server
+
+        Raises:
+            SynapseError: if the remote server returns an error or if the server
+                only supports the v1 endpoint and a room version other than "1"
+                or "2" is requested.
         """
         time_now = self._clock.time_msec()
 
@@ -820,28 +946,19 @@ async def _do_send_invite(
                 },
             )
         except HttpResponseException as e:
-            if e.code in [400, 404]:
-                err = e.to_synapse_error()
-
-                # If we receive an error response that isn't a generic error, we
-                # assume that the remote understands the v2 invite API and this
-                # is a legitimate error.
-                if err.errcode != Codes.UNKNOWN:
-                    raise err
-
-                # Otherwise, we assume that the remote server doesn't understand
-                # the v2 invite API. That's ok provided the room uses old-style event
-                # IDs.
+            # If an error is received that is due to an unrecognised endpoint,
+            # fallback to the v1 endpoint if the room uses old-style event IDs.
+            # Otherwise consider it a legitmate error and raise.
+            err = e.to_synapse_error()
+            if self._is_unknown_endpoint(e, err):
                 if room_version.event_format != EventFormatVersions.V1:
                     raise SynapseError(
                         400,
                         "User's homeserver does not support this room version",
                         Codes.UNSUPPORTED_ROOM_VERSION,
                     )
-            elif e.code in (403, 429):
-                raise e.to_synapse_error()
             else:
-                raise
+                raise err
 
         # Didn't work, try v1 API.
         # Note the v1 API returns a tuple of `(200, content)`
@@ -868,9 +985,8 @@ async def send_leave(self, destinations: Iterable[str], pdu: EventBase) -> None:
             pdu: event to be sent
 
         Raises:
-            SynapseError if the chosen remote server returns a 300/400 code.
-
-            RuntimeError if no servers were reachable.
+            SynapseError: if the chosen remote server returns a 300/400 code, or
+                no servers successfully handle the request.
         """
 
         async def send_request(destination: str) -> None:
@@ -892,16 +1008,11 @@ async def _do_send_leave(self, destination: str, pdu: EventBase) -> JsonDict:
                 content=pdu.get_pdu_json(time_now),
             )
         except HttpResponseException as e:
-            if e.code in [400, 404]:
-                err = e.to_synapse_error()
-
-                # If we receive an error response that isn't a generic error, or an
-                # unrecognised endpoint error, we  assume that the remote understands
-                # the v2 invite API and this is a legitimate error.
-                if err.errcode not in [Codes.UNKNOWN, Codes.UNRECOGNIZED]:
-                    raise err
-            else:
-                raise e.to_synapse_error()
+            # If an error is received that is due to an unrecognised endpoint,
+            # fallback to the v1 endpoint. Otherwise consider it a legitmate error
+            # and raise.
+            if not self._is_unknown_endpoint(e):
+                raise
 
         logger.debug("Couldn't send_leave with the v2 API, falling back to the v1 API")
 
@@ -945,7 +1056,7 @@ async def send_request(destination: str) -> JsonDict:
             return await self._do_send_knock(destination, pdu)
 
         return await self._try_destination_list(
-            "xyz.amorgan.knock/send_knock", destinations, send_request
+            "send_knock", destinations, send_request
         )
 
     async def _do_send_knock(self, destination: str, pdu: EventBase) -> JsonDict:
@@ -965,7 +1076,7 @@ async def _do_send_knock(self, destination: str, pdu: EventBase) -> JsonDict:
         """
         time_now = self._clock.time_msec()
 
-        return await self.transport_layer.send_knock_v2(
+        return await self.transport_layer.send_knock_v1(
             destination=destination,
             room_id=pdu.room_id,
             event_id=pdu.event_id,
@@ -997,7 +1108,8 @@ async def get_public_rooms(
             The response from the remote server.
 
         Raises:
-            HttpResponseException: There was an exception returned from the remote server
+            HttpResponseException / RequestSendFailed: There was an exception
+                returned from the remote server
             SynapseException: M_FORBIDDEN when the remote server has disallowed publicRoom
                 requests over federation
 
@@ -1178,8 +1290,136 @@ async def send_request(destination: str) -> FederationSpaceSummaryResult:
             failover_on_unknown_endpoint=True,
         )
 
+    async def get_room_hierarchy(
+        self,
+        destinations: Iterable[str],
+        room_id: str,
+        suggested_only: bool,
+    ) -> Tuple[JsonDict, Sequence[JsonDict], Sequence[str]]:
+        """
+        Call other servers to get a hierarchy of the given room.
+
+        Performs simple data validates and parsing of the response.
 
-@attr.s(frozen=True, slots=True)
+        Args:
+            destinations: The remote servers. We will try them in turn, omitting any
+                that have been blacklisted.
+            room_id: ID of the space to be queried
+            suggested_only:  If true, ask the remote server to only return children
+                with the "suggested" flag set
+
+        Returns:
+            A tuple of:
+                The room as a JSON dictionary.
+                A list of children rooms, as JSON dictionaries.
+                A list of inaccessible children room IDs.
+
+        Raises:
+            SynapseError if we were unable to get a valid summary from any of the
+               remote servers
+        """
+
+        async def send_request(
+            destination: str,
+        ) -> Tuple[JsonDict, Sequence[JsonDict], Sequence[str]]:
+            res = await self.transport_layer.get_room_hierarchy(
+                destination=destination,
+                room_id=room_id,
+                suggested_only=suggested_only,
+            )
+
+            room = res.get("room")
+            if not isinstance(room, dict):
+                raise InvalidResponseError("'room' must be a dict")
+
+            # Validate children_state of the room.
+            children_state = room.get("children_state", [])
+            if not isinstance(children_state, Sequence):
+                raise InvalidResponseError("'room.children_state' must be a list")
+            if any(not isinstance(e, dict) for e in children_state):
+                raise InvalidResponseError("Invalid event in 'children_state' list")
+            try:
+                [
+                    FederationSpaceSummaryEventResult.from_json_dict(e)
+                    for e in children_state
+                ]
+            except ValueError as e:
+                raise InvalidResponseError(str(e))
+
+            # Validate the children rooms.
+            children = res.get("children", [])
+            if not isinstance(children, Sequence):
+                raise InvalidResponseError("'children' must be a list")
+            if any(not isinstance(r, dict) for r in children):
+                raise InvalidResponseError("Invalid room in 'children' list")
+
+            # Validate the inaccessible children.
+            inaccessible_children = res.get("inaccessible_children", [])
+            if not isinstance(inaccessible_children, Sequence):
+                raise InvalidResponseError("'inaccessible_children' must be a list")
+            if any(not isinstance(r, str) for r in inaccessible_children):
+                raise InvalidResponseError(
+                    "Invalid room ID in 'inaccessible_children' list"
+                )
+
+            return room, children, inaccessible_children
+
+        try:
+            return await self._try_destination_list(
+                "fetch room hierarchy",
+                destinations,
+                send_request,
+                failover_on_unknown_endpoint=True,
+            )
+        except SynapseError as e:
+            # Fallback to the old federation API and translate the results if
+            # no servers implement the new API.
+            #
+            # The algorithm below is a bit inefficient as it only attempts to
+            # get information for the requested room, but the legacy API may
+            # return additional layers.
+            if e.code == 502:
+                legacy_result = await self.get_space_summary(
+                    destinations,
+                    room_id,
+                    suggested_only,
+                    max_rooms_per_space=None,
+                    exclude_rooms=[],
+                )
+
+                # Find the requested room in the response (and remove it).
+                for _i, room in enumerate(legacy_result.rooms):
+                    if room.get("room_id") == room_id:
+                        break
+                else:
+                    # The requested room was not returned, nothing we can do.
+                    raise
+                requested_room = legacy_result.rooms.pop(_i)
+
+                # Find any children events of the requested room.
+                children_events = []
+                children_room_ids = set()
+                for event in legacy_result.events:
+                    if event.room_id == room_id:
+                        children_events.append(event.data)
+                        children_room_ids.add(event.state_key)
+                # And add them under the requested room.
+                requested_room["children_state"] = children_events
+
+                # Find the children rooms.
+                children = []
+                for room in legacy_result.rooms:
+                    if room.get("room_id") in children_room_ids:
+                        children.append(room)
+
+                # It isn't clear from the response whether some of the rooms are
+                # not accessible.
+                return requested_room, children, ()
+
+            raise
+
+
+@attr.s(frozen=True, slots=True, auto_attribs=True)
 class FederationSpaceSummaryEventResult:
     """Represents a single event in the result of a successful get_space_summary call.
 
@@ -1188,12 +1428,13 @@ class FederationSpaceSummaryEventResult:
     object attributes.
     """
 
-    event_type = attr.ib(type=str)
-    state_key = attr.ib(type=str)
-    via = attr.ib(type=Sequence[str])
+    event_type: str
+    room_id: str
+    state_key: str
+    via: Sequence[str]
 
     # the raw data, including the above keys
-    data = attr.ib(type=JsonDict)
+    data: JsonDict
 
     @classmethod
     def from_json_dict(cls, d: JsonDict) -> "FederationSpaceSummaryEventResult":
@@ -1210,6 +1451,10 @@ def from_json_dict(cls, d: JsonDict) -> "FederationSpaceSummaryEventResult":
         if not isinstance(event_type, str):
             raise ValueError("Invalid event: 'event_type' must be a str")
 
+        room_id = d.get("room_id")
+        if not isinstance(room_id, str):
+            raise ValueError("Invalid event: 'room_id' must be a str")
+
         state_key = d.get("state_key")
         if not isinstance(state_key, str):
             raise ValueError("Invalid event: 'state_key' must be a str")
@@ -1224,15 +1469,15 @@ def from_json_dict(cls, d: JsonDict) -> "FederationSpaceSummaryEventResult":
         if any(not isinstance(v, str) for v in via):
             raise ValueError("Invalid event: 'via' must be a list of strings")
 
-        return cls(event_type, state_key, via, d)
+        return cls(event_type, room_id, state_key, via, d)
 
 
-@attr.s(frozen=True, slots=True)
+@attr.s(frozen=True, slots=True, auto_attribs=True)
 class FederationSpaceSummaryResult:
     """Represents the data returned by a successful get_space_summary call."""
 
-    rooms = attr.ib(type=Sequence[JsonDict])
-    events = attr.ib(type=Sequence[FederationSpaceSummaryEventResult])
+    rooms: List[JsonDict]
+    events: Sequence[FederationSpaceSummaryEventResult]
 
     @classmethod
     def from_json_dict(cls, d: JsonDict) -> "FederationSpaceSummaryResult":
@@ -1245,7 +1490,7 @@ def from_json_dict(cls, d: JsonDict) -> "FederationSpaceSummaryResult":
             ValueError if d is not a valid /spaces/ response
         """
         rooms = d.get("rooms")
-        if not isinstance(rooms, Sequence):
+        if not isinstance(rooms, List):
             raise ValueError("'rooms' must be a list")
         if any(not isinstance(r, dict) for r in rooms):
             raise ValueError("Invalid room in 'rooms' list")
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index cb48cc5722..afd8f8580a 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 # Copyright 2019 Matrix.org Federation C.I.C
@@ -35,7 +34,7 @@
 from twisted.internet.abstract import isIPAddress
 from twisted.python import failure
 
-from synapse.api.constants import EduTypes, EventTypes
+from synapse.api.constants import EduTypes, EventTypes, Membership
 from synapse.api.errors import (
     AuthError,
     Codes,
@@ -45,10 +44,10 @@
     SynapseError,
     UnsupportedRoomVersionError,
 )
-from synapse.api.ratelimiting import Ratelimiter
-from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
-from synapse.config.api import DEFAULT_ROOM_STATE_TYPES
+from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion
+from synapse.crypto.event_signing import compute_event_signature
 from synapse.events import EventBase
+from synapse.events.snapshot import EventContext
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.federation.persistence import TransactionActions
 from synapse.federation.units import Edu, Transaction
@@ -60,11 +59,13 @@
 )
 from synapse.logging.opentracing import log_kv, start_active_span_from_edu, trace
 from synapse.logging.utils import log_function
+from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.replication.http.federation import (
     ReplicationFederationSendEduRestServlet,
     ReplicationGetQueryRestServlet,
 )
-from synapse.types import JsonDict
+from synapse.storage.databases.main.lock import Lock
+from synapse.types import JsonDict, get_domain_from_id
 from synapse.util import glob_to_regex, json_decoder, unwrapFirstError
 from synapse.util.async_helpers import Linearizer, concurrently_execute
 from synapse.util.caches.response_cache import ResponseCache
@@ -99,13 +100,18 @@
 )
 
 
+# The name of the lock to use when process events in a room received over
+# federation.
+_INBOUND_EVENT_HANDLING_LOCK_NAME = "federation_inbound_pdu"
+
+
 class FederationServer(FederationBase):
     def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
-        self.auth = hs.get_auth()
         self.handler = hs.get_federation_handler()
         self.state = hs.get_state_handler()
+        self._event_auth_handler = hs.get_event_auth_handler()
 
         self.device_handler = hs.get_device_handler()
 
@@ -117,12 +123,12 @@ def __init__(self, hs: "HomeServer"):
 
         # origins that we are currently processing a transaction from.
         # a dict from origin to txn id.
-        self._active_transactions = {}  # type: Dict[str, str]
+        self._active_transactions: Dict[str, str] = {}
 
         # We cache results for transaction with the same ID
-        self._transaction_resp_cache = ResponseCache(
+        self._transaction_resp_cache: ResponseCache[Tuple[str, str]] = ResponseCache(
             hs.get_clock(), "fed_txn_handler", timeout_ms=30000
-        )  # type: ResponseCache[Tuple[str, str]]
+        )
 
         self.transaction_actions = TransactionActions(self.store)
 
@@ -130,17 +136,54 @@ def __init__(self, hs: "HomeServer"):
 
         # We cache responses to state queries, as they take a while and often
         # come in waves.
-        self._state_resp_cache = ResponseCache(
-            hs.get_clock(), "state_resp", timeout_ms=30000
-        )  # type: ResponseCache[Tuple[str, str]]
-        self._state_ids_resp_cache = ResponseCache(
+        self._state_resp_cache: ResponseCache[
+            Tuple[str, Optional[str]]
+        ] = ResponseCache(hs.get_clock(), "state_resp", timeout_ms=30000)
+        self._state_ids_resp_cache: ResponseCache[Tuple[str, str]] = ResponseCache(
             hs.get_clock(), "state_ids_resp", timeout_ms=30000
-        )  # type: ResponseCache[Tuple[str, str]]
+        )
 
         self._federation_metrics_domains = (
-            hs.get_config().federation.federation_metrics_domains
+            hs.config.federation.federation_metrics_domains
         )
 
+        self._room_prejoin_state_types = hs.config.api.room_prejoin_state
+
+        # Whether we have started handling old events in the staging area.
+        self._started_handling_of_staged_events = False
+
+    @wrap_as_background_process("_handle_old_staged_events")
+    async def _handle_old_staged_events(self) -> None:
+        """Handle old staged events by fetching all rooms that have staged
+        events and start the processing of each of those rooms.
+        """
+
+        # Get all the rooms IDs with staged events.
+        room_ids = await self.store.get_all_rooms_with_staged_incoming_events()
+
+        # We then shuffle them so that if there are multiple instances doing
+        # this work they're less likely to collide.
+        random.shuffle(room_ids)
+
+        for room_id in room_ids:
+            room_version = await self.store.get_room_version(room_id)
+
+            # Try and acquire the processing lock for the room, if we get it start a
+            # background process for handling the events in the room.
+            lock = await self.store.try_acquire_lock(
+                _INBOUND_EVENT_HANDLING_LOCK_NAME, room_id
+            )
+            if lock:
+                logger.info("Handling old staged inbound events in %s", room_id)
+                self._process_incoming_pdus_in_room_inner(
+                    room_id,
+                    room_version,
+                    lock,
+                )
+
+            # We pause a bit so that we don't start handling all rooms at once.
+            await self._clock.sleep(random.uniform(0, 0.1))
+
     async def on_backfill_request(
         self, origin: str, room_id: str, versions: List[str], limit: int
     ) -> Tuple[int, Dict[str, Any]]:
@@ -152,19 +195,35 @@ async def on_backfill_request(
                 origin, room_id, versions, limit
             )
 
-            res = self._transaction_from_pdus(pdus).get_dict()
+            res = self._transaction_dict_from_pdus(pdus)
 
         return 200, res
 
     async def on_incoming_transaction(
-        self, origin: str, transaction_data: JsonDict
-    ) -> Tuple[int, Dict[str, Any]]:
+        self,
+        origin: str,
+        transaction_id: str,
+        destination: str,
+        transaction_data: JsonDict,
+    ) -> Tuple[int, JsonDict]:
+        # If we receive a transaction we should make sure that kick off handling
+        # any old events in the staging area.
+        if not self._started_handling_of_staged_events:
+            self._started_handling_of_staged_events = True
+            self._handle_old_staged_events()
+
         # keep this as early as possible to make the calculated origin ts as
         # accurate as possible.
         request_time = self._clock.time_msec()
 
-        transaction = Transaction(**transaction_data)
-        transaction_id = transaction.transaction_id  # type: ignore
+        transaction = Transaction(
+            transaction_id=transaction_id,
+            destination=destination,
+            origin=origin,
+            origin_server_ts=transaction_data.get("origin_server_ts"),  # type: ignore
+            pdus=transaction_data.get("pdus"),  # type: ignore
+            edus=transaction_data.get("edus"),
+        )
 
         if not transaction_id:
             raise Exception("Transaction missing transaction_id")
@@ -172,9 +231,7 @@ async def on_incoming_transaction(
         logger.debug("[%s] Got transaction", transaction_id)
 
         # Reject malformed transactions early: reject if too many PDUs/EDUs
-        if len(transaction.pdus) > 50 or (  # type: ignore
-            hasattr(transaction, "edus") and len(transaction.edus) > 100  # type: ignore
-        ):
+        if len(transaction.pdus) > 50 or len(transaction.edus) > 100:
             logger.info("Transaction PDU or EDU count too large. Returning 400")
             return 400, {}
 
@@ -214,7 +271,7 @@ async def _on_incoming_transaction_inner(
         # CRITICAL SECTION: the first thing we must do (before awaiting) is
         # add an entry to _active_transactions.
         assert origin not in self._active_transactions
-        self._active_transactions[origin] = transaction.transaction_id  # type: ignore
+        self._active_transactions[origin] = transaction.transaction_id
 
         try:
             result = await self._handle_incoming_transaction(
@@ -242,11 +299,11 @@ async def _handle_incoming_transaction(
         if response:
             logger.debug(
                 "[%s] We've already responded to this request",
-                transaction.transaction_id,  # type: ignore
+                transaction.transaction_id,
             )
             return response
 
-        logger.debug("[%s] Transaction is new", transaction.transaction_id)  # type: ignore
+        logger.debug("[%s] Transaction is new", transaction.transaction_id)
 
         # We process PDUs and EDUs in parallel. This is important as we don't
         # want to block things like to device messages from reaching clients
@@ -285,15 +342,15 @@ async def _handle_pdus_in_txn(
             report back to the sending server.
         """
 
-        received_pdus_counter.inc(len(transaction.pdus))  # type: ignore
+        received_pdus_counter.inc(len(transaction.pdus))
 
         origin_host, _ = parse_server_name(origin)
 
-        pdus_by_room = {}  # type: Dict[str, List[EventBase]]
+        pdus_by_room: Dict[str, List[EventBase]] = {}
 
         newest_pdu_ts = 0
 
-        for p in transaction.pdus:  # type: ignore
+        for p in transaction.pdus:
             # FIXME (richardv): I don't think this works:
             #  https://github.com/matrix-org/synapse/issues/8429
             if "unsigned" in p:
@@ -362,22 +419,21 @@ async def process_pdus_for_room(room_id: str):
 
         async def process_pdu(pdu: EventBase) -> JsonDict:
             event_id = pdu.event_id
-            with pdu_process_time.time():
-                with nested_logging_context(event_id):
-                    try:
-                        await self._handle_received_pdu(origin, pdu)
-                        return {}
-                    except FederationError as e:
-                        logger.warning("Error handling PDU %s: %s", event_id, e)
-                        return {"error": str(e)}
-                    except Exception as e:
-                        f = failure.Failure()
-                        logger.error(
-                            "Failed to handle PDU %s",
-                            event_id,
-                            exc_info=(f.type, f.value, f.getTracebackObject()),  # type: ignore
-                        )
-                        return {"error": str(e)}
+            with nested_logging_context(event_id):
+                try:
+                    await self._handle_received_pdu(origin, pdu)
+                    return {}
+                except FederationError as e:
+                    logger.warning("Error handling PDU %s: %s", event_id, e)
+                    return {"error": str(e)}
+                except Exception as e:
+                    f = failure.Failure()
+                    logger.error(
+                        "Failed to handle PDU %s",
+                        event_id,
+                        exc_info=(f.type, f.value, f.getTracebackObject()),  # type: ignore
+                    )
+                    return {"error": str(e)}
 
         await concurrently_execute(
             process_pdus_for_room, pdus_by_room.keys(), TRANSACTION_CONCURRENCY_LIMIT
@@ -388,10 +444,10 @@ async def process_pdu(pdu: EventBase) -> JsonDict:
 
         return pdu_results
 
-    async def _handle_edus_in_txn(self, origin: str, transaction: Transaction):
+    async def _handle_edus_in_txn(self, origin: str, transaction: Transaction) -> None:
         """Process the EDUs in a received transaction."""
 
-        async def _process_edu(edu_dict):
+        async def _process_edu(edu_dict: JsonDict) -> None:
             received_edus_counter.inc()
 
             edu = Edu(
@@ -404,17 +460,17 @@ async def _process_edu(edu_dict):
 
         await concurrently_execute(
             _process_edu,
-            getattr(transaction, "edus", []),
+            transaction.edus,
             TRANSACTION_CONCURRENCY_LIMIT,
         )
 
     async def on_room_state_request(
-        self, origin: str, room_id: str, event_id: str
+        self, origin: str, room_id: str, event_id: Optional[str]
     ) -> Tuple[int, Dict[str, Any]]:
         origin_host, _ = parse_server_name(origin)
         await self.check_server_matches_acl(origin_host, room_id)
 
-        in_room = await self.auth.check_host_in_room(room_id, origin)
+        in_room = await self._event_auth_handler.check_host_in_room(room_id, origin)
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
@@ -447,7 +503,7 @@ async def on_state_ids_request(
         origin_host, _ = parse_server_name(origin)
         await self.check_server_matches_acl(origin_host, room_id)
 
-        in_room = await self.auth.check_host_in_room(room_id, origin)
+        in_room = await self._event_auth_handler.check_host_in_room(room_id, origin)
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
@@ -466,12 +522,12 @@ async def _on_state_ids_request_compute(self, room_id, event_id):
         return {"pdu_ids": state_ids, "auth_chain_ids": auth_chain_ids}
 
     async def _on_context_state_request_compute(
-        self, room_id: str, event_id: str
+        self, room_id: str, event_id: Optional[str]
     ) -> Dict[str, list]:
         if event_id:
-            pdus = await self.handler.get_state_for_pdu(
+            pdus: Iterable[EventBase] = await self.handler.get_state_for_pdu(
                 room_id, event_id
-            )  # type: Iterable[EventBase]
+            )
         else:
             pdus = (await self.state.get_current_state(room_id)).values()
 
@@ -490,7 +546,7 @@ async def on_pdu_request(
         pdu = await self.handler.get_persisted_pdu(origin, event_id)
 
         if pdu:
-            return 200, self._transaction_from_pdus([pdu]).get_dict()
+            return 200, self._transaction_dict_from_pdus([pdu])
         else:
             return 404, ""
 
@@ -515,8 +571,7 @@ async def on_make_join_request(
             raise IncompatibleRoomVersionError(room_version=room_version)
 
         pdu = await self.handler.on_make_join_request(origin, room_id, user_id)
-        time_now = self._clock.time_msec()
-        return {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
+        return {"event": pdu.get_templated_pdu_json(), "room_version": room_version}
 
     async def on_invite_request(
         self, origin: str, content: JsonDict, room_version_id: str
@@ -538,26 +593,22 @@ async def on_invite_request(
         return {"event": ret_pdu.get_pdu_json(time_now)}
 
     async def on_send_join_request(
-        self, origin: str, content: JsonDict
+        self, origin: str, content: JsonDict, room_id: str
     ) -> Dict[str, Any]:
-        logger.debug("on_send_join_request: content: %s", content)
-
-        assert_params_in_dict(content, ["room_id"])
-        room_version = await self.store.get_room_version(content["room_id"])
-        pdu = event_from_pdu_json(content, room_version)
-
-        origin_host, _ = parse_server_name(origin)
-        await self.check_server_matches_acl(origin_host, pdu.room_id)
-
-        logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures)
+        event, context = await self._on_send_membership_event(
+            origin, content, Membership.JOIN, room_id
+        )
 
-        pdu = await self._check_sigs_and_hash(room_version, pdu)
+        prev_state_ids = await context.get_prev_state_ids()
+        state_ids = list(prev_state_ids.values())
+        auth_chain = await self.store.get_auth_chain(room_id, state_ids)
+        state = await self.store.get_events(state_ids)
 
-        res_pdus = await self.handler.on_send_join_request(origin, pdu)
         time_now = self._clock.time_msec()
         return {
-            "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]],
-            "auth_chain": [p.get_pdu_json(time_now) for p in res_pdus["auth_chain"]],
+            "org.matrix.msc3083.v2.event": event.get_pdu_json(),
+            "state": [p.get_pdu_json(time_now) for p in state.values()],
+            "auth_chain": [p.get_pdu_json(time_now) for p in auth_chain],
         }
 
     async def on_make_leave_request(
@@ -569,24 +620,13 @@ async def on_make_leave_request(
 
         room_version = await self.store.get_room_version_id(room_id)
 
-        time_now = self._clock.time_msec()
-        return {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
+        return {"event": pdu.get_templated_pdu_json(), "room_version": room_version}
 
-    async def on_send_leave_request(self, origin: str, content: JsonDict) -> dict:
+    async def on_send_leave_request(
+        self, origin: str, content: JsonDict, room_id: str
+    ) -> dict:
         logger.debug("on_send_leave_request: content: %s", content)
-
-        assert_params_in_dict(content, ["room_id"])
-        room_version = await self.store.get_room_version(content["room_id"])
-        pdu = event_from_pdu_json(content, room_version)
-
-        origin_host, _ = parse_server_name(origin)
-        await self.check_server_matches_acl(origin_host, pdu.room_id)
-
-        logger.debug("on_send_leave_request: pdu sigs: %s", pdu.signatures)
-
-        pdu = await self._check_sigs_and_hash(room_version, pdu)
-
-        await self.handler.on_send_leave_request(origin, pdu)
+        await self._on_send_membership_event(origin, content, Membership.LEAVE, room_id)
         return {}
 
     async def on_make_knock_request(
@@ -609,16 +649,28 @@ async def on_make_knock_request(
         origin_host, _ = parse_server_name(origin)
         await self.check_server_matches_acl(origin_host, room_id)
 
-        room_version = await self.store.get_room_version_id(room_id)
-        if room_version not in supported_versions:
+        room_version = await self.store.get_room_version(room_id)
+
+        # Check that this room version is supported by the remote homeserver
+        if room_version.identifier not in supported_versions:
             logger.warning(
-                "Room version %s not in %s", room_version, supported_versions
+                "Room version %s not in %s", room_version.identifier, supported_versions
+            )
+            raise IncompatibleRoomVersionError(room_version=room_version.identifier)
+
+        # Check that this room supports knocking as defined by its room version
+        if not room_version.msc2403_knocking:
+            raise SynapseError(
+                403,
+                "This room version does not support knocking",
+                errcode=Codes.FORBIDDEN,
             )
-            raise IncompatibleRoomVersionError(room_version=room_version)
 
         pdu = await self.handler.on_make_knock_request(origin, room_id, user_id)
-        time_now = self._clock.time_msec()
-        return {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
+        return {
+            "event": pdu.get_templated_pdu_json(),
+            "room_version": room_version.identifier,
+        }
 
     async def on_send_knock_request(
         self,
@@ -639,31 +691,104 @@ async def on_send_knock_request(
         Returns:
             The stripped room state.
         """
-        logger.debug("on_send_knock_request: content: %s", content)
-
-        room_version = await self.store.get_room_version(room_id)
-        pdu = event_from_pdu_json(content, room_version)
-
-        origin_host, _ = parse_server_name(origin)
-        await self.check_server_matches_acl(origin_host, pdu.room_id)
-
-        logger.debug("on_send_knock_request: pdu sigs: %s", pdu.signatures)
-
-        pdu = await self._check_sigs_and_hash(room_version, pdu)
-
-        # Handle the event, and retrieve the EventContext
-        event_context = await self.handler.on_send_knock_request(origin, pdu)
+        _, context = await self._on_send_membership_event(
+            origin, content, Membership.KNOCK, room_id
+        )
 
         # Retrieve stripped state events from the room and send them back to the remote
         # server. This will allow the remote server's clients to display information
         # related to the room while the knock request is pending.
         stripped_room_state = (
             await self.store.get_stripped_room_state_from_event_context(
-                event_context, DEFAULT_ROOM_STATE_TYPES
+                context, self._room_prejoin_state_types
             )
         )
         return {"knock_state_events": stripped_room_state}
 
+    async def _on_send_membership_event(
+        self, origin: str, content: JsonDict, membership_type: str, room_id: str
+    ) -> Tuple[EventBase, EventContext]:
+        """Handle an on_send_{join,leave,knock} request
+
+        Does some preliminary validation before passing the request on to the
+        federation handler.
+
+        Args:
+            origin: The (authenticated) requesting server
+            content: The body of the send_* request - a complete membership event
+            membership_type: The expected membership type (join or leave, depending
+                on the endpoint)
+            room_id: The room_id from the request, to be validated against the room_id
+                in the event
+
+        Returns:
+            The event and context of the event after inserting it into the room graph.
+
+        Raises:
+            SynapseError if there is a problem with the request, including things like
+               the room_id not matching or the event not being authorized.
+        """
+        assert_params_in_dict(content, ["room_id"])
+        if content["room_id"] != room_id:
+            raise SynapseError(
+                400,
+                "Room ID in body does not match that in request path",
+                Codes.BAD_JSON,
+            )
+
+        room_version = await self.store.get_room_version(room_id)
+
+        if membership_type == Membership.KNOCK and not room_version.msc2403_knocking:
+            raise SynapseError(
+                403,
+                "This room version does not support knocking",
+                errcode=Codes.FORBIDDEN,
+            )
+
+        event = event_from_pdu_json(content, room_version)
+
+        if event.type != EventTypes.Member or not event.is_state():
+            raise SynapseError(400, "Not an m.room.member event", Codes.BAD_JSON)
+
+        if event.content.get("membership") != membership_type:
+            raise SynapseError(400, "Not a %s event" % membership_type, Codes.BAD_JSON)
+
+        origin_host, _ = parse_server_name(origin)
+        await self.check_server_matches_acl(origin_host, event.room_id)
+
+        logger.debug("_on_send_membership_event: pdu sigs: %s", event.signatures)
+
+        # Sign the event since we're vouching on behalf of the remote server that
+        # the event is valid to be sent into the room. Currently this is only done
+        # if the user is being joined via restricted join rules.
+        if (
+            room_version.msc3083_join_rules
+            and event.membership == Membership.JOIN
+            and "join_authorised_via_users_server" in event.content
+        ):
+            # We can only authorise our own users.
+            authorising_server = get_domain_from_id(
+                event.content["join_authorised_via_users_server"]
+            )
+            if authorising_server != self.server_name:
+                raise SynapseError(
+                    400,
+                    f"Cannot authorise request from resident server: {authorising_server}",
+                )
+
+            event.signatures.update(
+                compute_event_signature(
+                    room_version,
+                    event.get_pdu_json(),
+                    self.hs.hostname,
+                    self.hs.signing_key,
+                )
+            )
+
+        event = await self._check_sigs_and_hash(room_version, event)
+
+        return await self.handler.on_send_membership_event(origin, event)
+
     async def on_event_auth(
         self, origin: str, room_id: str, event_id: str
     ) -> Tuple[int, Dict[str, Any]]:
@@ -700,7 +825,7 @@ async def on_claim_client_keys(
         log_kv({"message": "Claiming one time keys.", "user, device pairs": query})
         results = await self.store.claim_e2e_one_time_keys(query)
 
-        json_result = {}  # type: Dict[str, Dict[str, dict]]
+        json_result: Dict[str, Dict[str, dict]] = {}
         for user_id, device_keys in results.items():
             for device_id, keys in device_keys.items():
                 for key_id, json_str in keys.items():
@@ -762,18 +887,20 @@ async def on_openid_userinfo(self, token: str) -> Optional[str]:
         ts_now_ms = self._clock.time_msec()
         return await self.store.get_user_id_for_open_id_token(token, ts_now_ms)
 
-    def _transaction_from_pdus(self, pdu_list: List[EventBase]) -> Transaction:
+    def _transaction_dict_from_pdus(self, pdu_list: List[EventBase]) -> JsonDict:
         """Returns a new Transaction containing the given PDUs suitable for
         transmission.
         """
         time_now = self._clock.time_msec()
         pdus = [p.get_pdu_json(time_now) for p in pdu_list]
         return Transaction(
+            # Just need a dummy transaction ID and destination since it won't be used.
+            transaction_id="",
             origin=self.server_name,
             pdus=pdus,
             origin_server_ts=int(time_now),
-            destination=None,
-        )
+            destination="",
+        ).get_dict()
 
     async def _handle_received_pdu(self, origin: str, pdu: EventBase) -> None:
         """Process a PDU received in a federation /send/ transaction.
@@ -813,24 +940,143 @@ async def _handle_received_pdu(self, origin: str, pdu: EventBase) -> None:
         except SynapseError as e:
             raise FederationError("ERROR", e.code, e.msg, affected=pdu.event_id)
 
-        await self.handler.on_receive_pdu(origin, pdu, sent_to_us_directly=True)
+        # Add the event to our staging area
+        await self.store.insert_received_event_to_staging(origin, pdu)
+
+        # Try and acquire the processing lock for the room, if we get it start a
+        # background process for handling the events in the room.
+        lock = await self.store.try_acquire_lock(
+            _INBOUND_EVENT_HANDLING_LOCK_NAME, pdu.room_id
+        )
+        if lock:
+            self._process_incoming_pdus_in_room_inner(
+                pdu.room_id, room_version, lock, origin, pdu
+            )
+
+    @wrap_as_background_process("_process_incoming_pdus_in_room_inner")
+    async def _process_incoming_pdus_in_room_inner(
+        self,
+        room_id: str,
+        room_version: RoomVersion,
+        lock: Lock,
+        latest_origin: Optional[str] = None,
+        latest_event: Optional[EventBase] = None,
+    ) -> None:
+        """Process events in the staging area for the given room.
+
+        The latest_origin and latest_event args are the latest origin and event
+        received (or None to simply pull the next event from the database).
+        """
+
+        # The common path is for the event we just received be the only event in
+        # the room, so instead of pulling the event out of the DB and parsing
+        # the event we just pull out the next event ID and check if that matches.
+        if latest_event is not None and latest_origin is not None:
+            result = await self.store.get_next_staged_event_id_for_room(room_id)
+            if result is None:
+                latest_origin = None
+                latest_event = None
+            else:
+                next_origin, next_event_id = result
+                if (
+                    next_origin != latest_origin
+                    or next_event_id != latest_event.event_id
+                ):
+                    latest_origin = None
+                    latest_event = None
+
+        if latest_origin is None or latest_event is None:
+            next = await self.store.get_next_staged_event_for_room(
+                room_id, room_version
+            )
+            if not next:
+                await lock.release()
+                return
+
+            origin, event = next
+        else:
+            origin = latest_origin
+            event = latest_event
+
+        # We loop round until there are no more events in the room in the
+        # staging area, or we fail to get the lock (which means another process
+        # has started processing).
+        while True:
+            async with lock:
+                logger.info("handling received PDU: %s", event)
+                try:
+                    await self.handler.on_receive_pdu(
+                        origin, event, sent_to_us_directly=True
+                    )
+                except FederationError as e:
+                    # XXX: Ideally we'd inform the remote we failed to process
+                    # the event, but we can't return an error in the transaction
+                    # response (as we've already responded).
+                    logger.warning("Error handling PDU %s: %s", event.event_id, e)
+                except Exception:
+                    f = failure.Failure()
+                    logger.error(
+                        "Failed to handle PDU %s",
+                        event.event_id,
+                        exc_info=(f.type, f.value, f.getTracebackObject()),  # type: ignore
+                    )
+
+                received_ts = await self.store.remove_received_event_from_staging(
+                    origin, event.event_id
+                )
+                if received_ts is not None:
+                    pdu_process_time.observe(
+                        (self._clock.time_msec() - received_ts) / 1000
+                    )
+
+            # We need to do this check outside the lock to avoid a race between
+            # a new event being inserted by another instance and it attempting
+            # to acquire the lock.
+            next = await self.store.get_next_staged_event_for_room(
+                room_id, room_version
+            )
+            if not next:
+                break
+
+            origin, event = next
+
+            # Prune the event queue if it's getting large.
+            #
+            # We do this *after* handling the first event as the common case is
+            # that the queue is empty (/has the single event in), and so there's
+            # no need to do this check.
+            pruned = await self.store.prune_staged_events_in_room(room_id, room_version)
+            if pruned:
+                # If we have pruned the queue check we need to refetch the next
+                # event to handle.
+                next = await self.store.get_next_staged_event_for_room(
+                    room_id, room_version
+                )
+                if not next:
+                    break
+
+                origin, event = next
 
-    def __str__(self):
+            lock = await self.store.try_acquire_lock(
+                _INBOUND_EVENT_HANDLING_LOCK_NAME, room_id
+            )
+            if not lock:
+                return
+
+    def __str__(self) -> str:
         return "<ReplicationLayer(%s)>" % self.server_name
 
     async def exchange_third_party_invite(
         self, sender_user_id: str, target_user_id: str, room_id: str, signed: Dict
-    ):
-        ret = await self.handler.exchange_third_party_invite(
+    ) -> None:
+        await self.handler.exchange_third_party_invite(
             sender_user_id, target_user_id, room_id, signed
         )
-        return ret
 
-    async def on_exchange_third_party_invite_request(self, event_dict: Dict):
-        ret = await self.handler.on_exchange_third_party_invite_request(event_dict)
-        return ret
+    async def on_exchange_third_party_invite_request(self, event_dict: Dict) -> None:
+        await self.handler.on_exchange_third_party_invite_request(event_dict)
 
-    async def check_server_matches_acl(self, server_name: str, room_id: str):
+    async def check_server_matches_acl(self, server_name: str, room_id: str) -> None:
         """Check if the given server is allowed by the server ACLs in the room
 
         Args:
@@ -932,28 +1178,17 @@ def __init__(self, hs: "HomeServer"):
         self._get_query_client = ReplicationGetQueryRestServlet.make_client(hs)
         self._send_edu = ReplicationFederationSendEduRestServlet.make_client(hs)
 
-        self.edu_handlers = (
-            {}
-        )  # type: Dict[str, Callable[[str, dict], Awaitable[None]]]
-        self.query_handlers = (
-            {}
-        )  # type: Dict[str, Callable[[dict], Awaitable[JsonDict]]]
+        self.edu_handlers: Dict[str, Callable[[str, dict], Awaitable[None]]] = {}
+        self.query_handlers: Dict[str, Callable[[dict], Awaitable[JsonDict]]] = {}
 
         # Map from type to instance names that we should route EDU handling to.
         # We randomly choose one instance from the list to route to for each new
         # EDU received.
-        self._edu_type_to_instance = {}  # type: Dict[str, List[str]]
-
-        # A rate limiter for incoming room key requests per origin.
-        self._room_key_request_rate_limiter = Ratelimiter(
-            clock=self.clock,
-            rate_hz=self.config.rc_key_requests.per_second,
-            burst_count=self.config.rc_key_requests.burst_count,
-        )
+        self._edu_type_to_instance: Dict[str, List[str]] = {}
 
     def register_edu_handler(
         self, edu_type: str, handler: Callable[[str, JsonDict], Awaitable[None]]
-    ):
+    ) -> None:
         """Sets the handler callable that will be used to handle an incoming
         federation EDU of the given type.
 
@@ -972,7 +1207,7 @@ def register_edu_handler(
 
     def register_query_handler(
         self, query_type: str, handler: Callable[[dict], Awaitable[JsonDict]]
-    ):
+    ) -> None:
         """Sets the handler callable that will be used to handle an incoming
         federation query of the given type.
 
@@ -990,26 +1225,20 @@ def register_query_handler(
 
         self.query_handlers[query_type] = handler
 
-    def register_instance_for_edu(self, edu_type: str, instance_name: str):
+    def register_instance_for_edu(self, edu_type: str, instance_name: str) -> None:
         """Register that the EDU handler is on a different instance than master."""
         self._edu_type_to_instance[edu_type] = [instance_name]
 
-    def register_instances_for_edu(self, edu_type: str, instance_names: List[str]):
+    def register_instances_for_edu(
+        self, edu_type: str, instance_names: List[str]
+    ) -> None:
         """Register that the EDU handler is on multiple instances."""
         self._edu_type_to_instance[edu_type] = instance_names
 
-    async def on_edu(self, edu_type: str, origin: str, content: dict):
+    async def on_edu(self, edu_type: str, origin: str, content: dict) -> None:
         if not self.config.use_presence and edu_type == EduTypes.Presence:
             return
 
-        # If the incoming room key requests from a particular origin are over
-        # the limit, drop them.
-        if (
-            edu_type == EduTypes.RoomKeyRequest
-            and not self._room_key_request_rate_limiter.can_do_action(origin)
-        ):
-            return
-
         # Check if we have a handler on this instance
         handler = self.edu_handlers.get(edu_type)
         if handler:
diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py
index ce5fc758f0..4fead6ca29 100644
--- a/synapse/federation/persistence.py
+++ b/synapse/federation/persistence.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -46,7 +45,7 @@ async def have_responded(
             `None` if we have not previously responded to this transaction or a
             2-tuple of `(int, dict)` representing the response code and response body.
         """
-        transaction_id = transaction.transaction_id  # type: ignore
+        transaction_id = transaction.transaction_id
         if not transaction_id:
             raise RuntimeError("Cannot persist a transaction with no transaction_id")
 
@@ -57,7 +56,7 @@ async def set_response(
         self, origin: str, transaction: Transaction, code: int, response: JsonDict
     ) -> None:
         """Persist how we responded to a transaction."""
-        transaction_id = transaction.transaction_id  # type: ignore
+        transaction_id = transaction.transaction_id
         if not transaction_id:
             raise RuntimeError("Cannot persist a transaction with no transaction_id")
 
diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py
index 0c18c49abb..1fbf325fdc 100644
--- a/synapse/federation/send_queue.py
+++ b/synapse/federation/send_queue.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -72,37 +71,32 @@ def __init__(self, hs: "HomeServer"):
         # We may have multiple federation sender instances, so we need to track
         # their positions separately.
         self._sender_instances = hs.config.worker.federation_shard_config.instances
-        self._sender_positions = {}  # type: Dict[str, int]
+        self._sender_positions: Dict[str, int] = {}
 
         # Pending presence map user_id -> UserPresenceState
-        self.presence_map = {}  # type: Dict[str, UserPresenceState]
-
-        # Stream position -> list[user_id]
-        self.presence_changed = SortedDict()  # type: SortedDict[int, List[str]]
+        self.presence_map: Dict[str, UserPresenceState] = {}
 
         # Stores the destinations we need to explicitly send presence to about a
         # given user.
         # Stream position -> (user_id, destinations)
-        self.presence_destinations = (
-            SortedDict()
-        )  # type: SortedDict[int, Tuple[str, Iterable[str]]]
+        self.presence_destinations: SortedDict[
+            int, Tuple[str, Iterable[str]]
+        ] = SortedDict()
 
         # (destination, key) -> EDU
-        self.keyed_edu = {}  # type: Dict[Tuple[str, tuple], Edu]
+        self.keyed_edu: Dict[Tuple[str, tuple], Edu] = {}
 
         # stream position -> (destination, key)
-        self.keyed_edu_changed = (
-            SortedDict()
-        )  # type: SortedDict[int, Tuple[str, tuple]]
+        self.keyed_edu_changed: SortedDict[int, Tuple[str, tuple]] = SortedDict()
 
-        self.edus = SortedDict()  # type: SortedDict[int, Edu]
+        self.edus: SortedDict[int, Edu] = SortedDict()
 
-        # stream ID for the next entry into presence_changed/keyed_edu_changed/edus.
+        # stream ID for the next entry into keyed_edu_changed/edus.
         self.pos = 1
 
         # map from stream ID to the time that stream entry was generated, so that we
         # can clear out entries after a while
-        self.pos_time = SortedDict()  # type: SortedDict[int, int]
+        self.pos_time: SortedDict[int, int] = SortedDict()
 
         # EVERYTHING IS SAD. In particular, python only makes new scopes when
         # we make a new function, so we need to make a new function so the inner
@@ -118,7 +112,6 @@ def register(name: str, queue: Sized) -> None:
 
         for queue_name in [
             "presence_map",
-            "presence_changed",
             "keyed_edu",
             "keyed_edu_changed",
             "edus",
@@ -156,23 +149,12 @@ def _clear_queue_before_pos(self, position_to_delete: int) -> None:
         """Clear all the queues from before a given position"""
         with Measure(self.clock, "send_queue._clear"):
             # Delete things out of presence maps
-            keys = self.presence_changed.keys()
-            i = self.presence_changed.bisect_left(position_to_delete)
-            for key in keys[:i]:
-                del self.presence_changed[key]
-
-            user_ids = {
-                user_id for uids in self.presence_changed.values() for user_id in uids
-            }
-
             keys = self.presence_destinations.keys()
             i = self.presence_destinations.bisect_left(position_to_delete)
             for key in keys[:i]:
                 del self.presence_destinations[key]
 
-            user_ids.update(
-                user_id for user_id, _ in self.presence_destinations.values()
-            )
+            user_ids = {user_id for user_id, _ in self.presence_destinations.values()}
 
             to_del = [
                 user_id for user_id in self.presence_map if user_id not in user_ids
@@ -245,23 +227,6 @@ async def send_read_receipt(self, receipt: ReadReceipt) -> None:
         """
         # nothing to do here: the replication listener will handle it.
 
-    def send_presence(self, states: List[UserPresenceState]) -> None:
-        """As per FederationSender
-
-        Args:
-            states
-        """
-        pos = self._next_pos()
-
-        # We only want to send presence for our own users, so lets always just
-        # filter here just in case.
-        local_states = [s for s in states if self.is_mine_id(s.user_id)]
-
-        self.presence_map.update({state.user_id: state for state in local_states})
-        self.presence_changed[pos] = [state.user_id for state in local_states]
-
-        self.notifier.on_new_replication_data()
-
     def send_presence_to_destinations(
         self, states: Iterable[UserPresenceState], destinations: Iterable[str]
     ) -> None:
@@ -324,19 +289,7 @@ async def get_replication_rows(
 
         # list of tuple(int, BaseFederationRow), where the first is the position
         # of the federation stream.
-        rows = []  # type: List[Tuple[int, BaseFederationRow]]
-
-        # Fetch changed presence
-        i = self.presence_changed.bisect_right(from_token)
-        j = self.presence_changed.bisect_right(to_token) + 1
-        dest_user_ids = [
-            (pos, user_id)
-            for pos, user_id_list in self.presence_changed.items()[i:j]
-            for user_id in user_id_list
-        ]
-
-        for (key, user_id) in dest_user_ids:
-            rows.append((key, PresenceRow(state=self.presence_map[user_id])))
+        rows: List[Tuple[int, BaseFederationRow]] = []
 
         # Fetch presence to send to destinations
         i = self.presence_destinations.bisect_right(from_token)
@@ -428,22 +381,6 @@ def add_to_buffer(self, buff):
         raise NotImplementedError()
 
 
-class PresenceRow(
-    BaseFederationRow, namedtuple("PresenceRow", ("state",))  # UserPresenceState
-):
-    TypeId = "p"
-
-    @staticmethod
-    def from_data(data):
-        return PresenceRow(state=UserPresenceState.from_dict(data))
-
-    def to_data(self):
-        return self.state.as_dict()
-
-    def add_to_buffer(self, buff):
-        buff.presence.append(self.state)
-
-
 class PresenceDestinationsRow(
     BaseFederationRow,
     namedtuple(
@@ -506,12 +443,11 @@ def add_to_buffer(self, buff):
         buff.edus.setdefault(self.edu.destination, []).append(self.edu)
 
 
-_rowtypes = (
-    PresenceRow,
+_rowtypes: Tuple[Type[BaseFederationRow], ...] = (
     PresenceDestinationsRow,
     KeyedEduRow,
     EduRow,
-)  # type: Tuple[Type[BaseFederationRow], ...]
+)
 
 TypeToRow = {Row.TypeId: Row for Row in _rowtypes}
 
@@ -519,7 +455,6 @@ def add_to_buffer(self, buff):
 ParsedFederationStreamData = namedtuple(
     "ParsedFederationStreamData",
     (
-        "presence",  # list(UserPresenceState)
         "presence_destinations",  # list of tuples of UserPresenceState and destinations
         "keyed_edus",  # dict of destination -> { key -> Edu }
         "edus",  # dict of destination -> [Edu]
@@ -544,7 +479,6 @@ def process_rows_for_federation(
     # them into the appropriate collection and then send them off.
 
     buff = ParsedFederationStreamData(
-        presence=[],
         presence_destinations=[],
         keyed_edus={},
         edus={},
@@ -560,18 +494,15 @@ def process_rows_for_federation(
         parsed_row = RowType.from_data(row.data)
         parsed_row.add_to_buffer(buff)
 
-    if buff.presence:
-        transaction_queue.send_presence(buff.presence)
-
     for state, destinations in buff.presence_destinations:
         transaction_queue.send_presence_to_destinations(
             states=[state], destinations=destinations
         )
 
-    for destination, edu_map in buff.keyed_edus.items():
+    for edu_map in buff.keyed_edus.values():
         for key, edu in edu_map.items():
             transaction_queue.send_edu(edu, key)
 
-    for destination, edu_list in buff.edus.items():
+    for edu_list in buff.edus.values():
         for edu in edu_list:
             transaction_queue.send_edu(edu, None)
diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py
index 8babb1ebbe..d980e0d986 100644
--- a/synapse/federation/sender/__init__.py
+++ b/synapse/federation/sender/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,9 +14,12 @@
 
 import abc
 import logging
+from collections import OrderedDict
 from typing import TYPE_CHECKING, Dict, Hashable, Iterable, List, Optional, Set, Tuple
 
+import attr
 from prometheus_client import Counter
+from typing_extensions import Literal
 
 from twisted.internet import defer
 
@@ -27,23 +29,23 @@
 from synapse.federation.sender.per_destination_queue import PerDestinationQueue
 from synapse.federation.sender.transaction_manager import TransactionManager
 from synapse.federation.units import Edu
-from synapse.handlers.presence import get_interested_remotes
-from synapse.logging.context import (
-    make_deferred_yieldable,
-    preserve_fn,
-    run_in_background,
-)
+from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.metrics import (
     LaterGauge,
     event_processing_loop_counter,
     event_processing_loop_room_count,
     events_processed_counter,
 )
-from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.metrics.background_process_metrics import (
+    run_as_background_process,
+    wrap_as_background_process,
+)
 from synapse.types import JsonDict, ReadReceipt, RoomStreamToken
-from synapse.util.metrics import Measure, measure_func
+from synapse.util import Clock
+from synapse.util.metrics import Measure
 
 if TYPE_CHECKING:
+    from synapse.events.presence_router import PresenceRouter
     from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
@@ -85,15 +87,6 @@ async def send_read_receipt(self, receipt: ReadReceipt) -> None:
         """
         raise NotImplementedError()
 
-    @abc.abstractmethod
-    def send_presence(self, states: List[UserPresenceState]) -> None:
-        """Send the new presence states to the appropriate destinations.
-
-        This actually queues up the presence states ready for sending and
-        triggers a background task to process them and send out the transactions.
-        """
-        raise NotImplementedError()
-
     @abc.abstractmethod
     def send_presence_to_destinations(
         self, states: Iterable[UserPresenceState], destinations: Iterable[str]
@@ -151,6 +144,84 @@ async def get_replication_rows(
         raise NotImplementedError()
 
 
+@attr.s
+class _PresenceQueue:
+    """A queue of destinations that need to be woken up due to new presence
+    updates.
+
+    Staggers waking up of per destination queues to ensure that we don't attempt
+    to start TLS connections with many hosts all at once, leading to pinned CPU.
+    """
+
+    # The maximum duration in seconds between queuing up a destination and it
+    # being woken up.
+    _MAX_TIME_IN_QUEUE = 30.0
+
+    # The maximum duration in seconds between waking up consecutive destination
+    # queues.
+    _MAX_DELAY = 0.1
+
+    sender: "FederationSender" = attr.ib()
+    clock: Clock = attr.ib()
+    queue: "OrderedDict[str, Literal[None]]" = attr.ib(factory=OrderedDict)
+    processing: bool = attr.ib(default=False)
+
+    def add_to_queue(self, destination: str) -> None:
+        """Add a destination to the queue to be woken up."""
+
+        self.queue[destination] = None
+
+        if not self.processing:
+            self._handle()
+
+    @wrap_as_background_process("_PresenceQueue.handle")
+    async def _handle(self) -> None:
+        """Background process to drain the queue."""
+
+        if not self.queue:
+            return
+
+        assert not self.processing
+        self.processing = True
+
+        try:
+            # We start with a delay that should drain the queue quickly enough that
+            # we process all destinations in the queue in _MAX_TIME_IN_QUEUE
+            # seconds.
+            #
+            # We also add an upper bound to the delay, to gracefully handle the
+            # case where the queue only has a few entries in it.
+            current_sleep_seconds = min(
+                self._MAX_DELAY, self._MAX_TIME_IN_QUEUE / len(self.queue)
+            )
+
+            while self.queue:
+                destination, _ = self.queue.popitem(last=False)
+
+                queue = self.sender._get_per_destination_queue(destination)
+
+                if not queue._new_data_to_send:
+                    # The per destination queue has already been woken up.
+                    continue
+
+                queue.attempt_new_transaction()
+
+                await self.clock.sleep(current_sleep_seconds)
+
+                if not self.queue:
+                    break
+
+                # More destinations may have been added to the queue, so we may
+                # need to reduce the delay to ensure everything gets processed
+                # within _MAX_TIME_IN_QUEUE seconds.
+                current_sleep_seconds = min(
+                    current_sleep_seconds, self._MAX_TIME_IN_QUEUE / len(self.queue)
+                )
+
+        finally:
+            self.processing = False
+
+
 class FederationSender(AbstractFederationSender):
     def __init__(self, hs: "HomeServer"):
         self.hs = hs
@@ -162,13 +233,14 @@ def __init__(self, hs: "HomeServer"):
         self.clock = hs.get_clock()
         self.is_mine_id = hs.is_mine_id
 
+        self._presence_router: Optional["PresenceRouter"] = None
         self._transaction_manager = TransactionManager(hs)
 
         self._instance_name = hs.get_instance_name()
         self._federation_shard_config = hs.config.worker.federation_shard_config
 
         # map from destination to PerDestinationQueue
-        self._per_destination_queues = {}  # type: Dict[str, PerDestinationQueue]
+        self._per_destination_queues: Dict[str, PerDestinationQueue] = {}
 
         LaterGauge(
             "synapse_federation_transaction_queue_pending_destinations",
@@ -181,11 +253,6 @@ def __init__(self, hs: "HomeServer"):
             ),
         )
 
-        # Map of user_id -> UserPresenceState for all the pending presence
-        # to be sent out by user_id. Entries here get processed and put in
-        # pending_presence_by_dest
-        self.pending_presence = {}  # type: Dict[str, UserPresenceState]
-
         LaterGauge(
             "synapse_federation_transaction_queue_pending_pdus",
             "",
@@ -206,15 +273,11 @@ def __init__(self, hs: "HomeServer"):
         self._is_processing = False
         self._last_poked_id = -1
 
-        self._processing_pending_presence = False
-
         # map from room_id to a set of PerDestinationQueues which we believe are
         # awaiting a call to flush_read_receipts_for_room. The presence of an entry
         # here for a given room means that we are rate-limiting RR flushes to that room,
         # and that there is a pending call to _flush_rrs_for_room in the system.
-        self._queues_awaiting_rr_flush_by_room = (
-            {}
-        )  # type: Dict[str, Set[PerDestinationQueue]]
+        self._queues_awaiting_rr_flush_by_room: Dict[str, Set[PerDestinationQueue]] = {}
 
         self._rr_txn_interval_per_room_ms = (
             1000.0 / hs.config.federation_rr_transactions_per_room_per_second
@@ -230,6 +293,8 @@ def __init__(self, hs: "HomeServer"):
 
         self._external_cache = hs.get_external_cache()
 
+        self._presence_queue = _PresenceQueue(self, self.clock)
+
     def _get_per_destination_queue(self, destination: str) -> PerDestinationQueue:
         """Get or create a PerDestinationQueue for the given destination
 
@@ -285,7 +350,7 @@ async def handle_event(event: EventBase) -> None:
                     if not event.internal_metadata.should_proactively_send():
                         return
 
-                    destinations = None  # type: Optional[Set[str]]
+                    destinations: Optional[Set[str]] = None
                     if not event.prev_event_ids():
                         # If there are no prev event IDs then the state is empty
                         # and so no remote servers in the room
@@ -351,7 +416,7 @@ async def handle_room_events(events: Iterable[EventBase]) -> None:
                         for event in events:
                             await handle_event(event)
 
-                events_by_room = {}  # type: Dict[str, List[EventBase]]
+                events_by_room: Dict[str, List[EventBase]] = {}
                 for event in events:
                     events_by_room.setdefault(event.room_id, []).append(event)
 
@@ -517,48 +582,6 @@ def _flush_rrs_for_room(self, room_id: str) -> None:
         for queue in queues:
             queue.flush_read_receipts_for_room(room_id)
 
-    @preserve_fn  # the caller should not yield on this
-    async def send_presence(self, states: List[UserPresenceState]) -> None:
-        """Send the new presence states to the appropriate destinations.
-
-        This actually queues up the presence states ready for sending and
-        triggers a background task to process them and send out the transactions.
-        """
-        if not self.hs.config.use_presence:
-            # No-op if presence is disabled.
-            return
-
-        # First we queue up the new presence by user ID, so multiple presence
-        # updates in quick succession are correctly handled.
-        # We only want to send presence for our own users, so lets always just
-        # filter here just in case.
-        self.pending_presence.update(
-            {state.user_id: state for state in states if self.is_mine_id(state.user_id)}
-        )
-
-        # We then handle the new pending presence in batches, first figuring
-        # out the destinations we need to send each state to and then poking it
-        # to attempt a new transaction. We linearize this so that we don't
-        # accidentally mess up the ordering and send multiple presence updates
-        # in the wrong order
-        if self._processing_pending_presence:
-            return
-
-        self._processing_pending_presence = True
-        try:
-            while True:
-                states_map = self.pending_presence
-                self.pending_presence = {}
-
-                if not states_map:
-                    break
-
-                await self._process_presence_inner(list(states_map.values()))
-        except Exception:
-            logger.exception("Error sending presence states to servers")
-        finally:
-            self._processing_pending_presence = False
-
     def send_presence_to_destinations(
         self, states: Iterable[UserPresenceState], destinations: Iterable[str]
     ) -> None:
@@ -570,6 +593,10 @@ def send_presence_to_destinations(
             # No-op if presence is disabled.
             return
 
+        # Ensure we only send out presence states for local users.
+        for state in states:
+            assert self.is_mine_id(state.user_id)
+
         for destination in destinations:
             if destination == self.server_name:
                 continue
@@ -577,26 +604,12 @@ def send_presence_to_destinations(
                 self._instance_name, destination
             ):
                 continue
-            self._get_per_destination_queue(destination).send_presence(states)
 
-    @measure_func("txnqueue._process_presence")
-    async def _process_presence_inner(self, states: List[UserPresenceState]) -> None:
-        """Given a list of states populate self.pending_presence_by_dest and
-        poke to send a new transaction to each destination
-        """
-        hosts_and_states = await get_interested_remotes(self.store, states, self.state)
-
-        for destinations, states in hosts_and_states:
-            for destination in destinations:
-                if destination == self.server_name:
-                    continue
-
-                if not self._federation_shard_config.should_handle(
-                    self._instance_name, destination
-                ):
-                    continue
+            self._get_per_destination_queue(destination).send_presence(
+                states, start_loop=False
+            )
 
-                self._get_per_destination_queue(destination).send_presence(states)
+            self._presence_queue.add_to_queue(destination)
 
     def build_and_send_edu(
         self,
@@ -705,7 +718,7 @@ async def _wake_destinations_needing_catchup(self) -> None:
         In order to reduce load spikes, adds a delay between each destination.
         """
 
-        last_processed = None  # type: Optional[str]
+        last_processed: Optional[str] = None
 
         while True:
             destinations_to_wake = (
@@ -717,16 +730,18 @@ async def _wake_destinations_needing_catchup(self) -> None:
                 self._catchup_after_startup_timer = None
                 break
 
+            last_processed = destinations_to_wake[-1]
+
             destinations_to_wake = [
                 d
                 for d in destinations_to_wake
                 if self._federation_shard_config.should_handle(self._instance_name, d)
             ]
 
-            for last_processed in destinations_to_wake:
+            for destination in destinations_to_wake:
                 logger.info(
                     "Destination %s has outstanding catch-up, waking up.",
                     last_processed,
                 )
-                self.wake_destination(last_processed)
+                self.wake_destination(destination)
                 await self.clock.sleep(CATCH_UP_STARTUP_INTERVAL_SEC)
diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py
index 89df9a619b..c11d1f6d31 100644
--- a/synapse/federation/sender/per_destination_queue.py
+++ b/synapse/federation/sender/per_destination_queue.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 #
@@ -29,6 +28,8 @@
 from synapse.events import EventBase
 from synapse.federation.units import Edu
 from synapse.handlers.presence import format_user_presence_state
+from synapse.logging import issue9533_logger
+from synapse.logging.opentracing import SynapseTags, set_tag
 from synapse.metrics import sent_transactions_counter
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import ReadReceipt
@@ -104,34 +105,34 @@ def __init__(
         # catch-up at startup.
         # New events will only be sent once this is finished, at which point
         # _catching_up is flipped to False.
-        self._catching_up = True  # type: bool
+        self._catching_up: bool = True
 
         # The stream_ordering of the most recent PDU that was discarded due to
         # being in catch-up mode.
-        self._catchup_last_skipped = 0  # type: int
+        self._catchup_last_skipped: int = 0
 
         # Cache of the last successfully-transmitted stream ordering for this
         # destination (we are the only updater so this is safe)
-        self._last_successful_stream_ordering = None  # type: Optional[int]
+        self._last_successful_stream_ordering: Optional[int] = None
 
         # a queue of pending PDUs
-        self._pending_pdus = []  # type: List[EventBase]
+        self._pending_pdus: List[EventBase] = []
 
         # XXX this is never actually used: see
         # https://github.com/matrix-org/synapse/issues/7549
-        self._pending_edus = []  # type: List[Edu]
+        self._pending_edus: List[Edu] = []
 
         # Pending EDUs by their "key". Keyed EDUs are EDUs that get clobbered
         # based on their key (e.g. typing events by room_id)
         # Map of (edu_type, key) -> Edu
-        self._pending_edus_keyed = {}  # type: Dict[Tuple[str, Hashable], Edu]
+        self._pending_edus_keyed: Dict[Tuple[str, Hashable], Edu] = {}
 
         # Map of user_id -> UserPresenceState of pending presence to be sent to this
         # destination
-        self._pending_presence = {}  # type: Dict[str, UserPresenceState]
+        self._pending_presence: Dict[str, UserPresenceState] = {}
 
         # room_id -> receipt_type -> user_id -> receipt_dict
-        self._pending_rrs = {}  # type: Dict[str, Dict[str, Dict[str, dict]]]
+        self._pending_rrs: Dict[str, Dict[str, Dict[str, dict]]] = {}
         self._rrs_pending_flush = False
 
         # stream_id of last successfully sent to-device message.
@@ -170,14 +171,24 @@ def send_pdu(self, pdu: EventBase) -> None:
 
         self.attempt_new_transaction()
 
-    def send_presence(self, states: Iterable[UserPresenceState]) -> None:
-        """Add presence updates to the queue. Start the transmission loop if necessary.
+    def send_presence(
+        self, states: Iterable[UserPresenceState], start_loop: bool = True
+    ) -> None:
+        """Add presence updates to the queue.
+
+        Args:
+            states: Presence updates to send
+            start_loop: Whether to start the transmission loop if not already
+                running.
 
         Args:
             states: presence to send
         """
         self._pending_presence.update({state.user_id: state for state in states})
-        self.attempt_new_transaction()
+        self._new_data_to_send = True
+
+        if start_loop:
+            self.attempt_new_transaction()
 
     def queue_read_receipt(self, receipt: ReadReceipt) -> None:
         """Add a RR to the list to be sent. Doesn't start the transmission loop yet
@@ -242,7 +253,7 @@ def attempt_new_transaction(self) -> None:
         )
 
     async def _transaction_transmission_loop(self) -> None:
-        pending_pdus = []  # type: List[EventBase]
+        pending_pdus: List[EventBase] = []
         try:
             self.transmission_loop_running = True
 
@@ -557,6 +568,13 @@ async def _get_to_device_message_edus(self, limit: int) -> Tuple[List[Edu], int]
         contents, stream_id = await self._store.get_new_device_msgs_for_remote(
             self._destination, last_device_stream_id, to_device_stream_id, limit
         )
+        for content in contents:
+            message_id = content.get("message_id")
+            if not message_id:
+                continue
+
+            set_tag(SynapseTags.TO_DEVICE_MESSAGE_ID, message_id)
+
         edus = [
             Edu(
                 origin=self._server_name,
@@ -567,6 +585,14 @@ async def _get_to_device_message_edus(self, limit: int) -> Tuple[List[Edu], int]
             for content in contents
         ]
 
+        if edus:
+            issue9533_logger.debug(
+                "Sending %i to-device messages to %s, up to stream id %i",
+                len(edus),
+                self._destination,
+                stream_id,
+            )
+
         return (edus, stream_id)
 
     def _start_catching_up(self) -> None:
diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py
index 07b740c2f2..dc555cca0b 100644
--- a/synapse/federation/sender/transaction_manager.py
+++ b/synapse/federation/sender/transaction_manager.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,6 +27,7 @@
     tags,
     whitelisted_homeserver,
 )
+from synapse.types import JsonDict
 from synapse.util import json_decoder
 from synapse.util.metrics import measure_func
 
@@ -57,7 +57,7 @@ def __init__(self, hs: "synapse.server.HomeServer"):
         self._transport_layer = hs.get_federation_transport_client()
 
         self._federation_metrics_domains = (
-            hs.get_config().federation.federation_metrics_domains
+            hs.config.federation.federation_metrics_domains
         )
 
         # HACK to get unique tx id
@@ -105,13 +105,13 @@ async def send_new_transaction(
                 len(edus),
             )
 
-            transaction = Transaction.create_new(
+            transaction = Transaction(
                 origin_server_ts=int(self.clock.time_msec()),
                 transaction_id=txn_id,
                 origin=self._server_name,
                 destination=destination,
-                pdus=pdus,
-                edus=edus,
+                pdus=[p.get_pdu_json() for p in pdus],
+                edus=[edu.get_dict() for edu in edus],
             )
 
             self._next_txn_id += 1
@@ -132,7 +132,7 @@ async def send_new_transaction(
             # FIXME (richardv): I also believe it no longer works. We (now?) store
             #  "age_ts" in "unsigned" rather than at the top level. See
             #  https://github.com/matrix-org/synapse/issues/8429.
-            def json_data_cb():
+            def json_data_cb() -> JsonDict:
                 data = transaction.get_dict()
                 now = int(self.clock.time_msec())
                 if "pdus" in data:
diff --git a/synapse/federation/transport/__init__.py b/synapse/federation/transport/__init__.py
index 5db733af98..3c9a0f6944 100644
--- a/synapse/federation/transport/__init__.py
+++ b/synapse/federation/transport/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index df7e9fbbc2..6283134dcc 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -1,6 +1,3 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
 # Copyright 2020 Sorunome
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -18,20 +15,32 @@
 
 import logging
 import urllib
-from typing import Any, Dict, List, Optional
+from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Tuple, Union
+
+import attr
+import ijson
 
 from synapse.api.constants import Membership
 from synapse.api.errors import Codes, HttpResponseException, SynapseError
+from synapse.api.room_versions import RoomVersion
 from synapse.api.urls import (
     FEDERATION_UNSTABLE_PREFIX,
     FEDERATION_V1_PREFIX,
     FEDERATION_V2_PREFIX,
 )
+from synapse.events import EventBase, make_event_from_dict
+from synapse.federation.units import Transaction
+from synapse.http.matrixfederationclient import ByteParser
 from synapse.logging.utils import log_function
 from synapse.types import JsonDict
 
 logger = logging.getLogger(__name__)
 
+# Send join responses can be huge, so we set a separate limit here. The response
+# is parsed in a streaming manner, which helps alleviate the issue of memory
+# usage a bit.
+MAX_RESPONSE_SIZE_SEND_JOIN = 500 * 1024 * 1024
+
 
 class TransportLayerClient:
     """Sends federation HTTP requests to other servers"""
@@ -41,23 +50,25 @@ def __init__(self, hs):
         self.client = hs.get_federation_http_client()
 
     @log_function
-    def get_room_state_ids(self, destination, room_id, event_id):
+    async def get_room_state_ids(
+        self, destination: str, room_id: str, event_id: str
+    ) -> JsonDict:
         """Requests all state for a given room from the given server at the
         given event. Returns the state's event_id's
 
         Args:
-            destination (str): The host name of the remote homeserver we want
+            destination: The host name of the remote homeserver we want
                 to get the state from.
-            context (str): The name of the context we want the state of
-            event_id (str): The event we want the context at.
+            context: The name of the context we want the state of
+            event_id: The event we want the context at.
 
         Returns:
-            Awaitable: Results in a dict received from the remote homeserver.
+            Results in a dict received from the remote homeserver.
         """
         logger.debug("get_room_state_ids dest=%s, room=%s", destination, room_id)
 
         path = _create_v1_path("/state_ids/%s", room_id)
-        return self.client.get_json(
+        return await self.client.get_json(
             destination,
             path=path,
             args={"event_id": event_id},
@@ -65,39 +76,43 @@ def get_room_state_ids(self, destination, room_id, event_id):
         )
 
     @log_function
-    def get_event(self, destination, event_id, timeout=None):
+    async def get_event(
+        self, destination: str, event_id: str, timeout: Optional[int] = None
+    ) -> JsonDict:
         """Requests the pdu with give id and origin from the given server.
 
         Args:
-            destination (str): The host name of the remote homeserver we want
+            destination: The host name of the remote homeserver we want
                 to get the state from.
-            event_id (str): The id of the event being requested.
-            timeout (int): How long to try (in ms) the destination for before
+            event_id: The id of the event being requested.
+            timeout: How long to try (in ms) the destination for before
                 giving up. None indicates no timeout.
 
         Returns:
-            Awaitable: Results in a dict received from the remote homeserver.
+            Results in a dict received from the remote homeserver.
         """
         logger.debug("get_pdu dest=%s, event_id=%s", destination, event_id)
 
         path = _create_v1_path("/event/%s", event_id)
-        return self.client.get_json(
+        return await self.client.get_json(
             destination, path=path, timeout=timeout, try_trailing_slash_on_400=True
         )
 
     @log_function
-    def backfill(self, destination, room_id, event_tuples, limit):
+    async def backfill(
+        self, destination: str, room_id: str, event_tuples: Iterable[str], limit: int
+    ) -> Optional[JsonDict]:
         """Requests `limit` previous PDUs in a given context before list of
         PDUs.
 
         Args:
-            dest (str)
-            room_id (str)
-            event_tuples (list)
-            limit (int)
+            destination
+            room_id
+            event_tuples
+            limit
 
         Returns:
-            Awaitable: Results in a dict received from the remote homeserver.
+            Results in a dict received from the remote homeserver.
         """
         logger.debug(
             "backfill dest=%s, room_id=%s, event_tuples=%r, limit=%s",
@@ -109,22 +124,26 @@ def backfill(self, destination, room_id, event_tuples, limit):
 
         if not event_tuples:
             # TODO: raise?
-            return
+            return None
 
         path = _create_v1_path("/backfill/%s", room_id)
 
         args = {"v": event_tuples, "limit": [str(limit)]}
 
-        return self.client.get_json(
+        return await self.client.get_json(
             destination, path=path, args=args, try_trailing_slash_on_400=True
         )
 
     @log_function
-    async def send_transaction(self, transaction, json_data_callback=None):
+    async def send_transaction(
+        self,
+        transaction: Transaction,
+        json_data_callback: Optional[Callable[[], JsonDict]] = None,
+    ) -> JsonDict:
         """Sends the given Transaction to its destination
 
         Args:
-            transaction (Transaction)
+            transaction
 
         Returns:
             Succeeds when we get a 2xx HTTP response. The result
@@ -141,21 +160,21 @@ async def send_transaction(self, transaction, json_data_callback=None):
         """
         logger.debug(
             "send_data dest=%s, txid=%s",
-            transaction.destination,
-            transaction.transaction_id,
+            transaction.destination,  # type: ignore
+            transaction.transaction_id,  # type: ignore
         )
 
-        if transaction.destination == self.server_name:
+        if transaction.destination == self.server_name:  # type: ignore
             raise RuntimeError("Transport layer cannot send to itself!")
 
         # FIXME: This is only used by the tests. The actual json sent is
         # generated by the json_data_callback.
         json_data = transaction.get_dict()
 
-        path = _create_v1_path("/send/%s", transaction.transaction_id)
+        path = _create_v1_path("/send/%s", transaction.transaction_id)  # type: ignore
 
-        response = await self.client.put_json(
-            transaction.destination,
+        return await self.client.put_json(
+            transaction.destination,  # type: ignore
             path=path,
             data=json_data,
             json_data_callback=json_data_callback,
@@ -164,8 +183,6 @@ async def send_transaction(self, transaction, json_data_callback=None):
             try_trailing_slash_on_400=True,
         )
 
-        return response
-
     @log_function
     async def make_query(
         self, destination, query_type, args, retry_on_dns_fail, ignore_backoff=False
@@ -185,8 +202,13 @@ async def make_query(
 
     @log_function
     async def make_membership_event(
-        self, destination, room_id, user_id, membership, params
-    ):
+        self,
+        destination: str,
+        room_id: str,
+        user_id: str,
+        membership: str,
+        params: Optional[Mapping[str, Union[str, Iterable[str]]]],
+    ) -> JsonDict:
         """Asks a remote server to build and sign us a membership event
 
         Note that this does not append any events to any graphs.
@@ -213,23 +235,13 @@ async def make_membership_event(
             is not in our federation whitelist
         """
         valid_memberships = {Membership.JOIN, Membership.LEAVE, Membership.KNOCK}
+
         if membership not in valid_memberships:
             raise RuntimeError(
                 "make_membership_event called with membership='%s', must be one of %s"
                 % (membership, ",".join(valid_memberships))
             )
-
-        # Knock currently uses an unstable prefix
-        if membership == Membership.KNOCK:
-            # Create a path in the form of /unstable/xyz.amorgan.knock/make_knock/...
-            path = _create_path(
-                FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock",
-                "/make_knock/%s/%s",
-                room_id,
-                user_id,
-            )
-        else:
-            path = _create_v1_path("/make_%s/%s/%s", membership, room_id, user_id)
+        path = _create_v1_path("/make_%s/%s/%s", membership, room_id, user_id)
 
         ignore_backoff = False
         retry_on_dns_fail = False
@@ -242,7 +254,7 @@ async def make_membership_event(
             ignore_backoff = True
             retry_on_dns_fail = True
 
-        content = await self.client.get_json(
+        return await self.client.get_json(
             destination=destination,
             path=path,
             args=params,
@@ -251,33 +263,51 @@ async def make_membership_event(
             ignore_backoff=ignore_backoff,
         )
 
-        return content
-
     @log_function
-    async def send_join_v1(self, destination, room_id, event_id, content):
+    async def send_join_v1(
+        self,
+        room_version: RoomVersion,
+        destination: str,
+        room_id: str,
+        event_id: str,
+        content: JsonDict,
+    ) -> "SendJoinResponse":
         path = _create_v1_path("/send_join/%s/%s", room_id, event_id)
 
-        response = await self.client.put_json(
-            destination=destination, path=path, data=content
+        return await self.client.put_json(
+            destination=destination,
+            path=path,
+            data=content,
+            parser=SendJoinParser(room_version, v1_api=True),
+            max_response_size=MAX_RESPONSE_SIZE_SEND_JOIN,
         )
 
-        return response
-
     @log_function
-    async def send_join_v2(self, destination, room_id, event_id, content):
+    async def send_join_v2(
+        self,
+        room_version: RoomVersion,
+        destination: str,
+        room_id: str,
+        event_id: str,
+        content: JsonDict,
+    ) -> "SendJoinResponse":
         path = _create_v2_path("/send_join/%s/%s", room_id, event_id)
 
-        response = await self.client.put_json(
-            destination=destination, path=path, data=content
+        return await self.client.put_json(
+            destination=destination,
+            path=path,
+            data=content,
+            parser=SendJoinParser(room_version, v1_api=False),
+            max_response_size=MAX_RESPONSE_SIZE_SEND_JOIN,
         )
 
-        return response
-
     @log_function
-    async def send_leave_v1(self, destination, room_id, event_id, content):
+    async def send_leave_v1(
+        self, destination: str, room_id: str, event_id: str, content: JsonDict
+    ) -> Tuple[int, JsonDict]:
         path = _create_v1_path("/send_leave/%s/%s", room_id, event_id)
 
-        response = await self.client.put_json(
+        return await self.client.put_json(
             destination=destination,
             path=path,
             data=content,
@@ -288,13 +318,13 @@ async def send_leave_v1(self, destination, room_id, event_id, content):
             ignore_backoff=True,
         )
 
-        return response
-
     @log_function
-    async def send_leave_v2(self, destination, room_id, event_id, content):
+    async def send_leave_v2(
+        self, destination: str, room_id: str, event_id: str, content: JsonDict
+    ) -> JsonDict:
         path = _create_v2_path("/send_leave/%s/%s", room_id, event_id)
 
-        response = await self.client.put_json(
+        return await self.client.put_json(
             destination=destination,
             path=path,
             data=content,
@@ -305,10 +335,8 @@ async def send_leave_v2(self, destination, room_id, event_id, content):
             ignore_backoff=True,
         )
 
-        return response
-
     @log_function
-    async def send_knock_v2(
+    async def send_knock_v1(
         self,
         destination: str,
         room_id: str,
@@ -335,37 +363,32 @@ async def send_knock_v2(
 
             The list of state events may be empty.
         """
-        path = _create_path(
-            FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock",
-            "/send_knock/%s/%s",
-            room_id,
-            event_id,
-        )
+        path = _create_v1_path("/send_knock/%s/%s", room_id, event_id)
 
         return await self.client.put_json(
             destination=destination, path=path, data=content
         )
 
     @log_function
-    async def send_invite_v1(self, destination, room_id, event_id, content):
+    async def send_invite_v1(
+        self, destination: str, room_id: str, event_id: str, content: JsonDict
+    ) -> Tuple[int, JsonDict]:
         path = _create_v1_path("/invite/%s/%s", room_id, event_id)
 
-        response = await self.client.put_json(
+        return await self.client.put_json(
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
-        return response
-
     @log_function
-    async def send_invite_v2(self, destination, room_id, event_id, content):
+    async def send_invite_v2(
+        self, destination: str, room_id: str, event_id: str, content: JsonDict
+    ) -> JsonDict:
         path = _create_v2_path("/invite/%s/%s", room_id, event_id)
 
-        response = await self.client.put_json(
+        return await self.client.put_json(
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
-        return response
-
     @log_function
     async def get_public_rooms(
         self,
@@ -375,7 +398,7 @@ async def get_public_rooms(
         search_filter: Optional[Dict] = None,
         include_all_networks: bool = False,
         third_party_instance_id: Optional[str] = None,
-    ):
+    ) -> JsonDict:
         """Get the list of public rooms from a remote homeserver
 
         See synapse.federation.federation_client.FederationClient.get_public_rooms for
@@ -385,9 +408,9 @@ async def get_public_rooms(
             # this uses MSC2197 (Search Filtering over Federation)
             path = _create_v1_path("/publicRooms")
 
-            data = {
+            data: Dict[str, Any] = {
                 "include_all_networks": "true" if include_all_networks else "false"
-            }  # type: Dict[str, Any]
+            }
             if third_party_instance_id:
                 data["third_party_instance_id"] = third_party_instance_id
             if limit:
@@ -413,9 +436,9 @@ async def get_public_rooms(
         else:
             path = _create_v1_path("/publicRooms")
 
-            args = {
+            args: Dict[str, Any] = {
                 "include_all_networks": "true" if include_all_networks else "false"
-            }  # type: Dict[str, Any]
+            }
             if third_party_instance_id:
                 args["third_party_instance_id"] = (third_party_instance_id,)
             if limit:
@@ -440,25 +463,27 @@ async def get_public_rooms(
         return response
 
     @log_function
-    async def exchange_third_party_invite(self, destination, room_id, event_dict):
+    async def exchange_third_party_invite(
+        self, destination: str, room_id: str, event_dict: JsonDict
+    ) -> JsonDict:
         path = _create_v1_path("/exchange_third_party_invite/%s", room_id)
 
-        response = await self.client.put_json(
+        return await self.client.put_json(
             destination=destination, path=path, data=event_dict
         )
 
-        return response
-
     @log_function
-    async def get_event_auth(self, destination, room_id, event_id):
+    async def get_event_auth(
+        self, destination: str, room_id: str, event_id: str
+    ) -> JsonDict:
         path = _create_v1_path("/event_auth/%s/%s", room_id, event_id)
 
-        content = await self.client.get_json(destination=destination, path=path)
-
-        return content
+        return await self.client.get_json(destination=destination, path=path)
 
     @log_function
-    async def query_client_keys(self, destination, query_content, timeout):
+    async def query_client_keys(
+        self, destination: str, query_content: JsonDict, timeout: int
+    ) -> JsonDict:
         """Query the device keys for a list of user ids hosted on a remote
         server.
 
@@ -486,20 +511,21 @@ async def query_client_keys(self, destination, query_content, timeout):
             }
 
         Args:
-            destination(str): The server to query.
-            query_content(dict): The user ids to query.
+            destination: The server to query.
+            query_content: The user ids to query.
         Returns:
             A dict containing device and cross-signing keys.
         """
         path = _create_v1_path("/user/keys/query")
 
-        content = await self.client.post_json(
+        return await self.client.post_json(
             destination=destination, path=path, data=query_content, timeout=timeout
         )
-        return content
 
     @log_function
-    async def query_user_devices(self, destination, user_id, timeout):
+    async def query_user_devices(
+        self, destination: str, user_id: str, timeout: int
+    ) -> JsonDict:
         """Query the devices for a user id hosted on a remote server.
 
         Response:
@@ -525,20 +551,21 @@ async def query_user_devices(self, destination, user_id, timeout):
             }
 
         Args:
-            destination(str): The server to query.
-            query_content(dict): The user ids to query.
+            destination: The server to query.
+            query_content: The user ids to query.
         Returns:
             A dict containing device and cross-signing keys.
         """
         path = _create_v1_path("/user/devices/%s", user_id)
 
-        content = await self.client.get_json(
+        return await self.client.get_json(
             destination=destination, path=path, timeout=timeout
         )
-        return content
 
     @log_function
-    async def claim_client_keys(self, destination, query_content, timeout):
+    async def claim_client_keys(
+        self, destination: str, query_content: JsonDict, timeout: int
+    ) -> JsonDict:
         """Claim one-time keys for a list of devices hosted on a remote server.
 
         Request:
@@ -562,33 +589,32 @@ async def claim_client_keys(self, destination, query_content, timeout):
             }
 
         Args:
-            destination(str): The server to query.
-            query_content(dict): The user ids to query.
+            destination: The server to query.
+            query_content: The user ids to query.
         Returns:
             A dict containing the one-time keys.
         """
 
         path = _create_v1_path("/user/keys/claim")
 
-        content = await self.client.post_json(
+        return await self.client.post_json(
             destination=destination, path=path, data=query_content, timeout=timeout
         )
-        return content
 
     @log_function
     async def get_missing_events(
         self,
-        destination,
-        room_id,
-        earliest_events,
-        latest_events,
-        limit,
-        min_depth,
-        timeout,
-    ):
+        destination: str,
+        room_id: str,
+        earliest_events: Iterable[str],
+        latest_events: Iterable[str],
+        limit: int,
+        min_depth: int,
+        timeout: int,
+    ) -> JsonDict:
         path = _create_v1_path("/get_missing_events/%s", room_id)
 
-        content = await self.client.post_json(
+        return await self.client.post_json(
             destination=destination,
             path=path,
             data={
@@ -600,14 +626,14 @@ async def get_missing_events(
             timeout=timeout,
         )
 
-        return content
-
     @log_function
-    def get_group_profile(self, destination, group_id, requester_user_id):
+    async def get_group_profile(
+        self, destination: str, group_id: str, requester_user_id: str
+    ) -> JsonDict:
         """Get a group profile"""
         path = _create_v1_path("/groups/%s/profile", group_id)
 
-        return self.client.get_json(
+        return await self.client.get_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -615,14 +641,16 @@ def get_group_profile(self, destination, group_id, requester_user_id):
         )
 
     @log_function
-    def update_group_profile(self, destination, group_id, requester_user_id, content):
+    async def update_group_profile(
+        self, destination: str, group_id: str, requester_user_id: str, content: JsonDict
+    ) -> JsonDict:
         """Update a remote group profile
 
         Args:
-            destination (str)
-            group_id (str)
-            requester_user_id (str)
-            content (dict): The new profile of the group
+            destination
+            group_id
+            requester_user_id
+            content: The new profile of the group
         """
         path = _create_v1_path("/groups/%s/profile", group_id)
 
@@ -635,11 +663,13 @@ def update_group_profile(self, destination, group_id, requester_user_id, content
         )
 
     @log_function
-    def get_group_summary(self, destination, group_id, requester_user_id):
+    async def get_group_summary(
+        self, destination: str, group_id: str, requester_user_id: str
+    ) -> JsonDict:
         """Get a group summary"""
         path = _create_v1_path("/groups/%s/summary", group_id)
 
-        return self.client.get_json(
+        return await self.client.get_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -647,24 +677,31 @@ def get_group_summary(self, destination, group_id, requester_user_id):
         )
 
     @log_function
-    def get_rooms_in_group(self, destination, group_id, requester_user_id):
+    async def get_rooms_in_group(
+        self, destination: str, group_id: str, requester_user_id: str
+    ) -> JsonDict:
         """Get all rooms in a group"""
         path = _create_v1_path("/groups/%s/rooms", group_id)
 
-        return self.client.get_json(
+        return await self.client.get_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
             ignore_backoff=True,
         )
 
-    def add_room_to_group(
-        self, destination, group_id, requester_user_id, room_id, content
-    ):
+    async def add_room_to_group(
+        self,
+        destination: str,
+        group_id: str,
+        requester_user_id: str,
+        room_id: str,
+        content: JsonDict,
+    ) -> JsonDict:
         """Add a room to a group"""
         path = _create_v1_path("/groups/%s/room/%s", group_id, room_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -672,15 +709,21 @@ def add_room_to_group(
             ignore_backoff=True,
         )
 
-    def update_room_in_group(
-        self, destination, group_id, requester_user_id, room_id, config_key, content
-    ):
+    async def update_room_in_group(
+        self,
+        destination: str,
+        group_id: str,
+        requester_user_id: str,
+        room_id: str,
+        config_key: str,
+        content: JsonDict,
+    ) -> JsonDict:
         """Update room in group"""
         path = _create_v1_path(
             "/groups/%s/room/%s/config/%s", group_id, room_id, config_key
         )
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -688,11 +731,13 @@ def update_room_in_group(
             ignore_backoff=True,
         )
 
-    def remove_room_from_group(self, destination, group_id, requester_user_id, room_id):
+    async def remove_room_from_group(
+        self, destination: str, group_id: str, requester_user_id: str, room_id: str
+    ) -> JsonDict:
         """Remove a room from a group"""
         path = _create_v1_path("/groups/%s/room/%s", group_id, room_id)
 
-        return self.client.delete_json(
+        return await self.client.delete_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -700,11 +745,13 @@ def remove_room_from_group(self, destination, group_id, requester_user_id, room_
         )
 
     @log_function
-    def get_users_in_group(self, destination, group_id, requester_user_id):
+    async def get_users_in_group(
+        self, destination: str, group_id: str, requester_user_id: str
+    ) -> JsonDict:
         """Get users in a group"""
         path = _create_v1_path("/groups/%s/users", group_id)
 
-        return self.client.get_json(
+        return await self.client.get_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -712,11 +759,13 @@ def get_users_in_group(self, destination, group_id, requester_user_id):
         )
 
     @log_function
-    def get_invited_users_in_group(self, destination, group_id, requester_user_id):
+    async def get_invited_users_in_group(
+        self, destination: str, group_id: str, requester_user_id: str
+    ) -> JsonDict:
         """Get users that have been invited to a group"""
         path = _create_v1_path("/groups/%s/invited_users", group_id)
 
-        return self.client.get_json(
+        return await self.client.get_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -724,16 +773,20 @@ def get_invited_users_in_group(self, destination, group_id, requester_user_id):
         )
 
     @log_function
-    def accept_group_invite(self, destination, group_id, user_id, content):
+    async def accept_group_invite(
+        self, destination: str, group_id: str, user_id: str, content: JsonDict
+    ) -> JsonDict:
         """Accept a group invite"""
         path = _create_v1_path("/groups/%s/users/%s/accept_invite", group_id, user_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
     @log_function
-    def join_group(self, destination, group_id, user_id, content):
+    def join_group(
+        self, destination: str, group_id: str, user_id: str, content: JsonDict
+    ) -> JsonDict:
         """Attempts to join a group"""
         path = _create_v1_path("/groups/%s/users/%s/join", group_id, user_id)
 
@@ -742,13 +795,18 @@ def join_group(self, destination, group_id, user_id, content):
         )
 
     @log_function
-    def invite_to_group(
-        self, destination, group_id, user_id, requester_user_id, content
-    ):
+    async def invite_to_group(
+        self,
+        destination: str,
+        group_id: str,
+        user_id: str,
+        requester_user_id: str,
+        content: JsonDict,
+    ) -> JsonDict:
         """Invite a user to a group"""
         path = _create_v1_path("/groups/%s/users/%s/invite", group_id, user_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -757,25 +815,32 @@ def invite_to_group(
         )
 
     @log_function
-    def invite_to_group_notification(self, destination, group_id, user_id, content):
+    async def invite_to_group_notification(
+        self, destination: str, group_id: str, user_id: str, content: JsonDict
+    ) -> JsonDict:
         """Sent by group server to inform a user's server that they have been
         invited.
         """
 
         path = _create_v1_path("/groups/local/%s/users/%s/invite", group_id, user_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
     @log_function
-    def remove_user_from_group(
-        self, destination, group_id, requester_user_id, user_id, content
-    ):
+    async def remove_user_from_group(
+        self,
+        destination: str,
+        group_id: str,
+        requester_user_id: str,
+        user_id: str,
+        content: JsonDict,
+    ) -> JsonDict:
         """Remove a user from a group"""
         path = _create_v1_path("/groups/%s/users/%s/remove", group_id, user_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -784,35 +849,43 @@ def remove_user_from_group(
         )
 
     @log_function
-    def remove_user_from_group_notification(
-        self, destination, group_id, user_id, content
-    ):
+    async def remove_user_from_group_notification(
+        self, destination: str, group_id: str, user_id: str, content: JsonDict
+    ) -> JsonDict:
         """Sent by group server to inform a user's server that they have been
         kicked from the group.
         """
 
         path = _create_v1_path("/groups/local/%s/users/%s/remove", group_id, user_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
     @log_function
-    def renew_group_attestation(self, destination, group_id, user_id, content):
+    async def renew_group_attestation(
+        self, destination: str, group_id: str, user_id: str, content: JsonDict
+    ) -> JsonDict:
         """Sent by either a group server or a user's server to periodically update
         the attestations
         """
 
         path = _create_v1_path("/groups/%s/renew_attestation/%s", group_id, user_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
     @log_function
-    def update_group_summary_room(
-        self, destination, group_id, user_id, room_id, category_id, content
-    ):
+    async def update_group_summary_room(
+        self,
+        destination: str,
+        group_id: str,
+        user_id: str,
+        room_id: str,
+        category_id: str,
+        content: JsonDict,
+    ) -> JsonDict:
         """Update a room entry in a group summary"""
         if category_id:
             path = _create_v1_path(
@@ -824,7 +897,7 @@ def update_group_summary_room(
         else:
             path = _create_v1_path("/groups/%s/summary/rooms/%s", group_id, room_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination,
             path=path,
             args={"requester_user_id": user_id},
@@ -833,9 +906,14 @@ def update_group_summary_room(
         )
 
     @log_function
-    def delete_group_summary_room(
-        self, destination, group_id, user_id, room_id, category_id
-    ):
+    async def delete_group_summary_room(
+        self,
+        destination: str,
+        group_id: str,
+        user_id: str,
+        room_id: str,
+        category_id: str,
+    ) -> JsonDict:
         """Delete a room entry in a group summary"""
         if category_id:
             path = _create_v1_path(
@@ -847,7 +925,7 @@ def delete_group_summary_room(
         else:
             path = _create_v1_path("/groups/%s/summary/rooms/%s", group_id, room_id)
 
-        return self.client.delete_json(
+        return await self.client.delete_json(
             destination=destination,
             path=path,
             args={"requester_user_id": user_id},
@@ -855,11 +933,13 @@ def delete_group_summary_room(
         )
 
     @log_function
-    def get_group_categories(self, destination, group_id, requester_user_id):
+    async def get_group_categories(
+        self, destination: str, group_id: str, requester_user_id: str
+    ) -> JsonDict:
         """Get all categories in a group"""
         path = _create_v1_path("/groups/%s/categories", group_id)
 
-        return self.client.get_json(
+        return await self.client.get_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -867,11 +947,13 @@ def get_group_categories(self, destination, group_id, requester_user_id):
         )
 
     @log_function
-    def get_group_category(self, destination, group_id, requester_user_id, category_id):
+    async def get_group_category(
+        self, destination: str, group_id: str, requester_user_id: str, category_id: str
+    ) -> JsonDict:
         """Get category info in a group"""
         path = _create_v1_path("/groups/%s/categories/%s", group_id, category_id)
 
-        return self.client.get_json(
+        return await self.client.get_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -879,13 +961,18 @@ def get_group_category(self, destination, group_id, requester_user_id, category_
         )
 
     @log_function
-    def update_group_category(
-        self, destination, group_id, requester_user_id, category_id, content
-    ):
+    async def update_group_category(
+        self,
+        destination: str,
+        group_id: str,
+        requester_user_id: str,
+        category_id: str,
+        content: JsonDict,
+    ) -> JsonDict:
         """Update a category in a group"""
         path = _create_v1_path("/groups/%s/categories/%s", group_id, category_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -894,13 +981,13 @@ def update_group_category(
         )
 
     @log_function
-    def delete_group_category(
-        self, destination, group_id, requester_user_id, category_id
-    ):
+    async def delete_group_category(
+        self, destination: str, group_id: str, requester_user_id: str, category_id: str
+    ) -> JsonDict:
         """Delete a category in a group"""
         path = _create_v1_path("/groups/%s/categories/%s", group_id, category_id)
 
-        return self.client.delete_json(
+        return await self.client.delete_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -908,11 +995,13 @@ def delete_group_category(
         )
 
     @log_function
-    def get_group_roles(self, destination, group_id, requester_user_id):
+    async def get_group_roles(
+        self, destination: str, group_id: str, requester_user_id: str
+    ) -> JsonDict:
         """Get all roles in a group"""
         path = _create_v1_path("/groups/%s/roles", group_id)
 
-        return self.client.get_json(
+        return await self.client.get_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -920,11 +1009,13 @@ def get_group_roles(self, destination, group_id, requester_user_id):
         )
 
     @log_function
-    def get_group_role(self, destination, group_id, requester_user_id, role_id):
+    async def get_group_role(
+        self, destination: str, group_id: str, requester_user_id: str, role_id: str
+    ) -> JsonDict:
         """Get a roles info"""
         path = _create_v1_path("/groups/%s/roles/%s", group_id, role_id)
 
-        return self.client.get_json(
+        return await self.client.get_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -932,13 +1023,18 @@ def get_group_role(self, destination, group_id, requester_user_id, role_id):
         )
 
     @log_function
-    def update_group_role(
-        self, destination, group_id, requester_user_id, role_id, content
-    ):
+    async def update_group_role(
+        self,
+        destination: str,
+        group_id: str,
+        requester_user_id: str,
+        role_id: str,
+        content: JsonDict,
+    ) -> JsonDict:
         """Update a role in a group"""
         path = _create_v1_path("/groups/%s/roles/%s", group_id, role_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -947,11 +1043,13 @@ def update_group_role(
         )
 
     @log_function
-    def delete_group_role(self, destination, group_id, requester_user_id, role_id):
+    async def delete_group_role(
+        self, destination: str, group_id: str, requester_user_id: str, role_id: str
+    ) -> JsonDict:
         """Delete a role in a group"""
         path = _create_v1_path("/groups/%s/roles/%s", group_id, role_id)
 
-        return self.client.delete_json(
+        return await self.client.delete_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -959,9 +1057,15 @@ def delete_group_role(self, destination, group_id, requester_user_id, role_id):
         )
 
     @log_function
-    def update_group_summary_user(
-        self, destination, group_id, requester_user_id, user_id, role_id, content
-    ):
+    async def update_group_summary_user(
+        self,
+        destination: str,
+        group_id: str,
+        requester_user_id: str,
+        user_id: str,
+        role_id: str,
+        content: JsonDict,
+    ) -> JsonDict:
         """Update a users entry in a group"""
         if role_id:
             path = _create_v1_path(
@@ -970,7 +1074,7 @@ def update_group_summary_user(
         else:
             path = _create_v1_path("/groups/%s/summary/users/%s", group_id, user_id)
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -979,11 +1083,13 @@ def update_group_summary_user(
         )
 
     @log_function
-    def set_group_join_policy(self, destination, group_id, requester_user_id, content):
+    async def set_group_join_policy(
+        self, destination: str, group_id: str, requester_user_id: str, content: JsonDict
+    ) -> JsonDict:
         """Sets the join policy for a group"""
         path = _create_v1_path("/groups/%s/settings/m.join_policy", group_id)
 
-        return self.client.put_json(
+        return await self.client.put_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
@@ -992,9 +1098,14 @@ def set_group_join_policy(self, destination, group_id, requester_user_id, conten
         )
 
     @log_function
-    def delete_group_summary_user(
-        self, destination, group_id, requester_user_id, user_id, role_id
-    ):
+    async def delete_group_summary_user(
+        self,
+        destination: str,
+        group_id: str,
+        requester_user_id: str,
+        user_id: str,
+        role_id: str,
+    ) -> JsonDict:
         """Delete a users entry in a group"""
         if role_id:
             path = _create_v1_path(
@@ -1003,33 +1114,35 @@ def delete_group_summary_user(
         else:
             path = _create_v1_path("/groups/%s/summary/users/%s", group_id, user_id)
 
-        return self.client.delete_json(
+        return await self.client.delete_json(
             destination=destination,
             path=path,
             args={"requester_user_id": requester_user_id},
             ignore_backoff=True,
         )
 
-    def bulk_get_publicised_groups(self, destination, user_ids):
+    async def bulk_get_publicised_groups(
+        self, destination: str, user_ids: Iterable[str]
+    ) -> JsonDict:
         """Get the groups a list of users are publicising"""
 
         path = _create_v1_path("/get_groups_publicised")
 
         content = {"user_ids": user_ids}
 
-        return self.client.post_json(
+        return await self.client.post_json(
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
-    def get_room_complexity(self, destination, room_id):
+    async def get_room_complexity(self, destination: str, room_id: str) -> JsonDict:
         """
         Args:
-            destination (str): The remote server
-            room_id (str): The room ID to ask about.
+            destination: The remote server
+            room_id: The room ID to ask about.
         """
         path = _create_path(FEDERATION_UNSTABLE_PREFIX, "/rooms/%s/complexity", room_id)
 
-        return self.client.get_json(destination=destination, path=path)
+        return await self.client.get_json(destination=destination, path=path)
 
     async def get_space_summary(
         self,
@@ -1048,6 +1161,7 @@ async def get_space_summary(
                returned per space
             exclude_rooms: a list of any rooms we can skip
         """
+        # TODO When switching to the stable endpoint, use GET instead of POST.
         path = _create_path(
             FEDERATION_UNSTABLE_PREFIX, "/org.matrix.msc2946/spaces/%s", room_id
         )
@@ -1063,6 +1177,28 @@ async def get_space_summary(
             destination=destination, path=path, data=params
         )
 
+    async def get_room_hierarchy(
+        self,
+        destination: str,
+        room_id: str,
+        suggested_only: bool,
+    ) -> JsonDict:
+        """
+        Args:
+            destination: The remote server
+            room_id: The room ID to ask about.
+            suggested_only: if True, only suggested rooms will be returned
+        """
+        path = _create_path(
+            FEDERATION_UNSTABLE_PREFIX, "/org.matrix.msc2946/hierarchy/%s", room_id
+        )
+
+        return await self.client.get_json(
+            destination=destination,
+            path=path,
+            args={"suggested_only": "true" if suggested_only else "false"},
+        )
+
     def get_info_of_users(self, destination: str, user_ids: List[str]):
         """
         Args:
@@ -1078,14 +1214,14 @@ def get_info_of_users(self, destination: str, user_ids: List[str]):
         return self.client.post_json(destination=destination, path=path, data=data)
 
 
-def _create_path(federation_prefix, path, *args):
+def _create_path(federation_prefix: str, path: str, *args: str) -> str:
     """
     Ensures that all args are url encoded.
     """
     return federation_prefix + path % tuple(urllib.parse.quote(arg, "") for arg in args)
 
 
-def _create_v1_path(path, *args):
+def _create_v1_path(path: str, *args: str) -> str:
     """Creates a path against V1 federation API from the path template and
     args. Ensures that all args are url encoded.
 
@@ -1094,16 +1230,13 @@ def _create_v1_path(path, *args):
         _create_v1_path("/event/%s", event_id)
 
     Args:
-        path (str): String template for the path
-        args: ([str]): Args to insert into path. Each arg will be url encoded
-
-    Returns:
-        str
+        path: String template for the path
+        args: Args to insert into path. Each arg will be url encoded
     """
     return _create_path(FEDERATION_V1_PREFIX, path, *args)
 
 
-def _create_v2_path(path, *args):
+def _create_v2_path(path: str, *args: str) -> str:
     """Creates a path against V2 federation API from the path template and
     args. Ensures that all args are url encoded.
 
@@ -1112,10 +1245,91 @@ def _create_v2_path(path, *args):
         _create_v2_path("/event/%s", event_id)
 
     Args:
-        path (str): String template for the path
-        args: ([str]): Args to insert into path. Each arg will be url encoded
-
-    Returns:
-        str
+        path: String template for the path
+        args: Args to insert into path. Each arg will be url encoded
     """
     return _create_path(FEDERATION_V2_PREFIX, path, *args)
+
+
+@attr.s(slots=True, auto_attribs=True)
+class SendJoinResponse:
+    """The parsed response of a `/send_join` request."""
+
+    # The list of auth events from the /send_join response.
+    auth_events: List[EventBase]
+    # The list of state from the /send_join response.
+    state: List[EventBase]
+    # The raw join event from the /send_join response.
+    event_dict: JsonDict
+    # The parsed join event from the /send_join response. This will be None if
+    # "event" is not included in the response.
+    event: Optional[EventBase] = None
+
+
+@ijson.coroutine
+def _event_parser(event_dict: JsonDict):
+    """Helper function for use with `ijson.kvitems_coro` to parse key-value pairs
+    to add them to a given dictionary.
+    """
+
+    while True:
+        key, value = yield
+        event_dict[key] = value
+
+
+@ijson.coroutine
+def _event_list_parser(room_version: RoomVersion, events: List[EventBase]):
+    """Helper function for use with `ijson.items_coro` to parse an array of
+    events and add them to the given list.
+    """
+
+    while True:
+        obj = yield
+        event = make_event_from_dict(obj, room_version)
+        events.append(event)
+
+
+class SendJoinParser(ByteParser[SendJoinResponse]):
+    """A parser for the response to `/send_join` requests.
+
+    Args:
+        room_version: The version of the room.
+        v1_api: Whether the response is in the v1 format.
+    """
+
+    CONTENT_TYPE = "application/json"
+
+    def __init__(self, room_version: RoomVersion, v1_api: bool):
+        self._response = SendJoinResponse([], [], {})
+        self._room_version = room_version
+
+        # The V1 API has the shape of `[200, {...}]`, which we handle by
+        # prefixing with `item.*`.
+        prefix = "item." if v1_api else ""
+
+        self._coro_state = ijson.items_coro(
+            _event_list_parser(room_version, self._response.state),
+            prefix + "state.item",
+        )
+        self._coro_auth = ijson.items_coro(
+            _event_list_parser(room_version, self._response.auth_events),
+            prefix + "auth_chain.item",
+        )
+        self._coro_event = ijson.kvitems_coro(
+            _event_parser(self._response.event_dict),
+            prefix + "org.matrix.msc3083.v2.event",
+        )
+
+    def write(self, data: bytes) -> int:
+        self._coro_state.send(data)
+        self._coro_auth.send(data)
+        self._coro_event.send(data)
+
+        return len(data)
+
+    def finish(self) -> SendJoinResponse:
+        if self._response.event_dict:
+            self._response.event = make_event_from_dict(
+                self._response.event_dict, self._room_version
+            )
+        return self._response
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 294031e2a0..e69de29bb2 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1,1680 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
-# Copyright 2019-2020 The Matrix.org Foundation C.I.C.
-# Copyright 2020 Sorunome
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import functools
-import logging
-import re
-from typing import Container, Mapping, Optional, Sequence, Tuple, Type
-
-import synapse
-from synapse.api.constants import MAX_GROUP_CATEGORYID_LENGTH, MAX_GROUP_ROLEID_LENGTH
-from synapse.api.errors import Codes, FederationDeniedError, SynapseError
-from synapse.api.room_versions import RoomVersions
-from synapse.api.urls import (
-    FEDERATION_UNSTABLE_PREFIX,
-    FEDERATION_V1_PREFIX,
-    FEDERATION_V2_PREFIX,
-)
-from synapse.http.server import HttpServer, JsonResource
-from synapse.http.servlet import (
-    assert_params_in_dict,
-    parse_boolean_from_args,
-    parse_integer_from_args,
-    parse_json_object_from_request,
-    parse_list_from_args,
-    parse_string_from_args,
-)
-from synapse.logging.context import run_in_background
-from synapse.logging.opentracing import (
-    start_active_span,
-    start_active_span_from_request,
-    tags,
-    whitelisted_homeserver,
-)
-from synapse.server import HomeServer
-from synapse.types import JsonDict, ThirdPartyInstanceID, get_domain_from_id
-from synapse.util.ratelimitutils import FederationRateLimiter
-from synapse.util.stringutils import parse_and_validate_server_name
-from synapse.util.versionstring import get_version_string
-
-logger = logging.getLogger(__name__)
-
-
-class TransportLayerServer(JsonResource):
-    """Handles incoming federation HTTP requests"""
-
-    def __init__(self, hs, servlet_groups=None):
-        """Initialize the TransportLayerServer
-
-        Will by default register all servlets. For custom behaviour, pass in
-        a list of servlet_groups to register.
-
-        Args:
-            hs (synapse.server.HomeServer): homeserver
-            servlet_groups (list[str], optional): List of servlet groups to register.
-                Defaults to ``DEFAULT_SERVLET_GROUPS``.
-        """
-        self.hs = hs
-        self.clock = hs.get_clock()
-        self.servlet_groups = servlet_groups
-
-        super().__init__(hs, canonical_json=False)
-
-        self.authenticator = Authenticator(hs)
-        self.ratelimiter = hs.get_federation_ratelimiter()
-
-        self.register_servlets()
-
-    def register_servlets(self):
-        register_servlets(
-            self.hs,
-            resource=self,
-            ratelimiter=self.ratelimiter,
-            authenticator=self.authenticator,
-            servlet_groups=self.servlet_groups,
-        )
-
-
-class AuthenticationError(SynapseError):
-    """There was a problem authenticating the request"""
-
-    pass
-
-
-class NoAuthenticationError(AuthenticationError):
-    """The request had no authentication information"""
-
-    pass
-
-
-class Authenticator:
-    def __init__(self, hs: HomeServer):
-        self._clock = hs.get_clock()
-        self.keyring = hs.get_keyring()
-        self.server_name = hs.hostname
-        self.store = hs.get_datastore()
-        self.federation_domain_whitelist = hs.config.federation_domain_whitelist
-        self.notifier = hs.get_notifier()
-
-        self.replication_client = None
-        if hs.config.worker.worker_app:
-            self.replication_client = hs.get_tcp_replication()
-
-    # A method just so we can pass 'self' as the authenticator to the Servlets
-    async def authenticate_request(self, request, content):
-        now = self._clock.time_msec()
-        json_request = {
-            "method": request.method.decode("ascii"),
-            "uri": request.uri.decode("ascii"),
-            "destination": self.server_name,
-            "signatures": {},
-        }
-
-        if content is not None:
-            json_request["content"] = content
-
-        origin = None
-
-        auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
-
-        if not auth_headers:
-            raise NoAuthenticationError(
-                401, "Missing Authorization headers", Codes.UNAUTHORIZED
-            )
-
-        for auth in auth_headers:
-            if auth.startswith(b"X-Matrix"):
-                (origin, key, sig) = _parse_auth_header(auth)
-                json_request["origin"] = origin
-                json_request["signatures"].setdefault(origin, {})[key] = sig
-
-        if (
-            self.federation_domain_whitelist is not None
-            and origin not in self.federation_domain_whitelist
-        ):
-            raise FederationDeniedError(origin)
-
-        if origin is None or not json_request["signatures"]:
-            raise NoAuthenticationError(
-                401, "Missing Authorization headers", Codes.UNAUTHORIZED
-            )
-
-        await self.keyring.verify_json_for_server(
-            origin, json_request, now, "Incoming request"
-        )
-
-        logger.debug("Request from %s", origin)
-        request.requester = origin
-
-        # If we get a valid signed request from the other side, its probably
-        # alive
-        retry_timings = await self.store.get_destination_retry_timings(origin)
-        if retry_timings and retry_timings["retry_last_ts"]:
-            run_in_background(self._reset_retry_timings, origin)
-
-        return origin
-
-    async def _reset_retry_timings(self, origin):
-        try:
-            logger.info("Marking origin %r as up", origin)
-            await self.store.set_destination_retry_timings(origin, None, 0, 0)
-
-            # Inform the relevant places that the remote server is back up.
-            self.notifier.notify_remote_server_up(origin)
-            if self.replication_client:
-                # If we're on a worker we try and inform master about this. The
-                # replication client doesn't hook into the notifier to avoid
-                # infinite loops where we send a `REMOTE_SERVER_UP` command to
-                # master, which then echoes it back to us which in turn pokes
-                # the notifier.
-                self.replication_client.send_remote_server_up(origin)
-
-        except Exception:
-            logger.exception("Error resetting retry timings on %s", origin)
-
-
-def _parse_auth_header(header_bytes):
-    """Parse an X-Matrix auth header
-
-    Args:
-        header_bytes (bytes): header value
-
-    Returns:
-        Tuple[str, str, str]: origin, key id, signature.
-
-    Raises:
-        AuthenticationError if the header could not be parsed
-    """
-    try:
-        header_str = header_bytes.decode("utf-8")
-        params = header_str.split(" ")[1].split(",")
-        param_dict = dict(kv.split("=") for kv in params)
-
-        def strip_quotes(value):
-            if value.startswith('"'):
-                return value[1:-1]
-            else:
-                return value
-
-        origin = strip_quotes(param_dict["origin"])
-
-        # ensure that the origin is a valid server name
-        parse_and_validate_server_name(origin)
-
-        key = strip_quotes(param_dict["key"])
-        sig = strip_quotes(param_dict["sig"])
-        return origin, key, sig
-    except Exception as e:
-        logger.warning(
-            "Error parsing auth header '%s': %s",
-            header_bytes.decode("ascii", "replace"),
-            e,
-        )
-        raise AuthenticationError(
-            400, "Malformed Authorization header", Codes.UNAUTHORIZED
-        )
-
-
-class BaseFederationServlet:
-    """Abstract base class for federation servlet classes.
-
-    The servlet object should have a PATH attribute which takes the form of a regexp to
-    match against the request path (excluding the /federation/v1 prefix).
-
-    The servlet should also implement one or more of on_GET, on_POST, on_PUT, to match
-    the appropriate HTTP method. These methods must be *asynchronous* and have the
-    signature:
-
-        on_<METHOD>(self, origin, content, query, **kwargs)
-
-        With arguments:
-
-            origin (unicode|None): The authenticated server_name of the calling server,
-                unless REQUIRE_AUTH is set to False and authentication failed.
-
-            content (unicode|None): decoded json body of the request. None if the
-                request was a GET.
-
-            query (dict[bytes, list[bytes]]): Query params from the request. url-decoded
-                (ie, '+' and '%xx' are decoded) but note that it is *not* utf8-decoded
-                yet.
-
-            **kwargs (dict[unicode, unicode]): the dict mapping keys to path
-                components as specified in the path match regexp.
-
-        Returns:
-            Optional[Tuple[int, object]]: either (response code, response object) to
-                 return a JSON response, or None if the request has already been handled.
-
-        Raises:
-            SynapseError: to return an error code
-
-            Exception: other exceptions will be caught, logged, and a 500 will be
-                returned.
-    """
-
-    PATH = ""  # Overridden in subclasses, the regex to match against the path.
-
-    REQUIRE_AUTH = True
-
-    PREFIX = FEDERATION_V1_PREFIX  # Allows specifying the API version
-
-    RATELIMIT = True  # Whether to rate limit requests or not
-
-    def __init__(self, handler, authenticator, ratelimiter, server_name):
-        self.handler = handler
-        self.authenticator = authenticator
-        self.ratelimiter = ratelimiter
-
-    def _wrap(self, func):
-        authenticator = self.authenticator
-        ratelimiter = self.ratelimiter
-
-        @functools.wraps(func)
-        async def new_func(request, *args, **kwargs):
-            """A callback which can be passed to HttpServer.RegisterPaths
-
-            Args:
-                request (twisted.web.http.Request):
-                *args: unused?
-                **kwargs (dict[unicode, unicode]): the dict mapping keys to path
-                    components as specified in the path match regexp.
-
-            Returns:
-                Tuple[int, object]|None: (response code, response object) as returned by
-                    the callback method. None if the request has already been handled.
-            """
-            content = None
-            if request.method in [b"PUT", b"POST"]:
-                # TODO: Handle other method types? other content types?
-                content = parse_json_object_from_request(request)
-
-            try:
-                origin = await authenticator.authenticate_request(request, content)
-            except NoAuthenticationError:
-                origin = None
-                if self.REQUIRE_AUTH:
-                    logger.warning(
-                        "authenticate_request failed: missing authentication"
-                    )
-                    raise
-            except Exception as e:
-                logger.warning("authenticate_request failed: %s", e)
-                raise
-
-            request_tags = {
-                "request_id": request.get_request_id(),
-                tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER,
-                tags.HTTP_METHOD: request.get_method(),
-                tags.HTTP_URL: request.get_redacted_uri(),
-                tags.PEER_HOST_IPV6: request.getClientIP(),
-                "authenticated_entity": origin,
-                "servlet_name": request.request_metrics.name,
-            }
-
-            # Only accept the span context if the origin is authenticated
-            # and whitelisted
-            if origin and whitelisted_homeserver(origin):
-                scope = start_active_span_from_request(
-                    request, "incoming-federation-request", tags=request_tags
-                )
-            else:
-                scope = start_active_span(
-                    "incoming-federation-request", tags=request_tags
-                )
-
-            with scope:
-                if origin and self.RATELIMIT:
-                    with ratelimiter.ratelimit(origin) as d:
-                        await d
-                        if request._disconnected:
-                            logger.warning(
-                                "client disconnected before we started processing "
-                                "request"
-                            )
-                            return -1, None
-                        response = await func(
-                            origin, content, request.args, *args, **kwargs
-                        )
-                else:
-                    response = await func(
-                        origin, content, request.args, *args, **kwargs
-                    )
-
-            return response
-
-        return new_func
-
-    def register(self, server):
-        pattern = re.compile("^" + self.PREFIX + self.PATH + "$")
-
-        for method in ("GET", "PUT", "POST"):
-            code = getattr(self, "on_%s" % (method), None)
-            if code is None:
-                continue
-
-            server.register_paths(
-                method,
-                (pattern,),
-                self._wrap(code),
-                self.__class__.__name__,
-            )
-
-
-class FederationSendServlet(BaseFederationServlet):
-    PATH = "/send/(?P<transaction_id>[^/]*)/?"
-
-    # We ratelimit manually in the handler as we queue up the requests and we
-    # don't want to fill up the ratelimiter with blocked requests.
-    RATELIMIT = False
-
-    def __init__(self, handler, server_name, **kwargs):
-        super().__init__(handler, server_name=server_name, **kwargs)
-        self.server_name = server_name
-
-    # This is when someone is trying to send us a bunch of data.
-    async def on_PUT(self, origin, content, query, transaction_id):
-        """Called on PUT /send/<transaction_id>/
-
-        Args:
-            request (twisted.web.http.Request): The HTTP request.
-            transaction_id (str): The transaction_id associated with this
-                request. This is *not* None.
-
-        Returns:
-            Tuple of `(code, response)`, where
-            `response` is a python dict to be converted into JSON that is
-            used as the response body.
-        """
-        # Parse the request
-        try:
-            transaction_data = content
-
-            logger.debug("Decoded %s: %s", transaction_id, str(transaction_data))
-
-            logger.info(
-                "Received txn %s from %s. (PDUs: %d, EDUs: %d)",
-                transaction_id,
-                origin,
-                len(transaction_data.get("pdus", [])),
-                len(transaction_data.get("edus", [])),
-            )
-
-            # We should ideally be getting this from the security layer.
-            # origin = body["origin"]
-
-            # Add some extra data to the transaction dict that isn't included
-            # in the request body.
-            transaction_data.update(
-                transaction_id=transaction_id, destination=self.server_name
-            )
-
-        except Exception as e:
-            logger.exception(e)
-            return 400, {"error": "Invalid transaction"}
-
-        try:
-            code, response = await self.handler.on_incoming_transaction(
-                origin, transaction_data
-            )
-        except Exception:
-            logger.exception("on_incoming_transaction failed")
-            raise
-
-        return code, response
-
-
-class FederationEventServlet(BaseFederationServlet):
-    PATH = "/event/(?P<event_id>[^/]*)/?"
-
-    # This is when someone asks for a data item for a given server data_id pair.
-    async def on_GET(self, origin, content, query, event_id):
-        return await self.handler.on_pdu_request(origin, event_id)
-
-
-class FederationStateV1Servlet(BaseFederationServlet):
-    PATH = "/state/(?P<room_id>[^/]*)/?"
-
-    # This is when someone asks for all data for a given room.
-    async def on_GET(self, origin, content, query, room_id):
-        return await self.handler.on_room_state_request(
-            origin,
-            room_id,
-            parse_string_from_args(query, "event_id", None, required=False),
-        )
-
-
-class FederationStateIdsServlet(BaseFederationServlet):
-    PATH = "/state_ids/(?P<room_id>[^/]*)/?"
-
-    async def on_GET(self, origin, content, query, room_id):
-        return await self.handler.on_state_ids_request(
-            origin,
-            room_id,
-            parse_string_from_args(query, "event_id", None, required=True),
-        )
-
-
-class FederationBackfillServlet(BaseFederationServlet):
-    PATH = "/backfill/(?P<room_id>[^/]*)/?"
-
-    async def on_GET(self, origin, content, query, room_id):
-        versions = [x.decode("ascii") for x in query[b"v"]]
-        limit = parse_integer_from_args(query, "limit", None)
-
-        if not limit:
-            return 400, {"error": "Did not include limit param"}
-
-        return await self.handler.on_backfill_request(origin, room_id, versions, limit)
-
-
-class FederationQueryServlet(BaseFederationServlet):
-    PATH = "/query/(?P<query_type>[^/]*)"
-
-    # This is when we receive a server-server Query
-    async def on_GET(self, origin, content, query, query_type):
-        args = {k.decode("utf8"): v[0].decode("utf-8") for k, v in query.items()}
-        args["origin"] = origin
-        return await self.handler.on_query_request(query_type, args)
-
-
-class FederationMakeJoinServlet(BaseFederationServlet):
-    PATH = "/make_join/(?P<room_id>[^/]*)/(?P<user_id>[^/]*)"
-
-    async def on_GET(self, origin, _content, query, room_id, user_id):
-        """
-        Args:
-            origin (unicode): The authenticated server_name of the calling server
-
-            _content (None): (GETs don't have bodies)
-
-            query (dict[bytes, list[bytes]]): Query params from the request.
-
-            **kwargs (dict[unicode, unicode]): the dict mapping keys to path
-                components as specified in the path match regexp.
-
-        Returns:
-            Tuple[int, object]: (response code, response object)
-        """
-        versions = query.get(b"ver")
-        if versions is not None:
-            supported_versions = [v.decode("utf-8") for v in versions]
-        else:
-            supported_versions = ["1"]
-
-        content = await self.handler.on_make_join_request(
-            origin, room_id, user_id, supported_versions=supported_versions
-        )
-        return 200, content
-
-
-class FederationMakeLeaveServlet(BaseFederationServlet):
-    PATH = "/make_leave/(?P<room_id>[^/]*)/(?P<user_id>[^/]*)"
-
-    async def on_GET(self, origin, content, query, room_id, user_id):
-        content = await self.handler.on_make_leave_request(origin, room_id, user_id)
-        return 200, content
-
-
-class FederationV1SendLeaveServlet(BaseFederationServlet):
-    PATH = "/send_leave/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
-
-    async def on_PUT(self, origin, content, query, room_id, event_id):
-        content = await self.handler.on_send_leave_request(origin, content)
-        return 200, (200, content)
-
-
-class FederationV2SendLeaveServlet(BaseFederationServlet):
-    PATH = "/send_leave/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
-
-    PREFIX = FEDERATION_V2_PREFIX
-
-    async def on_PUT(self, origin, content, query, room_id, event_id):
-        content = await self.handler.on_send_leave_request(origin, content)
-        return 200, content
-
-
-class FederationMakeKnockServlet(BaseFederationServlet):
-    PATH = "/make_knock/(?P<room_id>[^/]*)/(?P<user_id>[^/]*)"
-
-    PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock"
-
-    async def on_GET(self, origin, content, query, room_id, user_id):
-        try:
-            # Retrieve the room versions the remote homeserver claims to support
-            supported_versions = parse_list_from_args(query, "ver", encoding="utf-8")
-        except KeyError:
-            raise SynapseError(400, "Missing required query parameter 'ver'")
-
-        content = await self.handler.on_make_knock_request(
-            origin, room_id, user_id, supported_versions=supported_versions
-        )
-        return 200, content
-
-
-class FederationV2SendKnockServlet(BaseFederationServlet):
-    PATH = "/send_knock/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
-
-    PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock"
-
-    async def on_PUT(self, origin, content, query, room_id, event_id):
-        content = await self.handler.on_send_knock_request(origin, content, room_id)
-        return 200, content
-
-
-class FederationEventAuthServlet(BaseFederationServlet):
-    PATH = "/event_auth/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
-
-    async def on_GET(self, origin, content, query, room_id, event_id):
-        return await self.handler.on_event_auth(origin, room_id, event_id)
-
-
-class FederationV1SendJoinServlet(BaseFederationServlet):
-    PATH = "/send_join/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
-
-    async def on_PUT(self, origin, content, query, room_id, event_id):
-        # TODO(paul): assert that room_id/event_id parsed from path actually
-        #   match those given in content
-        content = await self.handler.on_send_join_request(origin, content)
-        return 200, (200, content)
-
-
-class FederationV2SendJoinServlet(BaseFederationServlet):
-    PATH = "/send_join/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
-
-    PREFIX = FEDERATION_V2_PREFIX
-
-    async def on_PUT(self, origin, content, query, room_id, event_id):
-        # TODO(paul): assert that room_id/event_id parsed from path actually
-        #   match those given in content
-        content = await self.handler.on_send_join_request(origin, content)
-        return 200, content
-
-
-class FederationV1InviteServlet(BaseFederationServlet):
-    PATH = "/invite/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
-
-    async def on_PUT(self, origin, content, query, room_id, event_id):
-        # We don't get a room version, so we have to assume its EITHER v1 or
-        # v2. This is "fine" as the only difference between V1 and V2 is the
-        # state resolution algorithm, and we don't use that for processing
-        # invites
-        content = await self.handler.on_invite_request(
-            origin, content, room_version_id=RoomVersions.V1.identifier
-        )
-
-        # V1 federation API is defined to return a content of `[200, {...}]`
-        # due to a historical bug.
-        return 200, (200, content)
-
-
-class FederationV2InviteServlet(BaseFederationServlet):
-    PATH = "/invite/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
-
-    PREFIX = FEDERATION_V2_PREFIX
-
-    async def on_PUT(self, origin, content, query, room_id, event_id):
-        # TODO(paul): assert that room_id/event_id parsed from path actually
-        #   match those given in content
-
-        room_version = content["room_version"]
-        event = content["event"]
-        invite_room_state = content["invite_room_state"]
-
-        # Synapse expects invite_room_state to be in unsigned, as it is in v1
-        # API
-
-        event.setdefault("unsigned", {})["invite_room_state"] = invite_room_state
-
-        content = await self.handler.on_invite_request(
-            origin, event, room_version_id=room_version
-        )
-        return 200, content
-
-
-class FederationThirdPartyInviteExchangeServlet(BaseFederationServlet):
-    PATH = "/exchange_third_party_invite/(?P<room_id>[^/]*)"
-
-    async def on_PUT(self, origin, content, query, room_id):
-        content = await self.handler.on_exchange_third_party_invite_request(content)
-        return 200, content
-
-
-class FederationClientKeysQueryServlet(BaseFederationServlet):
-    PATH = "/user/keys/query"
-
-    async def on_POST(self, origin, content, query):
-        return await self.handler.on_query_client_keys(origin, content)
-
-
-class FederationUserDevicesQueryServlet(BaseFederationServlet):
-    PATH = "/user/devices/(?P<user_id>[^/]*)"
-
-    async def on_GET(self, origin, content, query, user_id):
-        return await self.handler.on_query_user_devices(origin, user_id)
-
-
-class FederationClientKeysClaimServlet(BaseFederationServlet):
-    PATH = "/user/keys/claim"
-
-    async def on_POST(self, origin, content, query):
-        response = await self.handler.on_claim_client_keys(origin, content)
-        return 200, response
-
-
-class FederationGetMissingEventsServlet(BaseFederationServlet):
-    # TODO(paul): Why does this path alone end with "/?" optional?
-    PATH = "/get_missing_events/(?P<room_id>[^/]*)/?"
-
-    async def on_POST(self, origin, content, query, room_id):
-        limit = int(content.get("limit", 10))
-        earliest_events = content.get("earliest_events", [])
-        latest_events = content.get("latest_events", [])
-
-        content = await self.handler.on_get_missing_events(
-            origin,
-            room_id=room_id,
-            earliest_events=earliest_events,
-            latest_events=latest_events,
-            limit=limit,
-        )
-
-        return 200, content
-
-
-class On3pidBindServlet(BaseFederationServlet):
-    PATH = "/3pid/onbind"
-
-    REQUIRE_AUTH = False
-
-    async def on_POST(self, origin, content, query):
-        if "invites" in content:
-            last_exception = None
-            for invite in content["invites"]:
-                try:
-                    if "signed" not in invite or "token" not in invite["signed"]:
-                        message = (
-                            "Rejecting received notification of third-"
-                            "party invite without signed: %s" % (invite,)
-                        )
-                        logger.info(message)
-                        raise SynapseError(400, message)
-                    await self.handler.exchange_third_party_invite(
-                        invite["sender"],
-                        invite["mxid"],
-                        invite["room_id"],
-                        invite["signed"],
-                    )
-                except Exception as e:
-                    last_exception = e
-            if last_exception:
-                raise last_exception
-        return 200, {}
-
-
-class OpenIdUserInfo(BaseFederationServlet):
-    """
-    Exchange a bearer token for information about a user.
-
-    The response format should be compatible with:
-        http://openid.net/specs/openid-connect-core-1_0.html#UserInfoResponse
-
-    GET /openid/userinfo?access_token=ABDEFGH HTTP/1.1
-
-    HTTP/1.1 200 OK
-    Content-Type: application/json
-
-    {
-        "sub": "@userpart:example.org",
-    }
-    """
-
-    PATH = "/openid/userinfo"
-
-    REQUIRE_AUTH = False
-
-    async def on_GET(self, origin, content, query):
-        token = query.get(b"access_token", [None])[0]
-        if token is None:
-            return (
-                401,
-                {"errcode": "M_MISSING_TOKEN", "error": "Access Token required"},
-            )
-
-        user_id = await self.handler.on_openid_userinfo(token.decode("ascii"))
-
-        if user_id is None:
-            return (
-                401,
-                {
-                    "errcode": "M_UNKNOWN_TOKEN",
-                    "error": "Access Token unknown or expired",
-                },
-            )
-
-        return 200, {"sub": user_id}
-
-
-class PublicRoomList(BaseFederationServlet):
-    """
-    Fetch the public room list for this server.
-
-    This API returns information in the same format as /publicRooms on the
-    client API, but will only ever include local public rooms and hence is
-    intended for consumption by other homeservers.
-
-    GET /publicRooms HTTP/1.1
-
-    HTTP/1.1 200 OK
-    Content-Type: application/json
-
-    {
-        "chunk": [
-            {
-                "aliases": [
-                    "#test:localhost"
-                ],
-                "guest_can_join": false,
-                "name": "test room",
-                "num_joined_members": 3,
-                "room_id": "!whkydVegtvatLfXmPN:localhost",
-                "world_readable": false
-            }
-        ],
-        "end": "END",
-        "start": "START"
-    }
-    """
-
-    PATH = "/publicRooms"
-
-    def __init__(self, handler, authenticator, ratelimiter, server_name, allow_access):
-        super().__init__(handler, authenticator, ratelimiter, server_name)
-        self.allow_access = allow_access
-
-    async def on_GET(self, origin, content, query):
-        if not self.allow_access:
-            raise FederationDeniedError(origin)
-
-        limit = parse_integer_from_args(query, "limit", 0)
-        since_token = parse_string_from_args(query, "since", None)
-        include_all_networks = parse_boolean_from_args(
-            query, "include_all_networks", False
-        )
-        third_party_instance_id = parse_string_from_args(
-            query, "third_party_instance_id", None
-        )
-
-        if include_all_networks:
-            network_tuple = None
-        elif third_party_instance_id:
-            network_tuple = ThirdPartyInstanceID.from_string(third_party_instance_id)
-        else:
-            network_tuple = ThirdPartyInstanceID(None, None)
-
-        if limit == 0:
-            # zero is a special value which corresponds to no limit.
-            limit = None
-
-        data = await self.handler.get_local_public_room_list(
-            limit, since_token, network_tuple=network_tuple, from_federation=True
-        )
-        return 200, data
-
-    async def on_POST(self, origin, content, query):
-        # This implements MSC2197 (Search Filtering over Federation)
-        if not self.allow_access:
-            raise FederationDeniedError(origin)
-
-        limit = int(content.get("limit", 100))  # type: Optional[int]
-        since_token = content.get("since", None)
-        search_filter = content.get("filter", None)
-
-        include_all_networks = content.get("include_all_networks", False)
-        third_party_instance_id = content.get("third_party_instance_id", None)
-
-        if include_all_networks:
-            network_tuple = None
-            if third_party_instance_id is not None:
-                raise SynapseError(
-                    400, "Can't use include_all_networks with an explicit network"
-                )
-        elif third_party_instance_id is None:
-            network_tuple = ThirdPartyInstanceID(None, None)
-        else:
-            network_tuple = ThirdPartyInstanceID.from_string(third_party_instance_id)
-
-        if search_filter is None:
-            logger.warning("Nonefilter")
-
-        if limit == 0:
-            # zero is a special value which corresponds to no limit.
-            limit = None
-
-        data = await self.handler.get_local_public_room_list(
-            limit=limit,
-            since_token=since_token,
-            search_filter=search_filter,
-            network_tuple=network_tuple,
-            from_federation=True,
-        )
-
-        return 200, data
-
-
-class FederationUserInfoServlet(BaseFederationServlet):
-    """
-    Return information about a set of users.
-
-    This API returns expiration and deactivation information about a set of
-    users. Requested users not local to this homeserver will be ignored.
-
-    Example request:
-        POST /users/info
-
-        {
-            "user_ids": [
-                "@alice:example.com",
-                "@bob:example.com"
-            ]
-        }
-
-    Example response
-        {
-            "@alice:example.com": {
-                "expired": false,
-                "deactivated": true
-            }
-        }
-    """
-
-    PATH = "/users/info"
-    PREFIX = FEDERATION_UNSTABLE_PREFIX
-
-    def __init__(self, handler, authenticator, ratelimiter, server_name):
-        super(FederationUserInfoServlet, self).__init__(
-            handler, authenticator, ratelimiter, server_name
-        )
-        self.handler = handler
-
-    async def on_POST(self, origin, content, query):
-        assert_params_in_dict(content, required=["user_ids"])
-
-        user_ids = content.get("user_ids", [])
-
-        if not isinstance(user_ids, list):
-            raise SynapseError(
-                400,
-                "'user_ids' must be a list of user ID strings",
-                errcode=Codes.INVALID_PARAM,
-            )
-
-        data = await self.handler.store.get_info_for_users(user_ids)
-        return 200, data
-
-
-class FederationVersionServlet(BaseFederationServlet):
-    PATH = "/version"
-
-    REQUIRE_AUTH = False
-
-    async def on_GET(self, origin, content, query):
-        return (
-            200,
-            {"server": {"name": "Synapse", "version": get_version_string(synapse)}},
-        )
-
-
-class FederationGroupsProfileServlet(BaseFederationServlet):
-    """Get/set the basic profile of a group on behalf of a user"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/profile"
-
-    async def on_GET(self, origin, content, query, group_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.get_group_profile(group_id, requester_user_id)
-
-        return 200, new_content
-
-    async def on_POST(self, origin, content, query, group_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.update_group_profile(
-            group_id, requester_user_id, content
-        )
-
-        return 200, new_content
-
-
-class FederationGroupsSummaryServlet(BaseFederationServlet):
-    PATH = "/groups/(?P<group_id>[^/]*)/summary"
-
-    async def on_GET(self, origin, content, query, group_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.get_group_summary(group_id, requester_user_id)
-
-        return 200, new_content
-
-
-class FederationGroupsRoomsServlet(BaseFederationServlet):
-    """Get the rooms in a group on behalf of a user"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/rooms"
-
-    async def on_GET(self, origin, content, query, group_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.get_rooms_in_group(group_id, requester_user_id)
-
-        return 200, new_content
-
-
-class FederationGroupsAddRoomsServlet(BaseFederationServlet):
-    """Add/remove room from group"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/room/(?P<room_id>[^/]*)"
-
-    async def on_POST(self, origin, content, query, group_id, room_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.add_room_to_group(
-            group_id, requester_user_id, room_id, content
-        )
-
-        return 200, new_content
-
-    async def on_DELETE(self, origin, content, query, group_id, room_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.remove_room_from_group(
-            group_id, requester_user_id, room_id
-        )
-
-        return 200, new_content
-
-
-class FederationGroupsAddRoomsConfigServlet(BaseFederationServlet):
-    """Update room config in group"""
-
-    PATH = (
-        "/groups/(?P<group_id>[^/]*)/room/(?P<room_id>[^/]*)"
-        "/config/(?P<config_key>[^/]*)"
-    )
-
-    async def on_POST(self, origin, content, query, group_id, room_id, config_key):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        result = await self.handler.update_room_in_group(
-            group_id, requester_user_id, room_id, config_key, content
-        )
-
-        return 200, result
-
-
-class FederationGroupsUsersServlet(BaseFederationServlet):
-    """Get the users in a group on behalf of a user"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/users"
-
-    async def on_GET(self, origin, content, query, group_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.get_users_in_group(group_id, requester_user_id)
-
-        return 200, new_content
-
-
-class FederationGroupsInvitedUsersServlet(BaseFederationServlet):
-    """Get the users that have been invited to a group"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/invited_users"
-
-    async def on_GET(self, origin, content, query, group_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.get_invited_users_in_group(
-            group_id, requester_user_id
-        )
-
-        return 200, new_content
-
-
-class FederationGroupsInviteServlet(BaseFederationServlet):
-    """Ask a group server to invite someone to the group"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite"
-
-    async def on_POST(self, origin, content, query, group_id, user_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.invite_to_group(
-            group_id, user_id, requester_user_id, content
-        )
-
-        return 200, new_content
-
-
-class FederationGroupsAcceptInviteServlet(BaseFederationServlet):
-    """Accept an invitation from the group server"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/accept_invite"
-
-    async def on_POST(self, origin, content, query, group_id, user_id):
-        if get_domain_from_id(user_id) != origin:
-            raise SynapseError(403, "user_id doesn't match origin")
-
-        new_content = await self.handler.accept_invite(group_id, user_id, content)
-
-        return 200, new_content
-
-
-class FederationGroupsJoinServlet(BaseFederationServlet):
-    """Attempt to join a group"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/join"
-
-    async def on_POST(self, origin, content, query, group_id, user_id):
-        if get_domain_from_id(user_id) != origin:
-            raise SynapseError(403, "user_id doesn't match origin")
-
-        new_content = await self.handler.join_group(group_id, user_id, content)
-
-        return 200, new_content
-
-
-class FederationGroupsRemoveUserServlet(BaseFederationServlet):
-    """Leave or kick a user from the group"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove"
-
-    async def on_POST(self, origin, content, query, group_id, user_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.remove_user_from_group(
-            group_id, user_id, requester_user_id, content
-        )
-
-        return 200, new_content
-
-
-class FederationGroupsLocalInviteServlet(BaseFederationServlet):
-    """A group server has invited a local user"""
-
-    PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite"
-
-    async def on_POST(self, origin, content, query, group_id, user_id):
-        if get_domain_from_id(group_id) != origin:
-            raise SynapseError(403, "group_id doesn't match origin")
-
-        new_content = await self.handler.on_invite(group_id, user_id, content)
-
-        return 200, new_content
-
-
-class FederationGroupsRemoveLocalUserServlet(BaseFederationServlet):
-    """A group server has removed a local user"""
-
-    PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove"
-
-    async def on_POST(self, origin, content, query, group_id, user_id):
-        if get_domain_from_id(group_id) != origin:
-            raise SynapseError(403, "user_id doesn't match origin")
-
-        new_content = await self.handler.user_removed_from_group(
-            group_id, user_id, content
-        )
-
-        return 200, new_content
-
-
-class FederationGroupsRenewAttestaionServlet(BaseFederationServlet):
-    """A group or user's server renews their attestation"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/renew_attestation/(?P<user_id>[^/]*)"
-
-    async def on_POST(self, origin, content, query, group_id, user_id):
-        # We don't need to check auth here as we check the attestation signatures
-
-        new_content = await self.handler.on_renew_attestation(
-            group_id, user_id, content
-        )
-
-        return 200, new_content
-
-
-class FederationGroupsSummaryRoomsServlet(BaseFederationServlet):
-    """Add/remove a room from the group summary, with optional category.
-
-    Matches both:
-        - /groups/:group/summary/rooms/:room_id
-        - /groups/:group/summary/categories/:category/rooms/:room_id
-    """
-
-    PATH = (
-        "/groups/(?P<group_id>[^/]*)/summary"
-        "(/categories/(?P<category_id>[^/]+))?"
-        "/rooms/(?P<room_id>[^/]*)"
-    )
-
-    async def on_POST(self, origin, content, query, group_id, category_id, room_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        if category_id == "":
-            raise SynapseError(
-                400, "category_id cannot be empty string", Codes.INVALID_PARAM
-            )
-
-        if len(category_id) > MAX_GROUP_CATEGORYID_LENGTH:
-            raise SynapseError(
-                400,
-                "category_id may not be longer than %s characters"
-                % (MAX_GROUP_CATEGORYID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
-        resp = await self.handler.update_group_summary_room(
-            group_id,
-            requester_user_id,
-            room_id=room_id,
-            category_id=category_id,
-            content=content,
-        )
-
-        return 200, resp
-
-    async def on_DELETE(self, origin, content, query, group_id, category_id, room_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        if category_id == "":
-            raise SynapseError(400, "category_id cannot be empty string")
-
-        resp = await self.handler.delete_group_summary_room(
-            group_id, requester_user_id, room_id=room_id, category_id=category_id
-        )
-
-        return 200, resp
-
-
-class FederationGroupsCategoriesServlet(BaseFederationServlet):
-    """Get all categories for a group"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/categories/?"
-
-    async def on_GET(self, origin, content, query, group_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        resp = await self.handler.get_group_categories(group_id, requester_user_id)
-
-        return 200, resp
-
-
-class FederationGroupsCategoryServlet(BaseFederationServlet):
-    """Add/remove/get a category in a group"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/categories/(?P<category_id>[^/]+)"
-
-    async def on_GET(self, origin, content, query, group_id, category_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        resp = await self.handler.get_group_category(
-            group_id, requester_user_id, category_id
-        )
-
-        return 200, resp
-
-    async def on_POST(self, origin, content, query, group_id, category_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        if category_id == "":
-            raise SynapseError(400, "category_id cannot be empty string")
-
-        if len(category_id) > MAX_GROUP_CATEGORYID_LENGTH:
-            raise SynapseError(
-                400,
-                "category_id may not be longer than %s characters"
-                % (MAX_GROUP_CATEGORYID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
-        resp = await self.handler.upsert_group_category(
-            group_id, requester_user_id, category_id, content
-        )
-
-        return 200, resp
-
-    async def on_DELETE(self, origin, content, query, group_id, category_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        if category_id == "":
-            raise SynapseError(400, "category_id cannot be empty string")
-
-        resp = await self.handler.delete_group_category(
-            group_id, requester_user_id, category_id
-        )
-
-        return 200, resp
-
-
-class FederationGroupsRolesServlet(BaseFederationServlet):
-    """Get roles in a group"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/roles/?"
-
-    async def on_GET(self, origin, content, query, group_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        resp = await self.handler.get_group_roles(group_id, requester_user_id)
-
-        return 200, resp
-
-
-class FederationGroupsRoleServlet(BaseFederationServlet):
-    """Add/remove/get a role in a group"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/roles/(?P<role_id>[^/]+)"
-
-    async def on_GET(self, origin, content, query, group_id, role_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        resp = await self.handler.get_group_role(group_id, requester_user_id, role_id)
-
-        return 200, resp
-
-    async def on_POST(self, origin, content, query, group_id, role_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        if role_id == "":
-            raise SynapseError(
-                400, "role_id cannot be empty string", Codes.INVALID_PARAM
-            )
-
-        if len(role_id) > MAX_GROUP_ROLEID_LENGTH:
-            raise SynapseError(
-                400,
-                "role_id may not be longer than %s characters"
-                % (MAX_GROUP_ROLEID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
-        resp = await self.handler.update_group_role(
-            group_id, requester_user_id, role_id, content
-        )
-
-        return 200, resp
-
-    async def on_DELETE(self, origin, content, query, group_id, role_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        if role_id == "":
-            raise SynapseError(400, "role_id cannot be empty string")
-
-        resp = await self.handler.delete_group_role(
-            group_id, requester_user_id, role_id
-        )
-
-        return 200, resp
-
-
-class FederationGroupsSummaryUsersServlet(BaseFederationServlet):
-    """Add/remove a user from the group summary, with optional role.
-
-    Matches both:
-        - /groups/:group/summary/users/:user_id
-        - /groups/:group/summary/roles/:role/users/:user_id
-    """
-
-    PATH = (
-        "/groups/(?P<group_id>[^/]*)/summary"
-        "(/roles/(?P<role_id>[^/]+))?"
-        "/users/(?P<user_id>[^/]*)"
-    )
-
-    async def on_POST(self, origin, content, query, group_id, role_id, user_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        if role_id == "":
-            raise SynapseError(400, "role_id cannot be empty string")
-
-        if len(role_id) > MAX_GROUP_ROLEID_LENGTH:
-            raise SynapseError(
-                400,
-                "role_id may not be longer than %s characters"
-                % (MAX_GROUP_ROLEID_LENGTH,),
-                Codes.INVALID_PARAM,
-            )
-
-        resp = await self.handler.update_group_summary_user(
-            group_id,
-            requester_user_id,
-            user_id=user_id,
-            role_id=role_id,
-            content=content,
-        )
-
-        return 200, resp
-
-    async def on_DELETE(self, origin, content, query, group_id, role_id, user_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        if role_id == "":
-            raise SynapseError(400, "role_id cannot be empty string")
-
-        resp = await self.handler.delete_group_summary_user(
-            group_id, requester_user_id, user_id=user_id, role_id=role_id
-        )
-
-        return 200, resp
-
-
-class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
-    """Get roles in a group"""
-
-    PATH = "/get_groups_publicised"
-
-    async def on_POST(self, origin, content, query):
-        resp = await self.handler.bulk_get_publicised_groups(
-            content["user_ids"], proxy=False
-        )
-
-        return 200, resp
-
-
-class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet):
-    """Sets whether a group is joinable without an invite or knock"""
-
-    PATH = "/groups/(?P<group_id>[^/]*)/settings/m.join_policy"
-
-    async def on_PUT(self, origin, content, query, group_id):
-        requester_user_id = parse_string_from_args(query, "requester_user_id")
-        if get_domain_from_id(requester_user_id) != origin:
-            raise SynapseError(403, "requester_user_id doesn't match origin")
-
-        new_content = await self.handler.set_group_join_policy(
-            group_id, requester_user_id, content
-        )
-
-        return 200, new_content
-
-
-class FederationSpaceSummaryServlet(BaseFederationServlet):
-    PREFIX = FEDERATION_UNSTABLE_PREFIX + "/org.matrix.msc2946"
-    PATH = "/spaces/(?P<room_id>[^/]*)"
-
-    async def on_POST(
-        self,
-        origin: str,
-        content: JsonDict,
-        query: Mapping[bytes, Sequence[bytes]],
-        room_id: str,
-    ) -> Tuple[int, JsonDict]:
-        suggested_only = content.get("suggested_only", False)
-        if not isinstance(suggested_only, bool):
-            raise SynapseError(
-                400, "'suggested_only' must be a boolean", Codes.BAD_JSON
-            )
-
-        exclude_rooms = content.get("exclude_rooms", [])
-        if not isinstance(exclude_rooms, list) or any(
-            not isinstance(x, str) for x in exclude_rooms
-        ):
-            raise SynapseError(400, "bad value for 'exclude_rooms'", Codes.BAD_JSON)
-
-        max_rooms_per_space = content.get("max_rooms_per_space")
-        if max_rooms_per_space is not None and not isinstance(max_rooms_per_space, int):
-            raise SynapseError(
-                400, "bad value for 'max_rooms_per_space'", Codes.BAD_JSON
-            )
-
-        return 200, await self.handler.federation_space_summary(
-            room_id, suggested_only, max_rooms_per_space, exclude_rooms
-        )
-
-
-class RoomComplexityServlet(BaseFederationServlet):
-    """
-    Indicates to other servers how complex (and therefore likely
-    resource-intensive) a public room this server knows about is.
-    """
-
-    PATH = "/rooms/(?P<room_id>[^/]*)/complexity"
-    PREFIX = FEDERATION_UNSTABLE_PREFIX
-
-    async def on_GET(self, origin, content, query, room_id):
-
-        store = self.handler.hs.get_datastore()
-
-        is_public = await store.is_room_world_readable_or_publicly_joinable(room_id)
-
-        if not is_public:
-            raise SynapseError(404, "Room not found", errcode=Codes.INVALID_PARAM)
-
-        complexity = await store.get_room_complexity(room_id)
-        return 200, complexity
-
-
-FEDERATION_SERVLET_CLASSES = (
-    FederationSendServlet,
-    FederationEventServlet,
-    FederationStateV1Servlet,
-    FederationStateIdsServlet,
-    FederationBackfillServlet,
-    FederationQueryServlet,
-    FederationMakeJoinServlet,
-    FederationMakeLeaveServlet,
-    FederationMakeKnockServlet,
-    FederationEventServlet,
-    FederationV1SendJoinServlet,
-    FederationV2SendJoinServlet,
-    FederationV1SendLeaveServlet,
-    FederationV2SendLeaveServlet,
-    FederationV2SendKnockServlet,
-    FederationV1InviteServlet,
-    FederationV2InviteServlet,
-    FederationGetMissingEventsServlet,
-    FederationEventAuthServlet,
-    FederationClientKeysQueryServlet,
-    FederationUserDevicesQueryServlet,
-    FederationClientKeysClaimServlet,
-    FederationThirdPartyInviteExchangeServlet,
-    On3pidBindServlet,
-    FederationVersionServlet,
-    RoomComplexityServlet,
-    FederationUserInfoServlet,
-)  # type: Tuple[Type[BaseFederationServlet], ...]
-
-OPENID_SERVLET_CLASSES = (
-    OpenIdUserInfo,
-)  # type: Tuple[Type[BaseFederationServlet], ...]
-
-ROOM_LIST_CLASSES = (PublicRoomList,)  # type: Tuple[Type[PublicRoomList], ...]
-
-GROUP_SERVER_SERVLET_CLASSES = (
-    FederationGroupsProfileServlet,
-    FederationGroupsSummaryServlet,
-    FederationGroupsRoomsServlet,
-    FederationGroupsUsersServlet,
-    FederationGroupsInvitedUsersServlet,
-    FederationGroupsInviteServlet,
-    FederationGroupsAcceptInviteServlet,
-    FederationGroupsJoinServlet,
-    FederationGroupsRemoveUserServlet,
-    FederationGroupsSummaryRoomsServlet,
-    FederationGroupsCategoriesServlet,
-    FederationGroupsCategoryServlet,
-    FederationGroupsRolesServlet,
-    FederationGroupsRoleServlet,
-    FederationGroupsSummaryUsersServlet,
-    FederationGroupsAddRoomsServlet,
-    FederationGroupsAddRoomsConfigServlet,
-    FederationGroupsSettingJoinPolicyServlet,
-)  # type: Tuple[Type[BaseFederationServlet], ...]
-
-
-GROUP_LOCAL_SERVLET_CLASSES = (
-    FederationGroupsLocalInviteServlet,
-    FederationGroupsRemoveLocalUserServlet,
-    FederationGroupsBulkPublicisedServlet,
-)  # type: Tuple[Type[BaseFederationServlet], ...]
-
-
-GROUP_ATTESTATION_SERVLET_CLASSES = (
-    FederationGroupsRenewAttestaionServlet,
-)  # type: Tuple[Type[BaseFederationServlet], ...]
-
-DEFAULT_SERVLET_GROUPS = (
-    "federation",
-    "room_list",
-    "group_server",
-    "group_local",
-    "group_attestation",
-    "openid",
-)
-
-
-def register_servlets(
-    hs: HomeServer,
-    resource: HttpServer,
-    authenticator: Authenticator,
-    ratelimiter: FederationRateLimiter,
-    servlet_groups: Optional[Container[str]] = None,
-):
-    """Initialize and register servlet classes.
-
-    Will by default register all servlets. For custom behaviour, pass in
-    a list of servlet_groups to register.
-
-    Args:
-        hs: homeserver
-        resource: resource class to register to
-        authenticator: authenticator to use
-        ratelimiter: ratelimiter to use
-        servlet_groups: List of servlet groups to register.
-            Defaults to ``DEFAULT_SERVLET_GROUPS``.
-    """
-    if not servlet_groups:
-        servlet_groups = DEFAULT_SERVLET_GROUPS
-
-    if "federation" in servlet_groups:
-        for servletclass in FEDERATION_SERVLET_CLASSES:
-            servletclass(
-                handler=hs.get_federation_server(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
-
-        if hs.config.experimental.spaces_enabled:
-            FederationSpaceSummaryServlet(
-                handler=hs.get_space_summary_handler(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
-
-    if "openid" in servlet_groups:
-        for servletclass in OPENID_SERVLET_CLASSES:
-            servletclass(
-                handler=hs.get_federation_server(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
-
-    if "room_list" in servlet_groups:
-        for servletclass in ROOM_LIST_CLASSES:
-            servletclass(
-                handler=hs.get_room_list_handler(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-                allow_access=hs.config.allow_public_rooms_over_federation,
-            ).register(resource)
-
-    if "group_server" in servlet_groups:
-        for servletclass in GROUP_SERVER_SERVLET_CLASSES:
-            servletclass(
-                handler=hs.get_groups_server_handler(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
-
-    if "group_local" in servlet_groups:
-        for servletclass in GROUP_LOCAL_SERVLET_CLASSES:
-            servletclass(
-                handler=hs.get_groups_local_handler(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
-
-    if "group_attestation" in servlet_groups:
-        for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES:
-            servletclass(
-                handler=hs.get_groups_attestation_renewer(),
-                authenticator=authenticator,
-                ratelimiter=ratelimiter,
-                server_name=hs.hostname,
-            ).register(resource)
diff --git a/synapse/federation/transport/server/__init__.py b/synapse/federation/transport/server/__init__.py
new file mode 100644
index 0000000000..95176ba6f9
--- /dev/null
+++ b/synapse/federation/transport/server/__init__.py
@@ -0,0 +1,332 @@
+# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
+# Copyright 2020 Sorunome
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from typing import Dict, Iterable, List, Optional, Tuple, Type
+
+from typing_extensions import Literal
+
+from synapse.api.errors import FederationDeniedError, SynapseError
+from synapse.federation.transport.server._base import (
+    Authenticator,
+    BaseFederationServlet,
+)
+from synapse.federation.transport.server.federation import FEDERATION_SERVLET_CLASSES
+from synapse.federation.transport.server.groups_local import GROUP_LOCAL_SERVLET_CLASSES
+from synapse.federation.transport.server.groups_server import (
+    GROUP_SERVER_SERVLET_CLASSES,
+)
+from synapse.http.server import HttpServer, JsonResource
+from synapse.http.servlet import (
+    parse_boolean_from_args,
+    parse_integer_from_args,
+    parse_string_from_args,
+)
+from synapse.server import HomeServer
+from synapse.types import JsonDict, ThirdPartyInstanceID
+from synapse.util.ratelimitutils import FederationRateLimiter
+
+logger = logging.getLogger(__name__)
+
+
+class TransportLayerServer(JsonResource):
+    """Handles incoming federation HTTP requests"""
+
+    def __init__(self, hs: HomeServer, servlet_groups: Optional[List[str]] = None):
+        """Initialize the TransportLayerServer
+
+        Will by default register all servlets. For custom behaviour, pass in
+        a list of servlet_groups to register.
+
+        Args:
+            hs: homeserver
+            servlet_groups: List of servlet groups to register.
+                Defaults to ``DEFAULT_SERVLET_GROUPS``.
+        """
+        self.hs = hs
+        self.clock = hs.get_clock()
+        self.servlet_groups = servlet_groups
+
+        super().__init__(hs, canonical_json=False)
+
+        self.authenticator = Authenticator(hs)
+        self.ratelimiter = hs.get_federation_ratelimiter()
+
+        self.register_servlets()
+
+    def register_servlets(self) -> None:
+        register_servlets(
+            self.hs,
+            resource=self,
+            ratelimiter=self.ratelimiter,
+            authenticator=self.authenticator,
+            servlet_groups=self.servlet_groups,
+        )
+
+
+class PublicRoomList(BaseFederationServlet):
+    """
+    Fetch the public room list for this server.
+
+    This API returns information in the same format as /publicRooms on the
+    client API, but will only ever include local public rooms and hence is
+    intended for consumption by other homeservers.
+
+    GET /publicRooms HTTP/1.1
+
+    HTTP/1.1 200 OK
+    Content-Type: application/json
+
+    {
+        "chunk": [
+            {
+                "aliases": [
+                    "#test:localhost"
+                ],
+                "guest_can_join": false,
+                "name": "test room",
+                "num_joined_members": 3,
+                "room_id": "!whkydVegtvatLfXmPN:localhost",
+                "world_readable": false
+            }
+        ],
+        "end": "END",
+        "start": "START"
+    }
+    """
+
+    PATH = "/publicRooms"
+
+    def __init__(
+        self,
+        hs: HomeServer,
+        authenticator: Authenticator,
+        ratelimiter: FederationRateLimiter,
+        server_name: str,
+    ):
+        super().__init__(hs, authenticator, ratelimiter, server_name)
+        self.handler = hs.get_room_list_handler()
+        self.allow_access = hs.config.allow_public_rooms_over_federation
+
+    async def on_GET(
+        self, origin: str, content: Literal[None], query: Dict[bytes, List[bytes]]
+    ) -> Tuple[int, JsonDict]:
+        if not self.allow_access:
+            raise FederationDeniedError(origin)
+
+        limit = parse_integer_from_args(query, "limit", 0)
+        since_token = parse_string_from_args(query, "since", None)
+        include_all_networks = parse_boolean_from_args(
+            query, "include_all_networks", default=False
+        )
+        third_party_instance_id = parse_string_from_args(
+            query, "third_party_instance_id", None
+        )
+
+        if include_all_networks:
+            network_tuple = None
+        elif third_party_instance_id:
+            network_tuple = ThirdPartyInstanceID.from_string(third_party_instance_id)
+        else:
+            network_tuple = ThirdPartyInstanceID(None, None)
+
+        if limit == 0:
+            # zero is a special value which corresponds to no limit.
+            limit = None
+
+        data = await self.handler.get_local_public_room_list(
+            limit, since_token, network_tuple=network_tuple, from_federation=True
+        )
+        return 200, data
+
+    async def on_POST(
+        self, origin: str, content: JsonDict, query: Dict[bytes, List[bytes]]
+    ) -> Tuple[int, JsonDict]:
+        # This implements MSC2197 (Search Filtering over Federation)
+        if not self.allow_access:
+            raise FederationDeniedError(origin)
+
+        limit: Optional[int] = int(content.get("limit", 100))
+        since_token = content.get("since", None)
+        search_filter = content.get("filter", None)
+
+        include_all_networks = content.get("include_all_networks", False)
+        third_party_instance_id = content.get("third_party_instance_id", None)
+
+        if include_all_networks:
+            network_tuple = None
+            if third_party_instance_id is not None:
+                raise SynapseError(
+                    400, "Can't use include_all_networks with an explicit network"
+                )
+        elif third_party_instance_id is None:
+            network_tuple = ThirdPartyInstanceID(None, None)
+        else:
+            network_tuple = ThirdPartyInstanceID.from_string(third_party_instance_id)
+
+        if search_filter is None:
+            logger.warning("Nonefilter")
+
+        if limit == 0:
+            # zero is a special value which corresponds to no limit.
+            limit = None
+
+        data = await self.handler.get_local_public_room_list(
+            limit=limit,
+            since_token=since_token,
+            search_filter=search_filter,
+            network_tuple=network_tuple,
+            from_federation=True,
+        )
+
+        return 200, data
+
+
+class FederationGroupsRenewAttestaionServlet(BaseFederationServlet):
+    """A group or user's server renews their attestation"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/renew_attestation/(?P<user_id>[^/]*)"
+
+    def __init__(
+        self,
+        hs: HomeServer,
+        authenticator: Authenticator,
+        ratelimiter: FederationRateLimiter,
+        server_name: str,
+    ):
+        super().__init__(hs, authenticator, ratelimiter, server_name)
+        self.handler = hs.get_groups_attestation_renewer()
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        # We don't need to check auth here as we check the attestation signatures
+
+        new_content = await self.handler.on_renew_attestation(
+            group_id, user_id, content
+        )
+
+        return 200, new_content
+
+
+class OpenIdUserInfo(BaseFederationServlet):
+    """
+    Exchange a bearer token for information about a user.
+
+    The response format should be compatible with:
+        http://openid.net/specs/openid-connect-core-1_0.html#UserInfoResponse
+
+    GET /openid/userinfo?access_token=ABDEFGH HTTP/1.1
+
+    HTTP/1.1 200 OK
+    Content-Type: application/json
+
+    {
+        "sub": "@userpart:example.org",
+    }
+    """
+
+    PATH = "/openid/userinfo"
+
+    REQUIRE_AUTH = False
+
+    def __init__(
+        self,
+        hs: HomeServer,
+        authenticator: Authenticator,
+        ratelimiter: FederationRateLimiter,
+        server_name: str,
+    ):
+        super().__init__(hs, authenticator, ratelimiter, server_name)
+        self.handler = hs.get_federation_server()
+
+    async def on_GET(
+        self,
+        origin: Optional[str],
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+    ) -> Tuple[int, JsonDict]:
+        token = parse_string_from_args(query, "access_token")
+        if token is None:
+            return (
+                401,
+                {"errcode": "M_MISSING_TOKEN", "error": "Access Token required"},
+            )
+
+        user_id = await self.handler.on_openid_userinfo(token)
+
+        if user_id is None:
+            return (
+                401,
+                {
+                    "errcode": "M_UNKNOWN_TOKEN",
+                    "error": "Access Token unknown or expired",
+                },
+            )
+
+        return 200, {"sub": user_id}
+
+
+DEFAULT_SERVLET_GROUPS: Dict[str, Iterable[Type[BaseFederationServlet]]] = {
+    "federation": FEDERATION_SERVLET_CLASSES,
+    "room_list": (PublicRoomList,),
+    "group_server": GROUP_SERVER_SERVLET_CLASSES,
+    "group_local": GROUP_LOCAL_SERVLET_CLASSES,
+    "group_attestation": (FederationGroupsRenewAttestaionServlet,),
+    "openid": (OpenIdUserInfo,),
+}
+
+
+def register_servlets(
+    hs: HomeServer,
+    resource: HttpServer,
+    authenticator: Authenticator,
+    ratelimiter: FederationRateLimiter,
+    servlet_groups: Optional[Iterable[str]] = None,
+):
+    """Initialize and register servlet classes.
+
+    Will by default register all servlets. For custom behaviour, pass in
+    a list of servlet_groups to register.
+
+    Args:
+        hs: homeserver
+        resource: resource class to register to
+        authenticator: authenticator to use
+        ratelimiter: ratelimiter to use
+        servlet_groups: List of servlet groups to register.
+            Defaults to ``DEFAULT_SERVLET_GROUPS``.
+    """
+    if not servlet_groups:
+        servlet_groups = DEFAULT_SERVLET_GROUPS.keys()
+
+    for servlet_group in servlet_groups:
+        # Skip unknown servlet groups.
+        if servlet_group not in DEFAULT_SERVLET_GROUPS:
+            raise RuntimeError(
+                f"Attempting to register unknown federation servlet: '{servlet_group}'"
+            )
+
+        for servletclass in DEFAULT_SERVLET_GROUPS[servlet_group]:
+            servletclass(
+                hs=hs,
+                authenticator=authenticator,
+                ratelimiter=ratelimiter,
+                server_name=hs.hostname,
+            ).register(resource)
diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py
new file mode 100644
index 0000000000..624c859f1e
--- /dev/null
+++ b/synapse/federation/transport/server/_base.py
@@ -0,0 +1,328 @@
+#  Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import functools
+import logging
+import re
+
+from synapse.api.errors import Codes, FederationDeniedError, SynapseError
+from synapse.api.urls import FEDERATION_V1_PREFIX
+from synapse.http.servlet import parse_json_object_from_request
+from synapse.logging import opentracing
+from synapse.logging.context import run_in_background
+from synapse.logging.opentracing import (
+    SynapseTags,
+    start_active_span,
+    start_active_span_from_request,
+    tags,
+    whitelisted_homeserver,
+)
+from synapse.server import HomeServer
+from synapse.util.ratelimitutils import FederationRateLimiter
+from synapse.util.stringutils import parse_and_validate_server_name
+
+logger = logging.getLogger(__name__)
+
+
+class AuthenticationError(SynapseError):
+    """There was a problem authenticating the request"""
+
+
+class NoAuthenticationError(AuthenticationError):
+    """The request had no authentication information"""
+
+
+class Authenticator:
+    def __init__(self, hs: HomeServer):
+        self._clock = hs.get_clock()
+        self.keyring = hs.get_keyring()
+        self.server_name = hs.hostname
+        self.store = hs.get_datastore()
+        self.federation_domain_whitelist = hs.config.federation_domain_whitelist
+        self.notifier = hs.get_notifier()
+
+        self.replication_client = None
+        if hs.config.worker.worker_app:
+            self.replication_client = hs.get_tcp_replication()
+
+    # A method just so we can pass 'self' as the authenticator to the Servlets
+    async def authenticate_request(self, request, content):
+        now = self._clock.time_msec()
+        json_request = {
+            "method": request.method.decode("ascii"),
+            "uri": request.uri.decode("ascii"),
+            "destination": self.server_name,
+            "signatures": {},
+        }
+
+        if content is not None:
+            json_request["content"] = content
+
+        origin = None
+
+        auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
+
+        if not auth_headers:
+            raise NoAuthenticationError(
+                401, "Missing Authorization headers", Codes.UNAUTHORIZED
+            )
+
+        for auth in auth_headers:
+            if auth.startswith(b"X-Matrix"):
+                (origin, key, sig) = _parse_auth_header(auth)
+                json_request["origin"] = origin
+                json_request["signatures"].setdefault(origin, {})[key] = sig
+
+        if (
+            self.federation_domain_whitelist is not None
+            and origin not in self.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(origin)
+
+        if origin is None or not json_request["signatures"]:
+            raise NoAuthenticationError(
+                401, "Missing Authorization headers", Codes.UNAUTHORIZED
+            )
+
+        await self.keyring.verify_json_for_server(
+            origin,
+            json_request,
+            now,
+        )
+
+        logger.debug("Request from %s", origin)
+        request.requester = origin
+
+        # If we get a valid signed request from the other side, its probably
+        # alive
+        retry_timings = await self.store.get_destination_retry_timings(origin)
+        if retry_timings and retry_timings.retry_last_ts:
+            run_in_background(self._reset_retry_timings, origin)
+
+        return origin
+
+    async def _reset_retry_timings(self, origin):
+        try:
+            logger.info("Marking origin %r as up", origin)
+            await self.store.set_destination_retry_timings(origin, None, 0, 0)
+
+            # Inform the relevant places that the remote server is back up.
+            self.notifier.notify_remote_server_up(origin)
+            if self.replication_client:
+                # If we're on a worker we try and inform master about this. The
+                # replication client doesn't hook into the notifier to avoid
+                # infinite loops where we send a `REMOTE_SERVER_UP` command to
+                # master, which then echoes it back to us which in turn pokes
+                # the notifier.
+                self.replication_client.send_remote_server_up(origin)
+
+        except Exception:
+            logger.exception("Error resetting retry timings on %s", origin)
+
+
+def _parse_auth_header(header_bytes):
+    """Parse an X-Matrix auth header
+
+    Args:
+        header_bytes (bytes): header value
+
+    Returns:
+        Tuple[str, str, str]: origin, key id, signature.
+
+    Raises:
+        AuthenticationError if the header could not be parsed
+    """
+    try:
+        header_str = header_bytes.decode("utf-8")
+        params = header_str.split(" ")[1].split(",")
+        param_dict = dict(kv.split("=") for kv in params)
+
+        def strip_quotes(value):
+            if value.startswith('"'):
+                return value[1:-1]
+            else:
+                return value
+
+        origin = strip_quotes(param_dict["origin"])
+
+        # ensure that the origin is a valid server name
+        parse_and_validate_server_name(origin)
+
+        key = strip_quotes(param_dict["key"])
+        sig = strip_quotes(param_dict["sig"])
+        return origin, key, sig
+    except Exception as e:
+        logger.warning(
+            "Error parsing auth header '%s': %s",
+            header_bytes.decode("ascii", "replace"),
+            e,
+        )
+        raise AuthenticationError(
+            400, "Malformed Authorization header", Codes.UNAUTHORIZED
+        )
+
+
+class BaseFederationServlet:
+    """Abstract base class for federation servlet classes.
+
+    The servlet object should have a PATH attribute which takes the form of a regexp to
+    match against the request path (excluding the /federation/v1 prefix).
+
+    The servlet should also implement one or more of on_GET, on_POST, on_PUT, to match
+    the appropriate HTTP method. These methods must be *asynchronous* and have the
+    signature:
+
+        on_<METHOD>(self, origin, content, query, **kwargs)
+
+        With arguments:
+
+            origin (unicode|None): The authenticated server_name of the calling server,
+                unless REQUIRE_AUTH is set to False and authentication failed.
+
+            content (unicode|None): decoded json body of the request. None if the
+                request was a GET.
+
+            query (dict[bytes, list[bytes]]): Query params from the request. url-decoded
+                (ie, '+' and '%xx' are decoded) but note that it is *not* utf8-decoded
+                yet.
+
+            **kwargs (dict[unicode, unicode]): the dict mapping keys to path
+                components as specified in the path match regexp.
+
+        Returns:
+            Optional[Tuple[int, object]]: either (response code, response object) to
+                 return a JSON response, or None if the request has already been handled.
+
+        Raises:
+            SynapseError: to return an error code
+
+            Exception: other exceptions will be caught, logged, and a 500 will be
+                returned.
+    """
+
+    PATH = ""  # Overridden in subclasses, the regex to match against the path.
+
+    REQUIRE_AUTH = True
+
+    PREFIX = FEDERATION_V1_PREFIX  # Allows specifying the API version
+
+    RATELIMIT = True  # Whether to rate limit requests or not
+
+    def __init__(
+        self,
+        hs: HomeServer,
+        authenticator: Authenticator,
+        ratelimiter: FederationRateLimiter,
+        server_name: str,
+    ):
+        self.hs = hs
+        self.authenticator = authenticator
+        self.ratelimiter = ratelimiter
+        self.server_name = server_name
+
+    def _wrap(self, func):
+        authenticator = self.authenticator
+        ratelimiter = self.ratelimiter
+
+        @functools.wraps(func)
+        async def new_func(request, *args, **kwargs):
+            """A callback which can be passed to HttpServer.RegisterPaths
+
+            Args:
+                request (twisted.web.http.Request):
+                *args: unused?
+                **kwargs (dict[unicode, unicode]): the dict mapping keys to path
+                    components as specified in the path match regexp.
+
+            Returns:
+                Tuple[int, object]|None: (response code, response object) as returned by
+                    the callback method. None if the request has already been handled.
+            """
+            content = None
+            if request.method in [b"PUT", b"POST"]:
+                # TODO: Handle other method types? other content types?
+                content = parse_json_object_from_request(request)
+
+            try:
+                origin = await authenticator.authenticate_request(request, content)
+            except NoAuthenticationError:
+                origin = None
+                if self.REQUIRE_AUTH:
+                    logger.warning(
+                        "authenticate_request failed: missing authentication"
+                    )
+                    raise
+            except Exception as e:
+                logger.warning("authenticate_request failed: %s", e)
+                raise
+
+            request_tags = {
+                SynapseTags.REQUEST_ID: request.get_request_id(),
+                tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER,
+                tags.HTTP_METHOD: request.get_method(),
+                tags.HTTP_URL: request.get_redacted_uri(),
+                tags.PEER_HOST_IPV6: request.getClientIP(),
+                "authenticated_entity": origin,
+                "servlet_name": request.request_metrics.name,
+            }
+
+            # Only accept the span context if the origin is authenticated
+            # and whitelisted
+            if origin and whitelisted_homeserver(origin):
+                scope = start_active_span_from_request(
+                    request, "incoming-federation-request", tags=request_tags
+                )
+            else:
+                scope = start_active_span(
+                    "incoming-federation-request", tags=request_tags
+                )
+
+            with scope:
+                opentracing.inject_response_headers(request.responseHeaders)
+
+                if origin and self.RATELIMIT:
+                    with ratelimiter.ratelimit(origin) as d:
+                        await d
+                        if request._disconnected:
+                            logger.warning(
+                                "client disconnected before we started processing "
+                                "request"
+                            )
+                            return -1, None
+                        response = await func(
+                            origin, content, request.args, *args, **kwargs
+                        )
+                else:
+                    response = await func(
+                        origin, content, request.args, *args, **kwargs
+                    )
+
+            return response
+
+        return new_func
+
+    def register(self, server):
+        pattern = re.compile("^" + self.PREFIX + self.PATH + "$")
+
+        for method in ("GET", "PUT", "POST"):
+            code = getattr(self, "on_%s" % (method), None)
+            if code is None:
+                continue
+
+            server.register_paths(
+                method,
+                (pattern,),
+                self._wrap(code),
+                self.__class__.__name__,
+            )
diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py
new file mode 100644
index 0000000000..640d8316b7
--- /dev/null
+++ b/synapse/federation/transport/server/federation.py
@@ -0,0 +1,759 @@
+#  Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import logging
+from typing import Dict, List, Mapping, Optional, Sequence, Tuple, Type, Union
+
+from typing_extensions import Literal
+
+import synapse
+from synapse.api.errors import Codes, SynapseError
+from synapse.api.room_versions import RoomVersions
+from synapse.api.urls import FEDERATION_UNSTABLE_PREFIX, FEDERATION_V2_PREFIX
+from synapse.federation.transport.server._base import (
+    Authenticator,
+    BaseFederationServlet,
+)
+from synapse.http.servlet import (
+    assert_params_in_dict,
+    parse_boolean_from_args,
+    parse_integer_from_args,
+    parse_string_from_args,
+    parse_strings_from_args,
+)
+from synapse.server import HomeServer
+from synapse.types import JsonDict
+from synapse.util.ratelimitutils import FederationRateLimiter
+from synapse.util.versionstring import get_version_string
+
+logger = logging.getLogger(__name__)
+
+
+class BaseFederationServerServlet(BaseFederationServlet):
+    """Abstract base class for federation servlet classes which provides a federation server handler.
+
+    See BaseFederationServlet for more information.
+    """
+
+    def __init__(
+        self,
+        hs: HomeServer,
+        authenticator: Authenticator,
+        ratelimiter: FederationRateLimiter,
+        server_name: str,
+    ):
+        super().__init__(hs, authenticator, ratelimiter, server_name)
+        self.handler = hs.get_federation_server()
+
+
+class FederationSendServlet(BaseFederationServerServlet):
+    PATH = "/send/(?P<transaction_id>[^/]*)/?"
+
+    # We ratelimit manually in the handler as we queue up the requests and we
+    # don't want to fill up the ratelimiter with blocked requests.
+    RATELIMIT = False
+
+    # This is when someone is trying to send us a bunch of data.
+    async def on_PUT(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        transaction_id: str,
+    ) -> Tuple[int, JsonDict]:
+        """Called on PUT /send/<transaction_id>/
+
+        Args:
+            transaction_id: The transaction_id associated with this request. This
+                is *not* None.
+
+        Returns:
+            Tuple of `(code, response)`, where
+            `response` is a python dict to be converted into JSON that is
+            used as the response body.
+        """
+        # Parse the request
+        try:
+            transaction_data = content
+
+            logger.debug("Decoded %s: %s", transaction_id, str(transaction_data))
+
+            logger.info(
+                "Received txn %s from %s. (PDUs: %d, EDUs: %d)",
+                transaction_id,
+                origin,
+                len(transaction_data.get("pdus", [])),
+                len(transaction_data.get("edus", [])),
+            )
+
+        except Exception as e:
+            logger.exception(e)
+            return 400, {"error": "Invalid transaction"}
+
+        code, response = await self.handler.on_incoming_transaction(
+            origin, transaction_id, self.server_name, transaction_data
+        )
+
+        return code, response
+
+
+class FederationEventServlet(BaseFederationServerServlet):
+    PATH = "/event/(?P<event_id>[^/]*)/?"
+
+    # This is when someone asks for a data item for a given server data_id pair.
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        event_id: str,
+    ) -> Tuple[int, Union[JsonDict, str]]:
+        return await self.handler.on_pdu_request(origin, event_id)
+
+
+class FederationStateV1Servlet(BaseFederationServerServlet):
+    PATH = "/state/(?P<room_id>[^/]*)/?"
+
+    # This is when someone asks for all data for a given room.
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        return await self.handler.on_room_state_request(
+            origin,
+            room_id,
+            parse_string_from_args(query, "event_id", None, required=False),
+        )
+
+
+class FederationStateIdsServlet(BaseFederationServerServlet):
+    PATH = "/state_ids/(?P<room_id>[^/]*)/?"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        return await self.handler.on_state_ids_request(
+            origin,
+            room_id,
+            parse_string_from_args(query, "event_id", None, required=True),
+        )
+
+
+class FederationBackfillServlet(BaseFederationServerServlet):
+    PATH = "/backfill/(?P<room_id>[^/]*)/?"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        versions = [x.decode("ascii") for x in query[b"v"]]
+        limit = parse_integer_from_args(query, "limit", None)
+
+        if not limit:
+            return 400, {"error": "Did not include limit param"}
+
+        return await self.handler.on_backfill_request(origin, room_id, versions, limit)
+
+
+class FederationQueryServlet(BaseFederationServerServlet):
+    PATH = "/query/(?P<query_type>[^/]*)"
+
+    # This is when we receive a server-server Query
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        query_type: str,
+    ) -> Tuple[int, JsonDict]:
+        args = {k.decode("utf8"): v[0].decode("utf-8") for k, v in query.items()}
+        args["origin"] = origin
+        return await self.handler.on_query_request(query_type, args)
+
+
+class FederationMakeJoinServlet(BaseFederationServerServlet):
+    PATH = "/make_join/(?P<room_id>[^/]*)/(?P<user_id>[^/]*)"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        """
+        Args:
+            origin: The authenticated server_name of the calling server
+
+            content: (GETs don't have bodies)
+
+            query: Query params from the request.
+
+            **kwargs: the dict mapping keys to path components as specified in
+                the path match regexp.
+
+        Returns:
+            Tuple of (response code, response object)
+        """
+        supported_versions = parse_strings_from_args(query, "ver", encoding="utf-8")
+        if supported_versions is None:
+            supported_versions = ["1"]
+
+        result = await self.handler.on_make_join_request(
+            origin, room_id, user_id, supported_versions=supported_versions
+        )
+        return 200, result
+
+
+class FederationMakeLeaveServlet(BaseFederationServerServlet):
+    PATH = "/make_leave/(?P<room_id>[^/]*)/(?P<user_id>[^/]*)"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        result = await self.handler.on_make_leave_request(origin, room_id, user_id)
+        return 200, result
+
+
+class FederationV1SendLeaveServlet(BaseFederationServerServlet):
+    PATH = "/send_leave/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
+
+    async def on_PUT(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        event_id: str,
+    ) -> Tuple[int, Tuple[int, JsonDict]]:
+        result = await self.handler.on_send_leave_request(origin, content, room_id)
+        return 200, (200, result)
+
+
+class FederationV2SendLeaveServlet(BaseFederationServerServlet):
+    PATH = "/send_leave/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
+
+    PREFIX = FEDERATION_V2_PREFIX
+
+    async def on_PUT(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        event_id: str,
+    ) -> Tuple[int, JsonDict]:
+        result = await self.handler.on_send_leave_request(origin, content, room_id)
+        return 200, result
+
+
+class FederationMakeKnockServlet(BaseFederationServerServlet):
+    PATH = "/make_knock/(?P<room_id>[^/]*)/(?P<user_id>[^/]*)"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        # Retrieve the room versions the remote homeserver claims to support
+        supported_versions = parse_strings_from_args(
+            query, "ver", required=True, encoding="utf-8"
+        )
+
+        result = await self.handler.on_make_knock_request(
+            origin, room_id, user_id, supported_versions=supported_versions
+        )
+        return 200, result
+
+
+class FederationV1SendKnockServlet(BaseFederationServerServlet):
+    PATH = "/send_knock/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
+
+    async def on_PUT(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        event_id: str,
+    ) -> Tuple[int, JsonDict]:
+        result = await self.handler.on_send_knock_request(origin, content, room_id)
+        return 200, result
+
+
+class FederationEventAuthServlet(BaseFederationServerServlet):
+    PATH = "/event_auth/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        event_id: str,
+    ) -> Tuple[int, JsonDict]:
+        return await self.handler.on_event_auth(origin, room_id, event_id)
+
+
+class FederationV1SendJoinServlet(BaseFederationServerServlet):
+    PATH = "/send_join/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
+
+    async def on_PUT(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        event_id: str,
+    ) -> Tuple[int, Tuple[int, JsonDict]]:
+        # TODO(paul): assert that event_id parsed from path actually
+        #   match those given in content
+        result = await self.handler.on_send_join_request(origin, content, room_id)
+        return 200, (200, result)
+
+
+class FederationV2SendJoinServlet(BaseFederationServerServlet):
+    PATH = "/send_join/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
+
+    PREFIX = FEDERATION_V2_PREFIX
+
+    async def on_PUT(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        event_id: str,
+    ) -> Tuple[int, JsonDict]:
+        # TODO(paul): assert that event_id parsed from path actually
+        #   match those given in content
+        result = await self.handler.on_send_join_request(origin, content, room_id)
+        return 200, result
+
+
+class FederationV1InviteServlet(BaseFederationServerServlet):
+    PATH = "/invite/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
+
+    async def on_PUT(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        event_id: str,
+    ) -> Tuple[int, Tuple[int, JsonDict]]:
+        # We don't get a room version, so we have to assume its EITHER v1 or
+        # v2. This is "fine" as the only difference between V1 and V2 is the
+        # state resolution algorithm, and we don't use that for processing
+        # invites
+        result = await self.handler.on_invite_request(
+            origin, content, room_version_id=RoomVersions.V1.identifier
+        )
+
+        # V1 federation API is defined to return a content of `[200, {...}]`
+        # due to a historical bug.
+        return 200, (200, result)
+
+
+class FederationV2InviteServlet(BaseFederationServerServlet):
+    PATH = "/invite/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
+
+    PREFIX = FEDERATION_V2_PREFIX
+
+    async def on_PUT(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+        event_id: str,
+    ) -> Tuple[int, JsonDict]:
+        # TODO(paul): assert that room_id/event_id parsed from path actually
+        #   match those given in content
+
+        room_version = content["room_version"]
+        event = content["event"]
+        invite_room_state = content["invite_room_state"]
+
+        # Synapse expects invite_room_state to be in unsigned, as it is in v1
+        # API
+
+        event.setdefault("unsigned", {})["invite_room_state"] = invite_room_state
+
+        result = await self.handler.on_invite_request(
+            origin, event, room_version_id=room_version
+        )
+        return 200, result
+
+
+class FederationThirdPartyInviteExchangeServlet(BaseFederationServerServlet):
+    PATH = "/exchange_third_party_invite/(?P<room_id>[^/]*)"
+
+    async def on_PUT(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        await self.handler.on_exchange_third_party_invite_request(content)
+        return 200, {}
+
+
+class FederationClientKeysQueryServlet(BaseFederationServerServlet):
+    PATH = "/user/keys/query"
+
+    async def on_POST(
+        self, origin: str, content: JsonDict, query: Dict[bytes, List[bytes]]
+    ) -> Tuple[int, JsonDict]:
+        return await self.handler.on_query_client_keys(origin, content)
+
+
+class FederationUserDevicesQueryServlet(BaseFederationServerServlet):
+    PATH = "/user/devices/(?P<user_id>[^/]*)"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        return await self.handler.on_query_user_devices(origin, user_id)
+
+
+class FederationClientKeysClaimServlet(BaseFederationServerServlet):
+    PATH = "/user/keys/claim"
+
+    async def on_POST(
+        self, origin: str, content: JsonDict, query: Dict[bytes, List[bytes]]
+    ) -> Tuple[int, JsonDict]:
+        response = await self.handler.on_claim_client_keys(origin, content)
+        return 200, response
+
+
+class FederationGetMissingEventsServlet(BaseFederationServerServlet):
+    # TODO(paul): Why does this path alone end with "/?" optional?
+    PATH = "/get_missing_events/(?P<room_id>[^/]*)/?"
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        limit = int(content.get("limit", 10))
+        earliest_events = content.get("earliest_events", [])
+        latest_events = content.get("latest_events", [])
+
+        result = await self.handler.on_get_missing_events(
+            origin,
+            room_id=room_id,
+            earliest_events=earliest_events,
+            latest_events=latest_events,
+            limit=limit,
+        )
+
+        return 200, result
+
+
+class On3pidBindServlet(BaseFederationServerServlet):
+    PATH = "/3pid/onbind"
+
+    REQUIRE_AUTH = False
+
+    async def on_POST(
+        self, origin: Optional[str], content: JsonDict, query: Dict[bytes, List[bytes]]
+    ) -> Tuple[int, JsonDict]:
+        if "invites" in content:
+            last_exception = None
+            for invite in content["invites"]:
+                try:
+                    if "signed" not in invite or "token" not in invite["signed"]:
+                        message = (
+                            "Rejecting received notification of third-"
+                            "party invite without signed: %s" % (invite,)
+                        )
+                        logger.info(message)
+                        raise SynapseError(400, message)
+                    await self.handler.exchange_third_party_invite(
+                        invite["sender"],
+                        invite["mxid"],
+                        invite["room_id"],
+                        invite["signed"],
+                    )
+                except Exception as e:
+                    last_exception = e
+            if last_exception:
+                raise last_exception
+        return 200, {}
+
+
+class FederationUserInfoServlet(BaseFederationServlet):
+    """
+    Return information about a set of users.
+
+    This API returns expiration and deactivation information about a set of
+    users. Requested users not local to this homeserver will be ignored.
+
+    Example request:
+        POST /users/info
+
+        {
+            "user_ids": [
+                "@alice:example.com",
+                "@bob:example.com"
+            ]
+        }
+
+    Example response
+        {
+            "@alice:example.com": {
+                "expired": false,
+                "deactivated": true
+            }
+        }
+    """
+
+    PATH = "/users/info"
+    PREFIX = FEDERATION_UNSTABLE_PREFIX
+
+    def __init__(self, hs, authenticator, ratelimiter, server_name):
+        super(FederationUserInfoServlet, self).__init__(
+            hs, authenticator, ratelimiter, server_name
+        )
+        self._store = hs.get_datastore()
+
+    async def on_POST(self, origin, content, query):
+        assert_params_in_dict(content, required=["user_ids"])
+
+        user_ids = content.get("user_ids", [])
+
+        if not isinstance(user_ids, list):
+            raise SynapseError(
+                400,
+                "'user_ids' must be a list of user ID strings",
+                errcode=Codes.INVALID_PARAM,
+            )
+
+        data = await self._store.get_info_for_users(user_ids)
+        return 200, data
+
+
+class FederationVersionServlet(BaseFederationServlet):
+    PATH = "/version"
+
+    REQUIRE_AUTH = False
+
+    async def on_GET(
+        self,
+        origin: Optional[str],
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+    ) -> Tuple[int, JsonDict]:
+        return (
+            200,
+            {"server": {"name": "Synapse", "version": get_version_string(synapse)}},
+        )
+
+
+class FederationSpaceSummaryServlet(BaseFederationServlet):
+    PREFIX = FEDERATION_UNSTABLE_PREFIX + "/org.matrix.msc2946"
+    PATH = "/spaces/(?P<room_id>[^/]*)"
+
+    def __init__(
+        self,
+        hs: HomeServer,
+        authenticator: Authenticator,
+        ratelimiter: FederationRateLimiter,
+        server_name: str,
+    ):
+        super().__init__(hs, authenticator, ratelimiter, server_name)
+        self.handler = hs.get_room_summary_handler()
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Mapping[bytes, Sequence[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        suggested_only = parse_boolean_from_args(query, "suggested_only", default=False)
+
+        max_rooms_per_space = parse_integer_from_args(query, "max_rooms_per_space")
+        if max_rooms_per_space is not None and max_rooms_per_space < 0:
+            raise SynapseError(
+                400,
+                "Value for 'max_rooms_per_space' must be a non-negative integer",
+                Codes.BAD_JSON,
+            )
+
+        exclude_rooms = parse_strings_from_args(query, "exclude_rooms", default=[])
+
+        return 200, await self.handler.federation_space_summary(
+            origin, room_id, suggested_only, max_rooms_per_space, exclude_rooms
+        )
+
+    # TODO When switching to the stable endpoint, remove the POST handler.
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Mapping[bytes, Sequence[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        suggested_only = content.get("suggested_only", False)
+        if not isinstance(suggested_only, bool):
+            raise SynapseError(
+                400, "'suggested_only' must be a boolean", Codes.BAD_JSON
+            )
+
+        exclude_rooms = content.get("exclude_rooms", [])
+        if not isinstance(exclude_rooms, list) or any(
+            not isinstance(x, str) for x in exclude_rooms
+        ):
+            raise SynapseError(400, "bad value for 'exclude_rooms'", Codes.BAD_JSON)
+
+        max_rooms_per_space = content.get("max_rooms_per_space")
+        if max_rooms_per_space is not None:
+            if not isinstance(max_rooms_per_space, int):
+                raise SynapseError(
+                    400, "bad value for 'max_rooms_per_space'", Codes.BAD_JSON
+                )
+            if max_rooms_per_space < 0:
+                raise SynapseError(
+                    400,
+                    "Value for 'max_rooms_per_space' must be a non-negative integer",
+                    Codes.BAD_JSON,
+                )
+
+        return 200, await self.handler.federation_space_summary(
+            origin, room_id, suggested_only, max_rooms_per_space, exclude_rooms
+        )
+
+
+class FederationRoomHierarchyServlet(BaseFederationServlet):
+    PREFIX = FEDERATION_UNSTABLE_PREFIX + "/org.matrix.msc2946"
+    PATH = "/hierarchy/(?P<room_id>[^/]*)"
+
+    def __init__(
+        self,
+        hs: HomeServer,
+        authenticator: Authenticator,
+        ratelimiter: FederationRateLimiter,
+        server_name: str,
+    ):
+        super().__init__(hs, authenticator, ratelimiter, server_name)
+        self.handler = hs.get_room_summary_handler()
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Mapping[bytes, Sequence[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        suggested_only = parse_boolean_from_args(query, "suggested_only", default=False)
+        return 200, await self.handler.get_federation_hierarchy(
+            origin, room_id, suggested_only
+        )
+
+
+class RoomComplexityServlet(BaseFederationServlet):
+    """
+    Indicates to other servers how complex (and therefore likely
+    resource-intensive) a public room this server knows about is.
+    """
+
+    PATH = "/rooms/(?P<room_id>[^/]*)/complexity"
+    PREFIX = FEDERATION_UNSTABLE_PREFIX
+
+    def __init__(
+        self,
+        hs: HomeServer,
+        authenticator: Authenticator,
+        ratelimiter: FederationRateLimiter,
+        server_name: str,
+    ):
+        super().__init__(hs, authenticator, ratelimiter, server_name)
+        self._store = self.hs.get_datastore()
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        is_public = await self._store.is_room_world_readable_or_publicly_joinable(
+            room_id
+        )
+
+        if not is_public:
+            raise SynapseError(404, "Room not found", errcode=Codes.INVALID_PARAM)
+
+        complexity = await self._store.get_room_complexity(room_id)
+        return 200, complexity
+
+
+FEDERATION_SERVLET_CLASSES: Tuple[Type[BaseFederationServlet], ...] = (
+    FederationSendServlet,
+    FederationEventServlet,
+    FederationStateV1Servlet,
+    FederationStateIdsServlet,
+    FederationBackfillServlet,
+    FederationQueryServlet,
+    FederationMakeJoinServlet,
+    FederationMakeLeaveServlet,
+    FederationEventServlet,
+    FederationV1SendJoinServlet,
+    FederationV2SendJoinServlet,
+    FederationV1SendLeaveServlet,
+    FederationV2SendLeaveServlet,
+    FederationV1InviteServlet,
+    FederationV2InviteServlet,
+    FederationGetMissingEventsServlet,
+    FederationEventAuthServlet,
+    FederationClientKeysQueryServlet,
+    FederationUserDevicesQueryServlet,
+    FederationClientKeysClaimServlet,
+    FederationThirdPartyInviteExchangeServlet,
+    On3pidBindServlet,
+    FederationVersionServlet,
+    RoomComplexityServlet,
+    FederationUserInfoServlet,
+    FederationSpaceSummaryServlet,
+    FederationRoomHierarchyServlet,
+    FederationV1SendKnockServlet,
+    FederationMakeKnockServlet,
+)
diff --git a/synapse/federation/transport/server/groups_local.py b/synapse/federation/transport/server/groups_local.py
new file mode 100644
index 0000000000..a12cd18d58
--- /dev/null
+++ b/synapse/federation/transport/server/groups_local.py
@@ -0,0 +1,113 @@
+#  Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import Dict, List, Tuple, Type
+
+from synapse.api.errors import SynapseError
+from synapse.federation.transport.server._base import (
+    Authenticator,
+    BaseFederationServlet,
+)
+from synapse.handlers.groups_local import GroupsLocalHandler
+from synapse.server import HomeServer
+from synapse.types import JsonDict, get_domain_from_id
+from synapse.util.ratelimitutils import FederationRateLimiter
+
+
+class BaseGroupsLocalServlet(BaseFederationServlet):
+    """Abstract base class for federation servlet classes which provides a groups local handler.
+
+    See BaseFederationServlet for more information.
+    """
+
+    def __init__(
+        self,
+        hs: HomeServer,
+        authenticator: Authenticator,
+        ratelimiter: FederationRateLimiter,
+        server_name: str,
+    ):
+        super().__init__(hs, authenticator, ratelimiter, server_name)
+        self.handler = hs.get_groups_local_handler()
+
+
+class FederationGroupsLocalInviteServlet(BaseGroupsLocalServlet):
+    """A group server has invited a local user"""
+
+    PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite"
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        if get_domain_from_id(group_id) != origin:
+            raise SynapseError(403, "group_id doesn't match origin")
+
+        assert isinstance(
+            self.handler, GroupsLocalHandler
+        ), "Workers cannot handle group invites."
+
+        new_content = await self.handler.on_invite(group_id, user_id, content)
+
+        return 200, new_content
+
+
+class FederationGroupsRemoveLocalUserServlet(BaseGroupsLocalServlet):
+    """A group server has removed a local user"""
+
+    PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove"
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        user_id: str,
+    ) -> Tuple[int, None]:
+        if get_domain_from_id(group_id) != origin:
+            raise SynapseError(403, "user_id doesn't match origin")
+
+        assert isinstance(
+            self.handler, GroupsLocalHandler
+        ), "Workers cannot handle group removals."
+
+        await self.handler.user_removed_from_group(group_id, user_id, content)
+
+        return 200, None
+
+
+class FederationGroupsBulkPublicisedServlet(BaseGroupsLocalServlet):
+    """Get roles in a group"""
+
+    PATH = "/get_groups_publicised"
+
+    async def on_POST(
+        self, origin: str, content: JsonDict, query: Dict[bytes, List[bytes]]
+    ) -> Tuple[int, JsonDict]:
+        resp = await self.handler.bulk_get_publicised_groups(
+            content["user_ids"], proxy=False
+        )
+
+        return 200, resp
+
+
+GROUP_LOCAL_SERVLET_CLASSES: Tuple[Type[BaseFederationServlet], ...] = (
+    FederationGroupsLocalInviteServlet,
+    FederationGroupsRemoveLocalUserServlet,
+    FederationGroupsBulkPublicisedServlet,
+)
diff --git a/synapse/federation/transport/server/groups_server.py b/synapse/federation/transport/server/groups_server.py
new file mode 100644
index 0000000000..b30e92a5eb
--- /dev/null
+++ b/synapse/federation/transport/server/groups_server.py
@@ -0,0 +1,753 @@
+#  Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import Dict, List, Tuple, Type
+
+from typing_extensions import Literal
+
+from synapse.api.constants import MAX_GROUP_CATEGORYID_LENGTH, MAX_GROUP_ROLEID_LENGTH
+from synapse.api.errors import Codes, SynapseError
+from synapse.federation.transport.server._base import (
+    Authenticator,
+    BaseFederationServlet,
+)
+from synapse.http.servlet import parse_string_from_args
+from synapse.server import HomeServer
+from synapse.types import JsonDict, get_domain_from_id
+from synapse.util.ratelimitutils import FederationRateLimiter
+
+
+class BaseGroupsServerServlet(BaseFederationServlet):
+    """Abstract base class for federation servlet classes which provides a groups server handler.
+
+    See BaseFederationServlet for more information.
+    """
+
+    def __init__(
+        self,
+        hs: HomeServer,
+        authenticator: Authenticator,
+        ratelimiter: FederationRateLimiter,
+        server_name: str,
+    ):
+        super().__init__(hs, authenticator, ratelimiter, server_name)
+        self.handler = hs.get_groups_server_handler()
+
+
+class FederationGroupsProfileServlet(BaseGroupsServerServlet):
+    """Get/set the basic profile of a group on behalf of a user"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/profile"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.get_group_profile(group_id, requester_user_id)
+
+        return 200, new_content
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.update_group_profile(
+            group_id, requester_user_id, content
+        )
+
+        return 200, new_content
+
+
+class FederationGroupsSummaryServlet(BaseGroupsServerServlet):
+    PATH = "/groups/(?P<group_id>[^/]*)/summary"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.get_group_summary(group_id, requester_user_id)
+
+        return 200, new_content
+
+
+class FederationGroupsRoomsServlet(BaseGroupsServerServlet):
+    """Get the rooms in a group on behalf of a user"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/rooms"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.get_rooms_in_group(group_id, requester_user_id)
+
+        return 200, new_content
+
+
+class FederationGroupsAddRoomsServlet(BaseGroupsServerServlet):
+    """Add/remove room from group"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/room/(?P<room_id>[^/]*)"
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.add_room_to_group(
+            group_id, requester_user_id, room_id, content
+        )
+
+        return 200, new_content
+
+    async def on_DELETE(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.remove_room_from_group(
+            group_id, requester_user_id, room_id
+        )
+
+        return 200, new_content
+
+
+class FederationGroupsAddRoomsConfigServlet(BaseGroupsServerServlet):
+    """Update room config in group"""
+
+    PATH = (
+        "/groups/(?P<group_id>[^/]*)/room/(?P<room_id>[^/]*)"
+        "/config/(?P<config_key>[^/]*)"
+    )
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        room_id: str,
+        config_key: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        result = await self.handler.update_room_in_group(
+            group_id, requester_user_id, room_id, config_key, content
+        )
+
+        return 200, result
+
+
+class FederationGroupsUsersServlet(BaseGroupsServerServlet):
+    """Get the users in a group on behalf of a user"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/users"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.get_users_in_group(group_id, requester_user_id)
+
+        return 200, new_content
+
+
+class FederationGroupsInvitedUsersServlet(BaseGroupsServerServlet):
+    """Get the users that have been invited to a group"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/invited_users"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.get_invited_users_in_group(
+            group_id, requester_user_id
+        )
+
+        return 200, new_content
+
+
+class FederationGroupsInviteServlet(BaseGroupsServerServlet):
+    """Ask a group server to invite someone to the group"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite"
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.invite_to_group(
+            group_id, user_id, requester_user_id, content
+        )
+
+        return 200, new_content
+
+
+class FederationGroupsAcceptInviteServlet(BaseGroupsServerServlet):
+    """Accept an invitation from the group server"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/accept_invite"
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        if get_domain_from_id(user_id) != origin:
+            raise SynapseError(403, "user_id doesn't match origin")
+
+        new_content = await self.handler.accept_invite(group_id, user_id, content)
+
+        return 200, new_content
+
+
+class FederationGroupsJoinServlet(BaseGroupsServerServlet):
+    """Attempt to join a group"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/join"
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        if get_domain_from_id(user_id) != origin:
+            raise SynapseError(403, "user_id doesn't match origin")
+
+        new_content = await self.handler.join_group(group_id, user_id, content)
+
+        return 200, new_content
+
+
+class FederationGroupsRemoveUserServlet(BaseGroupsServerServlet):
+    """Leave or kick a user from the group"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove"
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.remove_user_from_group(
+            group_id, user_id, requester_user_id, content
+        )
+
+        return 200, new_content
+
+
+class FederationGroupsSummaryRoomsServlet(BaseGroupsServerServlet):
+    """Add/remove a room from the group summary, with optional category.
+
+    Matches both:
+        - /groups/:group/summary/rooms/:room_id
+        - /groups/:group/summary/categories/:category/rooms/:room_id
+    """
+
+    PATH = (
+        "/groups/(?P<group_id>[^/]*)/summary"
+        "(/categories/(?P<category_id>[^/]+))?"
+        "/rooms/(?P<room_id>[^/]*)"
+    )
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        category_id: str,
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        if category_id == "":
+            raise SynapseError(
+                400, "category_id cannot be empty string", Codes.INVALID_PARAM
+            )
+
+        if len(category_id) > MAX_GROUP_CATEGORYID_LENGTH:
+            raise SynapseError(
+                400,
+                "category_id may not be longer than %s characters"
+                % (MAX_GROUP_CATEGORYID_LENGTH,),
+                Codes.INVALID_PARAM,
+            )
+
+        resp = await self.handler.update_group_summary_room(
+            group_id,
+            requester_user_id,
+            room_id=room_id,
+            category_id=category_id,
+            content=content,
+        )
+
+        return 200, resp
+
+    async def on_DELETE(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        category_id: str,
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        if category_id == "":
+            raise SynapseError(400, "category_id cannot be empty string")
+
+        resp = await self.handler.delete_group_summary_room(
+            group_id, requester_user_id, room_id=room_id, category_id=category_id
+        )
+
+        return 200, resp
+
+
+class FederationGroupsCategoriesServlet(BaseGroupsServerServlet):
+    """Get all categories for a group"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/categories/?"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        resp = await self.handler.get_group_categories(group_id, requester_user_id)
+
+        return 200, resp
+
+
+class FederationGroupsCategoryServlet(BaseGroupsServerServlet):
+    """Add/remove/get a category in a group"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/categories/(?P<category_id>[^/]+)"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        category_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        resp = await self.handler.get_group_category(
+            group_id, requester_user_id, category_id
+        )
+
+        return 200, resp
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        category_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        if category_id == "":
+            raise SynapseError(400, "category_id cannot be empty string")
+
+        if len(category_id) > MAX_GROUP_CATEGORYID_LENGTH:
+            raise SynapseError(
+                400,
+                "category_id may not be longer than %s characters"
+                % (MAX_GROUP_CATEGORYID_LENGTH,),
+                Codes.INVALID_PARAM,
+            )
+
+        resp = await self.handler.upsert_group_category(
+            group_id, requester_user_id, category_id, content
+        )
+
+        return 200, resp
+
+    async def on_DELETE(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        category_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        if category_id == "":
+            raise SynapseError(400, "category_id cannot be empty string")
+
+        resp = await self.handler.delete_group_category(
+            group_id, requester_user_id, category_id
+        )
+
+        return 200, resp
+
+
+class FederationGroupsRolesServlet(BaseGroupsServerServlet):
+    """Get roles in a group"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/roles/?"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        resp = await self.handler.get_group_roles(group_id, requester_user_id)
+
+        return 200, resp
+
+
+class FederationGroupsRoleServlet(BaseGroupsServerServlet):
+    """Add/remove/get a role in a group"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/roles/(?P<role_id>[^/]+)"
+
+    async def on_GET(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        role_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        resp = await self.handler.get_group_role(group_id, requester_user_id, role_id)
+
+        return 200, resp
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        role_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        if role_id == "":
+            raise SynapseError(
+                400, "role_id cannot be empty string", Codes.INVALID_PARAM
+            )
+
+        if len(role_id) > MAX_GROUP_ROLEID_LENGTH:
+            raise SynapseError(
+                400,
+                "role_id may not be longer than %s characters"
+                % (MAX_GROUP_ROLEID_LENGTH,),
+                Codes.INVALID_PARAM,
+            )
+
+        resp = await self.handler.update_group_role(
+            group_id, requester_user_id, role_id, content
+        )
+
+        return 200, resp
+
+    async def on_DELETE(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        role_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        if role_id == "":
+            raise SynapseError(400, "role_id cannot be empty string")
+
+        resp = await self.handler.delete_group_role(
+            group_id, requester_user_id, role_id
+        )
+
+        return 200, resp
+
+
+class FederationGroupsSummaryUsersServlet(BaseGroupsServerServlet):
+    """Add/remove a user from the group summary, with optional role.
+
+    Matches both:
+        - /groups/:group/summary/users/:user_id
+        - /groups/:group/summary/roles/:role/users/:user_id
+    """
+
+    PATH = (
+        "/groups/(?P<group_id>[^/]*)/summary"
+        "(/roles/(?P<role_id>[^/]+))?"
+        "/users/(?P<user_id>[^/]*)"
+    )
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        role_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        if role_id == "":
+            raise SynapseError(400, "role_id cannot be empty string")
+
+        if len(role_id) > MAX_GROUP_ROLEID_LENGTH:
+            raise SynapseError(
+                400,
+                "role_id may not be longer than %s characters"
+                % (MAX_GROUP_ROLEID_LENGTH,),
+                Codes.INVALID_PARAM,
+            )
+
+        resp = await self.handler.update_group_summary_user(
+            group_id,
+            requester_user_id,
+            user_id=user_id,
+            role_id=role_id,
+            content=content,
+        )
+
+        return 200, resp
+
+    async def on_DELETE(
+        self,
+        origin: str,
+        content: Literal[None],
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+        role_id: str,
+        user_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        if role_id == "":
+            raise SynapseError(400, "role_id cannot be empty string")
+
+        resp = await self.handler.delete_group_summary_user(
+            group_id, requester_user_id, user_id=user_id, role_id=role_id
+        )
+
+        return 200, resp
+
+
+class FederationGroupsSettingJoinPolicyServlet(BaseGroupsServerServlet):
+    """Sets whether a group is joinable without an invite or knock"""
+
+    PATH = "/groups/(?P<group_id>[^/]*)/settings/m.join_policy"
+
+    async def on_PUT(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Dict[bytes, List[bytes]],
+        group_id: str,
+    ) -> Tuple[int, JsonDict]:
+        requester_user_id = parse_string_from_args(
+            query, "requester_user_id", required=True
+        )
+        if get_domain_from_id(requester_user_id) != origin:
+            raise SynapseError(403, "requester_user_id doesn't match origin")
+
+        new_content = await self.handler.set_group_join_policy(
+            group_id, requester_user_id, content
+        )
+
+        return 200, new_content
+
+
+GROUP_SERVER_SERVLET_CLASSES: Tuple[Type[BaseFederationServlet], ...] = (
+    FederationGroupsProfileServlet,
+    FederationGroupsSummaryServlet,
+    FederationGroupsRoomsServlet,
+    FederationGroupsUsersServlet,
+    FederationGroupsInvitedUsersServlet,
+    FederationGroupsInviteServlet,
+    FederationGroupsAcceptInviteServlet,
+    FederationGroupsJoinServlet,
+    FederationGroupsRemoveUserServlet,
+    FederationGroupsSummaryRoomsServlet,
+    FederationGroupsCategoriesServlet,
+    FederationGroupsCategoryServlet,
+    FederationGroupsRolesServlet,
+    FederationGroupsRoleServlet,
+    FederationGroupsSummaryUsersServlet,
+    FederationGroupsAddRoomsServlet,
+    FederationGroupsAddRoomsConfigServlet,
+    FederationGroupsSettingJoinPolicyServlet,
+)
diff --git a/synapse/federation/units.py b/synapse/federation/units.py
index b662c42621..b9b12fbea5 100644
--- a/synapse/federation/units.py
+++ b/synapse/federation/units.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,17 +17,17 @@
 """
 
 import logging
+from typing import List, Optional
 
 import attr
 
 from synapse.types import JsonDict
-from synapse.util.jsonobject import JsonEncodedObject
 
 logger = logging.getLogger(__name__)
 
 
-@attr.s(slots=True)
-class Edu(JsonEncodedObject):
+@attr.s(slots=True, frozen=True, auto_attribs=True)
+class Edu:
     """An Edu represents a piece of data sent from one homeserver to another.
 
     In comparison to Pdus, Edus are not persisted for a long time on disk, are
@@ -36,10 +35,10 @@ class Edu(JsonEncodedObject):
     internal ID or previous references graph.
     """
 
-    edu_type = attr.ib(type=str)
-    content = attr.ib(type=dict)
-    origin = attr.ib(type=str)
-    destination = attr.ib(type=str)
+    edu_type: str
+    content: dict
+    origin: str
+    destination: str
 
     def get_dict(self) -> JsonDict:
         return {
@@ -55,14 +54,21 @@ def get_internal_dict(self) -> JsonDict:
             "destination": self.destination,
         }
 
-    def get_context(self):
+    def get_context(self) -> str:
         return getattr(self, "content", {}).get("org.matrix.opentracing_context", "{}")
 
-    def strip_context(self):
+    def strip_context(self) -> None:
         getattr(self, "content", {})["org.matrix.opentracing_context"] = "{}"
 
 
-class Transaction(JsonEncodedObject):
+def _none_to_list(edus: Optional[List[JsonDict]]) -> List[JsonDict]:
+    if edus is None:
+        return []
+    return edus
+
+
+@attr.s(slots=True, frozen=True, auto_attribs=True)
+class Transaction:
     """A transaction is a list of Pdus and Edus to be sent to a remote home
     server with some extra metadata.
 
@@ -78,47 +84,21 @@ class Transaction(JsonEncodedObject):
 
     """
 
-    valid_keys = [
-        "transaction_id",
-        "origin",
-        "destination",
-        "origin_server_ts",
-        "previous_ids",
-        "pdus",
-        "edus",
-    ]
-
-    internal_keys = ["transaction_id", "destination"]
-
-    required_keys = [
-        "transaction_id",
-        "origin",
-        "destination",
-        "origin_server_ts",
-        "pdus",
-    ]
-
-    def __init__(self, transaction_id=None, pdus=[], **kwargs):
-        """If we include a list of pdus then we decode then as PDU's
-        automatically.
-        """
-
-        # If there's no EDUs then remove the arg
-        if "edus" in kwargs and not kwargs["edus"]:
-            del kwargs["edus"]
-
-        super().__init__(transaction_id=transaction_id, pdus=pdus, **kwargs)
-
-    @staticmethod
-    def create_new(pdus, **kwargs):
-        """Used to create a new transaction. Will auto fill out
-        transaction_id and origin_server_ts keys.
-        """
-        if "origin_server_ts" not in kwargs:
-            raise KeyError("Require 'origin_server_ts' to construct a Transaction")
-        if "transaction_id" not in kwargs:
-            raise KeyError("Require 'transaction_id' to construct a Transaction")
-
-        kwargs["pdus"] = [p.get_pdu_json() for p in pdus]
-
-        return Transaction(**kwargs)
+    # Required keys.
+    transaction_id: str
+    origin: str
+    destination: str
+    origin_server_ts: int
+    pdus: List[JsonDict] = attr.ib(factory=list, converter=_none_to_list)
+    edus: List[JsonDict] = attr.ib(factory=list, converter=_none_to_list)
+
+    def get_dict(self) -> JsonDict:
+        """A JSON-ready dictionary of valid keys which aren't internal."""
+        result = {
+            "origin": self.origin,
+            "origin_server_ts": self.origin_server_ts,
+            "pdus": self.pdus,
+        }
+        if self.edus:
+            result["edus"] = self.edus
+        return result
diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py
index 368c44708d..ff8372c4e9 100644
--- a/synapse/groups/attestations.py
+++ b/synapse/groups/attestations.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -109,7 +108,9 @@ async def verify_attestation(
 
         assert server_name is not None
         await self.keyring.verify_json_for_server(
-            server_name, attestation, now, "Group attestation"
+            server_name,
+            attestation,
+            now,
         )
 
     def create_attestation(self, group_id: str, user_id: str) -> JsonDict:
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 4b16a4ac29..d6b75ac27f 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018 New Vector Ltd
 # Copyright 2019 Michael Telatynski <7t3chguy@gmail.com>
@@ -333,6 +332,13 @@ async def get_rooms_in_group(
             requester_user_id, group_id
         )
 
+        # Note! room_results["is_public"] is about whether the room is considered
+        # public from the group's point of view. (i.e. whether non-group members
+        # should be able to see the room is in the group).
+        # This is not the same as whether the room itself is public (in the sense
+        # of being visible in the room directory).
+        # As such, room_results["is_public"] itself is not sufficient to determine
+        # whether any given user is permitted to see the room's metadata.
         room_results = await self.store.get_rooms_in_group(
             group_id, include_private=is_user_in_group
         )
@@ -342,8 +348,15 @@ async def get_rooms_in_group(
             room_id = room_result["room_id"]
 
             joined_users = await self.store.get_users_in_room(room_id)
+
+            # check the user is actually allowed to see the room before showing it to them
+            allow_private = requester_user_id in joined_users
+
             entry = await self.room_list_handler.generate_room_entry(
-                room_id, len(joined_users), with_alias=False, allow_private=True
+                room_id,
+                len(joined_users),
+                with_alias=False,
+                allow_private=allow_private,
             )
 
             if not entry:
@@ -355,7 +368,7 @@ async def get_rooms_in_group(
 
         chunk.sort(key=lambda e: -e["num_joined_members"])
 
-        return {"chunk": chunk, "total_room_count_estimate": len(room_results)}
+        return {"chunk": chunk, "total_room_count_estimate": len(chunk)}
 
 
 class GroupsServerHandler(GroupsServerWorkerHandler):
@@ -708,9 +721,9 @@ async def _add_user(
         See accept_invite, join_group.
         """
         if not self.hs.is_mine_id(user_id):
-            local_attestation = self.attestations.create_attestation(
-                group_id, user_id
-            )  # type: Optional[JsonDict]
+            local_attestation: Optional[
+                JsonDict
+            ] = self.attestations.create_attestation(group_id, user_id)
 
             remote_attestation = content["attestation"]
 
@@ -869,9 +882,9 @@ async def create_group(
                 remote_attestation, user_id=requester_user_id, group_id=group_id
             )
 
-            local_attestation = self.attestations.create_attestation(
-                group_id, requester_user_id
-            )  # type: Optional[JsonDict]
+            local_attestation: Optional[
+                JsonDict
+            ] = self.attestations.create_attestation(group_id, requester_user_id)
         else:
             local_attestation = None
             remote_attestation = None
diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py
index bfebb0f644..5e83dba2ed 100644
--- a/synapse/handlers/__init__.py
+++ b/synapse/handlers/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index aade2c4a3a..6a05a65305 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,8 +15,6 @@
 import logging
 from typing import TYPE_CHECKING, Optional
 
-import synapse.state
-import synapse.storage
 import synapse.types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.ratelimiting import Ratelimiter
@@ -39,28 +36,29 @@ class BaseHandler:
     """
 
     def __init__(self, hs: "HomeServer"):
-        self.store = hs.get_datastore()  # type: synapse.storage.DataStore
+        self.store = hs.get_datastore()
         self.auth = hs.get_auth()
         self.notifier = hs.get_notifier()
-        self.state_handler = hs.get_state_handler()  # type: synapse.state.StateHandler
+        self.state_handler = hs.get_state_handler()
         self.distributor = hs.get_distributor()
         self.clock = hs.get_clock()
         self.hs = hs
 
         # The rate_hz and burst_count are overridden on a per-user basis
         self.request_ratelimiter = Ratelimiter(
-            clock=self.clock, rate_hz=0, burst_count=0
+            store=self.store, clock=self.clock, rate_hz=0, burst_count=0
         )
         self._rc_message = self.hs.config.rc_message
 
         # Check whether ratelimiting room admin message redaction is enabled
         # by the presence of rate limits in the config
         if self.hs.config.rc_admin_redaction:
-            self.admin_redaction_ratelimiter = Ratelimiter(
+            self.admin_redaction_ratelimiter: Optional[Ratelimiter] = Ratelimiter(
+                store=self.store,
                 clock=self.clock,
                 rate_hz=self.hs.config.rc_admin_redaction.per_second,
                 burst_count=self.hs.config.rc_admin_redaction.burst_count,
-            )  # type: Optional[Ratelimiter]
+            )
         else:
             self.admin_redaction_ratelimiter = None
 
@@ -91,11 +89,6 @@ async def ratelimit(self, requester, update=True, is_admin_redaction=False):
         if app_service is not None:
             return  # do not ratelimit app service senders
 
-        # Disable rate limiting of users belonging to any AS that is configured
-        # not to be rate limited in its registration file (rate_limited: true|false).
-        if requester.app_service and not requester.app_service.is_rate_limited():
-            return
-
         messages_per_second = self._rc_message.per_second
         burst_count = self._rc_message.burst_count
 
@@ -113,11 +106,11 @@ async def ratelimit(self, requester, update=True, is_admin_redaction=False):
         if is_admin_redaction and self.admin_redaction_ratelimiter:
             # If we have separate config for admin redactions, use a separate
             # ratelimiter as to not have user_ids clash
-            self.admin_redaction_ratelimiter.ratelimit(user_id, update=update)
+            await self.admin_redaction_ratelimiter.ratelimit(requester, update=update)
         else:
             # Override rate and burst count per-user
-            self.request_ratelimiter.ratelimit(
-                user_id,
+            await self.request_ratelimiter.ratelimit(
+                requester,
                 rate_hz=messages_per_second,
                 burst_count=burst_count,
                 update=update,
diff --git a/synapse/handlers/account_data.py b/synapse/handlers/account_data.py
index 1ce6d697ed..affb54e0ee 100644
--- a/synapse/handlers/account_data.py
+++ b/synapse/handlers/account_data.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py
index 125902ac17..4e02bd5a53 100644
--- a/synapse/handlers/account_validity.py
+++ b/synapse/handlers/account_validity.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,12 +15,11 @@
 import email.mime.multipart
 import email.utils
 import logging
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
-from typing import TYPE_CHECKING, List, Optional, Tuple
+from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional, Tuple
 
-from synapse.api.errors import StoreError, SynapseError
-from synapse.logging.context import make_deferred_yieldable
+from twisted.web.http import Request
+
+from synapse.api.errors import AuthError, StoreError, SynapseError
 from synapse.metrics.background_process_metrics import (
     run_as_background_process,
     wrap_as_background_process,
@@ -34,49 +32,55 @@
 
 logger = logging.getLogger(__name__)
 
+# Types for callbacks to be registered via the module api
+IS_USER_EXPIRED_CALLBACK = Callable[[str], Awaitable[Optional[bool]]]
+ON_USER_REGISTRATION_CALLBACK = Callable[[str], Awaitable]
+# Temporary hooks to allow for a transition from `/_matrix/client` endpoints
+# to `/_synapse/client/account_validity`. See `register_account_validity_callbacks`.
+ON_LEGACY_SEND_MAIL_CALLBACK = Callable[[str], Awaitable]
+ON_LEGACY_RENEW_CALLBACK = Callable[[str], Awaitable[Tuple[bool, bool, int]]]
+ON_LEGACY_ADMIN_REQUEST = Callable[[Request], Awaitable]
+
 
 class AccountValidityHandler:
     def __init__(self, hs: "HomeServer"):
         self.hs = hs
         self.config = hs.config
         self.store = self.hs.get_datastore()
-        self.sendmail = self.hs.get_sendmail()
+        self.send_email_handler = self.hs.get_send_email_handler()
         self.clock = self.hs.get_clock()
 
-        self._account_validity_enabled = self.hs.config.account_validity_enabled
+        self._app_name = self.hs.config.email_app_name
+
+        self._account_validity_enabled = (
+            hs.config.account_validity.account_validity_enabled
+        )
         self._account_validity_renew_by_email_enabled = (
-            self.hs.config.account_validity_renew_by_email_enabled
+            hs.config.account_validity.account_validity_renew_by_email_enabled
         )
+
         self._show_users_in_user_directory = self.hs.config.show_users_in_user_directory
-        self.profile_handler = self.hs.get_profile_handler()
 
         self._account_validity_period = None
         if self._account_validity_enabled:
-            self._account_validity_period = self.hs.config.account_validity_period
+            self._account_validity_period = (
+                hs.config.account_validity.account_validity_period
+            )
 
         if (
             self._account_validity_enabled
             and self._account_validity_renew_by_email_enabled
         ):
             # Don't do email-specific configuration if renewal by email is disabled.
-            self._template_html = self.config.account_validity_template_html
-            self._template_text = self.config.account_validity_template_text
-            account_validity_renew_email_subject = (
-                self.hs.config.account_validity_renew_email_subject
+            self._template_html = (
+                hs.config.account_validity.account_validity_template_html
+            )
+            self._template_text = (
+                hs.config.account_validity.account_validity_template_text
+            )
+            self._renew_email_subject = (
+                hs.config.account_validity.account_validity_renew_email_subject
             )
-
-            try:
-                app_name = self.hs.config.email_app_name
-
-                self._subject = account_validity_renew_email_subject % {"app": app_name}
-
-                self._from_string = self.hs.config.email_notif_from % {"app": app_name}
-            except Exception:
-                # If substitution failed, fall back to the bare strings.
-                self._subject = account_validity_renew_email_subject
-                self._from_string = self.hs.config.email_notif_from
-
-            self._raw_from = email.utils.parseaddr(self._from_string)[1]
 
             # Check the renewal emails to send and send them every 30min.
             if hs.config.run_background_tasks:
@@ -94,6 +98,99 @@ def mark_expired_users_as_inactive():
 
             self.clock.looping_call(mark_expired_users_as_inactive, 60 * 60 * 1000)
 
+        self._is_user_expired_callbacks: List[IS_USER_EXPIRED_CALLBACK] = []
+        self._on_user_registration_callbacks: List[ON_USER_REGISTRATION_CALLBACK] = []
+        self._on_legacy_send_mail_callback: Optional[
+            ON_LEGACY_SEND_MAIL_CALLBACK
+        ] = None
+        self._on_legacy_renew_callback: Optional[ON_LEGACY_RENEW_CALLBACK] = None
+
+        # The legacy admin requests callback isn't a protected attribute because we need
+        # to access it from the admin servlet, which is outside of this handler.
+        self.on_legacy_admin_request_callback: Optional[ON_LEGACY_ADMIN_REQUEST] = None
+
+    def register_account_validity_callbacks(
+        self,
+        is_user_expired: Optional[IS_USER_EXPIRED_CALLBACK] = None,
+        on_user_registration: Optional[ON_USER_REGISTRATION_CALLBACK] = None,
+        on_legacy_send_mail: Optional[ON_LEGACY_SEND_MAIL_CALLBACK] = None,
+        on_legacy_renew: Optional[ON_LEGACY_RENEW_CALLBACK] = None,
+        on_legacy_admin_request: Optional[ON_LEGACY_ADMIN_REQUEST] = None,
+    ):
+        """Register callbacks from module for each hook."""
+        if is_user_expired is not None:
+            self._is_user_expired_callbacks.append(is_user_expired)
+
+        if on_user_registration is not None:
+            self._on_user_registration_callbacks.append(on_user_registration)
+
+        # The builtin account validity feature exposes 3 endpoints (send_mail, renew, and
+        # an admin one). As part of moving the feature into a module, we need to change
+        # the path from /_matrix/client/unstable/account_validity/... to
+        # /_synapse/client/account_validity, because:
+        #
+        #   * the feature isn't part of the Matrix spec thus shouldn't live under /_matrix
+        #   * the way we register servlets means that modules can't register resources
+        #     under /_matrix/client
+        #
+        # We need to allow for a transition period between the old and new endpoints
+        # in order to allow for clients to update (and for emails to be processed).
+        #
+        # Once the email-account-validity module is loaded, it will take control of account
+        # validity by moving the rows from our `account_validity` table into its own table.
+        #
+        # Therefore, we need to allow modules (in practice just the one implementing the
+        # email-based account validity) to temporarily hook into the legacy endpoints so we
+        # can route the traffic coming into the old endpoints into the module, which is
+        # why we have the following three temporary hooks.
+        if on_legacy_send_mail is not None:
+            if self._on_legacy_send_mail_callback is not None:
+                raise RuntimeError("Tried to register on_legacy_send_mail twice")
+
+            self._on_legacy_send_mail_callback = on_legacy_send_mail
+
+        if on_legacy_renew is not None:
+            if self._on_legacy_renew_callback is not None:
+                raise RuntimeError("Tried to register on_legacy_renew twice")
+
+            self._on_legacy_renew_callback = on_legacy_renew
+
+        if on_legacy_admin_request is not None:
+            if self.on_legacy_admin_request_callback is not None:
+                raise RuntimeError("Tried to register on_legacy_admin_request twice")
+
+            self.on_legacy_admin_request_callback = on_legacy_admin_request
+
+    async def is_user_expired(self, user_id: str) -> bool:
+        """Checks if a user has expired against third-party modules.
+
+        Args:
+            user_id: The user to check the expiry of.
+
+        Returns:
+            Whether the user has expired.
+        """
+        for callback in self._is_user_expired_callbacks:
+            expired = await callback(user_id)
+            if expired is not None:
+                return expired
+
+        if self._account_validity_enabled:
+            # If no module could determine whether the user has expired and the legacy
+            # configuration is enabled, fall back to it.
+            return await self.store.is_account_expired(user_id, self.clock.time_msec())
+
+        return False
+
+    async def on_user_registration(self, user_id: str):
+        """Tell third-party modules about a user's registration.
+
+        Args:
+            user_id: The ID of the newly registered user.
+        """
+        for callback in self._on_user_registration_callbacks:
+            await callback(user_id)
+
     @wrap_as_background_process("send_renewals")
     async def _send_renewal_emails(self) -> None:
         """Gets the list of users whose account is expiring in the amount of time
@@ -119,6 +216,17 @@ async def send_renewal_email_to_user(self, user_id: str) -> None:
         Raises:
             SynapseError if the user is not set to renew.
         """
+        # If a module supports sending a renewal email from here, do that, otherwise do
+        # the legacy dance.
+        if self._on_legacy_send_mail_callback is not None:
+            await self._on_legacy_send_mail_callback(user_id)
+            return
+
+        if not self._account_validity_renew_by_email_enabled:
+            raise AuthError(
+                403, "Account renewal via email is disabled on this server."
+            )
+
         expiration_ts = await self.store.get_expiration_ts_for_user(user_id)
 
         # If this user isn't set to be expired, raise an error.
@@ -169,38 +277,17 @@ async def _send_renewal_email(self, user_id: str, expiration_ts: int) -> None:
         }
 
         html_text = self._template_html.render(**template_vars)
-        html_part = MIMEText(html_text, "html", "utf8")
-
         plain_text = self._template_text.render(**template_vars)
-        text_part = MIMEText(plain_text, "plain", "utf8")
 
         for address in addresses:
             raw_to = email.utils.parseaddr(address)[1]
 
-            multipart_msg = MIMEMultipart("alternative")
-            multipart_msg["Subject"] = self._subject
-            multipart_msg["From"] = self._from_string
-            multipart_msg["To"] = address
-            multipart_msg["Date"] = email.utils.formatdate()
-            multipart_msg["Message-ID"] = email.utils.make_msgid()
-            multipart_msg.attach(text_part)
-            multipart_msg.attach(html_part)
-
-            logger.info("Sending renewal email to %s", address)
-
-            await make_deferred_yieldable(
-                self.sendmail(
-                    self.hs.config.email_smtp_host,
-                    self._raw_from,
-                    raw_to,
-                    multipart_msg.as_string().encode("utf8"),
-                    reactor=self.hs.get_reactor(),
-                    port=self.hs.config.email_smtp_port,
-                    requireAuthentication=self.hs.config.email_smtp_user is not None,
-                    username=self.hs.config.email_smtp_user,
-                    password=self.hs.config.email_smtp_pass,
-                    requireTransportSecurity=self.hs.config.require_transport_security,
-                )
+            await self.send_email_handler.send_email(
+                email_address=raw_to,
+                subject=self._renew_email_subject,
+                app_name=self._app_name,
+                html=html_text,
+                text=plain_text,
             )
 
         await self.store.set_renewal_mail_status(user_id=user_id, email_sent=True)
@@ -254,6 +341,10 @@ async def renew_account(self, renewal_token: str) -> Tuple[bool, bool, int]:
         token is considered stale. A token is stale if the 'token_used_ts_ms' db column
         is non-null.
 
+        This method exists to support handling the legacy account validity /renew
+        endpoint. If a module implements the on_legacy_renew callback, then this process
+        is delegated to the module instead.
+
         Args:
             renewal_token: Token sent with the renewal request.
         Returns:
@@ -263,6 +354,11 @@ async def renew_account(self, renewal_token: str) -> Tuple[bool, bool, int]:
               * An int representing the user's expiry timestamp as milliseconds since the
                 epoch, or 0 if the token was invalid.
         """
+        # If a module supports triggering a renew from here, do that, otherwise do the
+        # legacy dance.
+        if self._on_legacy_renew_callback is not None:
+            return await self._on_legacy_renew_callback(renewal_token)
+
         try:
             (
                 user_id,
@@ -332,7 +428,12 @@ async def renew_account_for_user(
         # Check if renewed users should be reintroduced to the user directory
         if self._show_users_in_user_directory:
             # Show the user in the directory again by setting them to active
-            await self.profile_handler.set_active(
+            # We get the profile handler here so that we don't cause a cyclic dependency
+            # at startup.
+            # FIXME: this doesn't work with the new account validity module stuff, in
+            #  that this code is only run when no account validity module implements the
+            #  on_legacy_renew callback - which synapse-email-account-validity does.
+            await self.hs.get_profile_handler().set_active(
                 [UserID.from_string(user_id)], True, True
             )
 
@@ -349,4 +450,8 @@ async def _mark_expired_users_as_inactive(self):
         active_expired_users = await self.store.get_expired_users()
 
         # Mark each as non-active
-        await self.profile_handler.set_active(active_expired_users, False, True)
+        # We get the profile handler here so that we don't cause a cyclic dependency at
+        # startup.
+        await self.hs.get_profile_handler().set_active(
+            active_expired_users, False, True
+        )
diff --git a/synapse/handlers/acme.py b/synapse/handlers/acme.py
deleted file mode 100644
index 2a25af6288..0000000000
--- a/synapse/handlers/acme.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2019 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from typing import TYPE_CHECKING
-
-import twisted
-import twisted.internet.error
-from twisted.web import server, static
-from twisted.web.resource import Resource
-
-from synapse.app import check_bind_error
-
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
-logger = logging.getLogger(__name__)
-
-ACME_REGISTER_FAIL_ERROR = """
---------------------------------------------------------------------------------
-Failed to register with the ACME provider. This is likely happening because the installation
-is new, and ACME v1 has been deprecated by Let's Encrypt and disabled for
-new installations since November 2019.
-At the moment, Synapse doesn't support ACME v2. For more information and alternative
-solutions, please read https://github.com/matrix-org/synapse/blob/master/docs/ACME.md#deprecation-of-acme-v1
---------------------------------------------------------------------------------"""
-
-
-class AcmeHandler:
-    def __init__(self, hs: "HomeServer"):
-        self.hs = hs
-        self.reactor = hs.get_reactor()
-        self._acme_domain = hs.config.acme_domain
-
-    async def start_listening(self) -> None:
-        from synapse.handlers import acme_issuing_service
-
-        # Configure logging for txacme, if you need to debug
-        # from eliot import add_destinations
-        # from eliot.twisted import TwistedDestination
-        #
-        # add_destinations(TwistedDestination())
-
-        well_known = Resource()
-
-        self._issuer = acme_issuing_service.create_issuing_service(
-            self.reactor,
-            acme_url=self.hs.config.acme_url,
-            account_key_file=self.hs.config.acme_account_key_file,
-            well_known_resource=well_known,
-        )
-
-        responder_resource = Resource()
-        responder_resource.putChild(b".well-known", well_known)
-        responder_resource.putChild(b"check", static.Data(b"OK", b"text/plain"))
-        srv = server.Site(responder_resource)
-
-        bind_addresses = self.hs.config.acme_bind_addresses
-        for host in bind_addresses:
-            logger.info(
-                "Listening for ACME requests on %s:%i", host, self.hs.config.acme_port
-            )
-            try:
-                self.reactor.listenTCP(
-                    self.hs.config.acme_port, srv, backlog=50, interface=host
-                )
-            except twisted.internet.error.CannotListenError as e:
-                check_bind_error(e, host, bind_addresses)
-
-        # Make sure we are registered to the ACME server. There's no public API
-        # for this, it is usually triggered by startService, but since we don't
-        # want it to control where we save the certificates, we have to reach in
-        # and trigger the registration machinery ourselves.
-        self._issuer._registered = False
-
-        try:
-            await self._issuer._ensure_registered()
-        except Exception:
-            logger.error(ACME_REGISTER_FAIL_ERROR)
-            raise
-
-    async def provision_certificate(self) -> None:
-
-        logger.warning("Reprovisioning %s", self._acme_domain)
-
-        try:
-            await self._issuer.issue_cert(self._acme_domain)
-        except Exception:
-            logger.exception("Fail!")
-            raise
-        logger.warning("Reprovisioned %s, saving.", self._acme_domain)
-        cert_chain = self._issuer.cert_store.certs[self._acme_domain]
-
-        try:
-            with open(self.hs.config.tls_private_key_file, "wb") as private_key_file:
-                for x in cert_chain:
-                    if x.startswith(b"-----BEGIN RSA PRIVATE KEY-----"):
-                        private_key_file.write(x)
-
-            with open(self.hs.config.tls_certificate_file, "wb") as certificate_file:
-                for x in cert_chain:
-                    if x.startswith(b"-----BEGIN CERTIFICATE-----"):
-                        certificate_file.write(x)
-        except Exception:
-            logger.exception("Failed saving!")
-            raise
diff --git a/synapse/handlers/acme_issuing_service.py b/synapse/handlers/acme_issuing_service.py
deleted file mode 100644
index ae2a9dd9c2..0000000000
--- a/synapse/handlers/acme_issuing_service.py
+++ /dev/null
@@ -1,128 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2019 New Vector Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Utility function to create an ACME issuing service.
-
-This file contains the unconditional imports on the acme and cryptography bits that we
-only need (and may only have available) if we are doing ACME, so is designed to be
-imported conditionally.
-"""
-import logging
-from typing import Dict, Iterable, List
-
-import attr
-import pem
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
-from josepy import JWKRSA
-from josepy.jwa import RS256
-from txacme.challenges import HTTP01Responder
-from txacme.client import Client
-from txacme.interfaces import ICertificateStore
-from txacme.service import AcmeIssuingService
-from txacme.util import generate_private_key
-from zope.interface import implementer
-
-from twisted.internet import defer
-from twisted.internet.interfaces import IReactorTCP
-from twisted.python.filepath import FilePath
-from twisted.python.url import URL
-from twisted.web.resource import IResource
-
-logger = logging.getLogger(__name__)
-
-
-def create_issuing_service(
-    reactor: IReactorTCP,
-    acme_url: str,
-    account_key_file: str,
-    well_known_resource: IResource,
-) -> AcmeIssuingService:
-    """Create an ACME issuing service, and attach it to a web Resource
-
-    Args:
-        reactor: twisted reactor
-        acme_url: URL to use to request certificates
-        account_key_file: where to store the account key
-        well_known_resource: web resource for .well-known.
-            we will attach a child resource for "acme-challenge".
-
-    Returns:
-        AcmeIssuingService
-    """
-    responder = HTTP01Responder()
-
-    well_known_resource.putChild(b"acme-challenge", responder.resource)
-
-    store = ErsatzStore()
-
-    return AcmeIssuingService(
-        cert_store=store,
-        client_creator=(
-            lambda: Client.from_url(
-                reactor=reactor,
-                url=URL.from_text(acme_url),
-                key=load_or_create_client_key(account_key_file),
-                alg=RS256,
-            )
-        ),
-        clock=reactor,
-        responders=[responder],
-    )
-
-
-@attr.s(slots=True)
-@implementer(ICertificateStore)
-class ErsatzStore:
-    """
-    A store that only stores in memory.
-    """
-
-    certs = attr.ib(type=Dict[bytes, List[bytes]], default=attr.Factory(dict))
-
-    def store(
-        self, server_name: bytes, pem_objects: Iterable[pem.AbstractPEMObject]
-    ) -> defer.Deferred:
-        self.certs[server_name] = [o.as_bytes() for o in pem_objects]
-        return defer.succeed(None)
-
-
-def load_or_create_client_key(key_file: str) -> JWKRSA:
-    """Load the ACME account key from a file, creating it if it does not exist.
-
-    Args:
-        key_file: name of the file to use as the account key
-    """
-    # this is based on txacme.endpoint.load_or_create_client_key, but doesn't
-    # hardcode the 'client.key' filename
-    acme_key_file = FilePath(key_file)
-    if acme_key_file.exists():
-        logger.info("Loading ACME account key from '%s'", acme_key_file)
-        key = serialization.load_pem_private_key(
-            acme_key_file.getContent(), password=None, backend=default_backend()
-        )
-    else:
-        logger.info("Saving new ACME account key to '%s'", acme_key_file)
-        key = generate_private_key("rsa")
-        acme_key_file.setContent(
-            key.private_bytes(
-                encoding=serialization.Encoding.PEM,
-                format=serialization.PrivateFormat.TraditionalOpenSSL,
-                encryption_algorithm=serialization.NoEncryption(),
-            )
-        )
-    return JWKRSA(key=key)
diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py
index c494de49a3..bfa7f2c545 100644
--- a/synapse/handlers/admin.py
+++ b/synapse/handlers/admin.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -63,9 +62,16 @@ async def get_user(self, user: UserID) -> Optional[JsonDict]:
         if ret:
             profile = await self.store.get_profileinfo(user.localpart)
             threepids = await self.store.user_get_threepids(user.to_string())
+            external_ids = [
+                ({"auth_provider": auth_provider, "external_id": external_id})
+                for auth_provider, external_id in await self.store.get_external_ids_by_user(
+                    user.to_string()
+                )
+            ]
             ret["displayname"] = profile.display_name
             ret["avatar_url"] = profile.avatar_url
             ret["threepids"] = threepids
+            ret["external_ids"] = external_ids
         return ret
 
     async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") -> Any:
@@ -133,7 +139,7 @@ async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") ->
             to_key = RoomStreamToken(None, stream_ordering)
 
             # Events that we've processed in this room
-            written_events = set()  # type: Set[str]
+            written_events: Set[str] = set()
 
             # We need to track gaps in the events stream so that we can then
             # write out the state at those events. We do this by keeping track
@@ -146,7 +152,7 @@ async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") ->
             # The reverse mapping to above, i.e. map from unseen event to events
             # that have the unseen event in their prev_events, i.e. the unseen
             # events "children".
-            unseen_to_child_events = {}  # type: Dict[str, Set[str]]
+            unseen_to_child_events: Dict[str, Set[str]] = {}
 
             # We fetch events in the room the user could see by fetching *all*
             # events that we have and then filtering, this isn't the most
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 996f9e5deb..4ab4046650 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Collection, Dict, List, Optional, Union
 
 from prometheus_client import Counter
 
@@ -34,7 +33,7 @@
     wrap_as_background_process,
 )
 from synapse.storage.databases.main.directory import RoomAliasMapping
-from synapse.types import Collection, JsonDict, RoomAlias, RoomStreamToken, UserID
+from synapse.types import JsonDict, RoomAlias, RoomStreamToken, UserID
 from synapse.util.metrics import Measure
 
 if TYPE_CHECKING:
@@ -88,7 +87,8 @@ async def _notify_interested_services(self, max_token: RoomStreamToken):
             self.is_processing = True
             try:
                 limit = 100
-                while True:
+                upper_bound = -1
+                while upper_bound < self.current_max:
                     (
                         upper_bound,
                         events,
@@ -96,10 +96,7 @@ async def _notify_interested_services(self, max_token: RoomStreamToken):
                         self.current_max, limit
                     )
 
-                    if not events:
-                        break
-
-                    events_by_room = {}  # type: Dict[str, List[EventBase]]
+                    events_by_room: Dict[str, List[EventBase]] = {}
                     for event in events:
                         events_by_room.setdefault(event.room_id, []).append(event)
 
@@ -154,9 +151,6 @@ async def handle_room_events(events):
 
                     await self.store.set_appservice_last_pos(upper_bound)
 
-                    now = self.clock.time_msec()
-                    ts = await self.store.get_received_ts(events[-1].event_id)
-
                     synapse.metrics.event_processing_positions.labels(
                         "appservice_sender"
                     ).set(upper_bound)
@@ -169,12 +163,16 @@ async def handle_room_events(events):
 
                     event_processing_loop_counter.labels("appservice_sender").inc()
 
-                    synapse.metrics.event_processing_lag.labels(
-                        "appservice_sender"
-                    ).set(now - ts)
-                    synapse.metrics.event_processing_last_ts.labels(
-                        "appservice_sender"
-                    ).set(ts)
+                    if events:
+                        now = self.clock.time_msec()
+                        ts = await self.store.get_received_ts(events[-1].event_id)
+
+                        synapse.metrics.event_processing_lag.labels(
+                            "appservice_sender"
+                        ).set(now - ts)
+                        synapse.metrics.event_processing_last_ts.labels(
+                            "appservice_sender"
+                        ).set(ts)
             finally:
                 self.is_processing = False
 
@@ -182,7 +180,7 @@ def notify_interested_services_ephemeral(
         self,
         stream_key: str,
         new_token: Optional[int],
-        users: Collection[Union[str, UserID]] = [],
+        users: Optional[Collection[Union[str, UserID]]] = None,
     ):
         """This is called by the notifier in the background
         when a ephemeral event handled by the homeserver.
@@ -215,7 +213,7 @@ def notify_interested_services_ephemeral(
         # We only start a new background process if necessary rather than
         # optimistically (to cut down on overhead).
         self._notify_interested_services_ephemeral(
-            services, stream_key, new_token, users
+            services, stream_key, new_token, users or []
         )
 
     @wrap_as_background_process("notify_interested_services_ephemeral")
@@ -277,7 +275,7 @@ async def _handle_receipts(self, service: ApplicationService) -> List[JsonDict]:
     async def _handle_presence(
         self, service: ApplicationService, users: Collection[Union[str, UserID]]
     ) -> List[JsonDict]:
-        events = []  # type: List[JsonDict]
+        events: List[JsonDict] = []
         presence_source = self.event_sources.sources["presence"]
         from_key = await self.store.get_type_stream_id_for_appservice(
             service, "presence"
@@ -377,7 +375,7 @@ async def get_3pe_protocols(
         self, only_protocol: Optional[str] = None
     ) -> Dict[str, JsonDict]:
         services = self.store.get_app_services()
-        protocols = {}  # type: Dict[str, List[JsonDict]]
+        protocols: Dict[str, List[JsonDict]] = {}
 
         # Collect up all the individual protocol responses out of the ASes
         for s in services:
@@ -394,9 +392,6 @@ async def get_3pe_protocols(
                     protocols[p].append(info)
 
         def _merge_instances(infos: List[JsonDict]) -> JsonDict:
-            if not infos:
-                return {}
-
             # Merge the 'instances' lists of multiple results, but just take
             # the other fields from the first as they ought to be identical
             # copy the result so as not to corrupt the cached one
@@ -408,7 +403,9 @@ def _merge_instances(infos: List[JsonDict]) -> JsonDict:
 
             return combined
 
-        return {p: _merge_instances(protocols[p]) for p in protocols.keys()}
+        return {
+            p: _merge_instances(protocols[p]) for p in protocols.keys() if protocols[p]
+        }
 
     async def _get_services_for_event(
         self, event: EventBase
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index d537ea8137..161b3c933c 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2019 - 2020 The Matrix.org Foundation C.I.C.
@@ -18,6 +17,7 @@
 import time
 import unicodedata
 import urllib.parse
+from binascii import crc32
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -30,11 +30,13 @@
     Optional,
     Tuple,
     Union,
+    cast,
 )
 
 import attr
 import bcrypt
 import pymacaroons
+import unpaddedbase64
 
 from twisted.web.server import Request
 
@@ -67,9 +69,11 @@
 from synapse.util.async_helpers import maybe_awaitable
 from synapse.util.macaroons import get_value_from_macaroon, satisfy_expiry
 from synapse.util.msisdn import phone_number_to_msisdn
+from synapse.util.stringutils import base62_encode
 from synapse.util.threepids import canonicalise_email
 
 if TYPE_CHECKING:
+    from synapse.rest.client.login import LoginResponse
     from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
@@ -187,7 +191,7 @@ class AuthHandler(BaseHandler):
     def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
-        self.checkers = {}  # type: Dict[str, UserInteractiveAuthChecker]
+        self.checkers: Dict[str, UserInteractiveAuthChecker] = {}
         for auth_checker_class in INTERACTIVE_AUTH_CHECKERS:
             inst = auth_checker_class(hs)
             if inst.is_enabled():
@@ -238,6 +242,7 @@ def __init__(self, hs: "HomeServer"):
         # Ratelimiter for failed auth during UIA. Uses same ratelimit config
         # as per `rc_login.failed_attempts`.
         self._failed_uia_attempts_ratelimiter = Ratelimiter(
+            store=self.store,
             clock=self.clock,
             rate_hz=self.hs.config.rc_login_failed_attempts.per_second,
             burst_count=self.hs.config.rc_login_failed_attempts.burst_count,
@@ -248,6 +253,7 @@ def __init__(self, hs: "HomeServer"):
 
         # Ratelimitier for failed /login attempts
         self._failed_login_attempts_ratelimiter = Ratelimiter(
+            store=self.store,
             clock=hs.get_clock(),
             rate_hz=self.hs.config.rc_login_failed_attempts.per_second,
             burst_count=self.hs.config.rc_login_failed_attempts.burst_count,
@@ -290,7 +296,7 @@ def __init__(self, hs: "HomeServer"):
 
         # A mapping of user ID to extra attributes to include in the login
         # response.
-        self._extra_attributes = {}  # type: Dict[str, SsoLoginExtraAttributes]
+        self._extra_attributes: Dict[str, SsoLoginExtraAttributes] = {}
 
     async def validate_user_via_ui_auth(
         self,
@@ -298,6 +304,7 @@ async def validate_user_via_ui_auth(
         request: SynapseRequest,
         request_body: Dict[str, Any],
         description: str,
+        can_skip_ui_auth: bool = False,
     ) -> Tuple[dict, Optional[str]]:
         """
         Checks that the user is who they claim to be, via a UI auth.
@@ -316,6 +323,10 @@ async def validate_user_via_ui_auth(
             description: A human readable string to be displayed to the user that
                          describes the operation happening on their account.
 
+            can_skip_ui_auth: True if the UI auth session timeout applies this
+                              action. Should be set to False for any "dangerous"
+                              actions (e.g. deactivating an account).
+
         Returns:
             A tuple of (params, session_id).
 
@@ -339,7 +350,7 @@ async def validate_user_via_ui_auth(
         """
         if not requester.access_token_id:
             raise ValueError("Cannot validate a user without an access token")
-        if self._ui_auth_session_timeout:
+        if can_skip_ui_auth and self._ui_auth_session_timeout:
             last_validated = await self.store.get_access_token_last_validated(
                 requester.access_token_id
             )
@@ -352,7 +363,7 @@ async def validate_user_via_ui_auth(
         requester_user_id = requester.user.to_string()
 
         # Check if we should be ratelimited due to too many previous failed attempts
-        self._failed_uia_attempts_ratelimiter.ratelimit(requester_user_id, update=False)
+        await self._failed_uia_attempts_ratelimiter.ratelimit(requester, update=False)
 
         # build a list of supported flows
         supported_ui_auth_types = await self._get_available_ui_auth_types(
@@ -373,7 +384,9 @@ def get_new_session_data() -> JsonDict:
             )
         except LoginError:
             # Update the ratelimiter to say we failed (`can_do_action` doesn't raise).
-            self._failed_uia_attempts_ratelimiter.can_do_action(requester_user_id)
+            await self._failed_uia_attempts_ratelimiter.can_do_action(
+                requester,
+            )
             raise
 
         # find the completed login type
@@ -448,7 +461,7 @@ async def check_ui_auth(
 
         If no auth flows have been completed successfully, raises an
         InteractiveAuthIncompleteError. To handle this, you can use
-        synapse.rest.client.v2_alpha._base.interactive_auth_handler as a
+        synapse.rest.client._base.interactive_auth_handler as a
         decorator.
 
         Args:
@@ -487,7 +500,7 @@ async def check_ui_auth(
                 all the stages in any of the permitted flows.
         """
 
-        sid = None  # type: Optional[str]
+        sid: Optional[str] = None
         authdict = clientdict.pop("auth", {})
         if "session" in authdict:
             sid = authdict["session"]
@@ -530,7 +543,7 @@ async def check_ui_auth(
             # Note that the registration endpoint explicitly removes the
             # "initial_device_display_name" parameter if it is provided
             # without a "password" parameter. See the changes to
-            # synapse.rest.client.v2_alpha.register.RegisterRestServlet.on_POST
+            # synapse.rest.client.register.RegisterRestServlet.on_POST
             # in commit 544722bad23fc31056b9240189c3cbbbf0ffd3f9.
             if not clientdict:
                 clientdict = session.clientdict
@@ -575,9 +588,9 @@ async def check_ui_auth(
             )
 
         # check auth type currently being presented
-        errordict = {}  # type: Dict[str, Any]
+        errordict: Dict[str, Any] = {}
         if "type" in authdict:
-            login_type = authdict["type"]  # type: str
+            login_type: str = authdict["type"]
             try:
                 result = await self._check_auth_dict(authdict, clientip)
                 if result:
@@ -753,7 +766,7 @@ def _auth_dict_for_flows(
             LoginType.TERMS: self._get_params_terms,
         }
 
-        params = {}  # type: Dict[str, Any]
+        params: Dict[str, Any] = {}
 
         for f in public_flows:
             for stage in f:
@@ -766,6 +779,108 @@ def _auth_dict_for_flows(
             "params": params,
         }
 
+    async def refresh_token(
+        self,
+        refresh_token: str,
+        valid_until_ms: Optional[int],
+    ) -> Tuple[str, str]:
+        """
+        Consumes a refresh token and generate both a new access token and a new refresh token from it.
+
+        The consumed refresh token is considered invalid after the first use of the new access token or the new refresh token.
+
+        Args:
+            refresh_token: The token to consume.
+            valid_until_ms: The expiration timestamp of the new access token.
+
+        Returns:
+            A tuple containing the new access token and refresh token
+        """
+
+        # Verify the token signature first before looking up the token
+        if not self._verify_refresh_token(refresh_token):
+            raise SynapseError(401, "invalid refresh token", Codes.UNKNOWN_TOKEN)
+
+        existing_token = await self.store.lookup_refresh_token(refresh_token)
+        if existing_token is None:
+            raise SynapseError(401, "refresh token does not exist", Codes.UNKNOWN_TOKEN)
+
+        if (
+            existing_token.has_next_access_token_been_used
+            or existing_token.has_next_refresh_token_been_refreshed
+        ):
+            raise SynapseError(
+                403, "refresh token isn't valid anymore", Codes.FORBIDDEN
+            )
+
+        (
+            new_refresh_token,
+            new_refresh_token_id,
+        ) = await self.get_refresh_token_for_user_id(
+            user_id=existing_token.user_id, device_id=existing_token.device_id
+        )
+        access_token = await self.get_access_token_for_user_id(
+            user_id=existing_token.user_id,
+            device_id=existing_token.device_id,
+            valid_until_ms=valid_until_ms,
+            refresh_token_id=new_refresh_token_id,
+        )
+        await self.store.replace_refresh_token(
+            existing_token.token_id, new_refresh_token_id
+        )
+        return access_token, new_refresh_token
+
+    def _verify_refresh_token(self, token: str) -> bool:
+        """
+        Verifies the shape of a refresh token.
+
+        Args:
+            token: The refresh token to verify
+
+        Returns:
+            Whether the token has the right shape
+        """
+        parts = token.split("_", maxsplit=4)
+        if len(parts) != 4:
+            return False
+
+        type, localpart, rand, crc = parts
+
+        # Refresh tokens are prefixed by "syr_", let's check that
+        if type != "syr":
+            return False
+
+        # Check the CRC
+        base = f"{type}_{localpart}_{rand}"
+        expected_crc = base62_encode(crc32(base.encode("ascii")), minwidth=6)
+        if crc != expected_crc:
+            return False
+
+        return True
+
+    async def get_refresh_token_for_user_id(
+        self,
+        user_id: str,
+        device_id: str,
+    ) -> Tuple[str, int]:
+        """
+        Creates a new refresh token for the user with the given user ID.
+
+        Args:
+            user_id: canonical user ID
+            device_id: the device ID to associate with the token.
+
+        Returns:
+            The newly created refresh token and its ID in the database
+        """
+        refresh_token = self.generate_refresh_token(UserID.from_string(user_id))
+        refresh_token_id = await self.store.add_refresh_token_to_user(
+            user_id=user_id,
+            token=refresh_token,
+            device_id=device_id,
+        )
+        return refresh_token, refresh_token_id
+
     async def get_access_token_for_user_id(
         self,
         user_id: str,
@@ -773,6 +888,7 @@ async def get_access_token_for_user_id(
         valid_until_ms: Optional[int],
         puppets_user_id: Optional[str] = None,
         is_appservice_ghost: bool = False,
+        refresh_token_id: Optional[int] = None,
     ) -> str:
         """
         Creates a new access token for the user with the given user ID.
@@ -790,6 +906,8 @@ async def get_access_token_for_user_id(
             valid_until_ms: when the token is valid until. None for
                 no expiry.
             is_appservice_ghost: Whether the user is an application ghost user
+            refresh_token_id: the refresh token ID that will be associated with
+                this access token.
         Returns:
               The access token for the user's session.
         Raises:
@@ -805,10 +923,12 @@ async def get_access_token_for_user_id(
             logger.info(
                 "Logging in user %s as %s%s", user_id, puppets_user_id, fmt_expiry
             )
+            target_user_id_obj = UserID.from_string(puppets_user_id)
         else:
             logger.info(
                 "Logging in user %s on device %s%s", user_id, device_id, fmt_expiry
             )
+            target_user_id_obj = UserID.from_string(user_id)
 
         if (
             not is_appservice_ghost
@@ -816,13 +936,14 @@ async def get_access_token_for_user_id(
         ):
             await self.auth.check_auth_blocking(user_id)
 
-        access_token = self.macaroon_gen.generate_access_token(user_id)
+        access_token = self.generate_access_token(target_user_id_obj)
         await self.store.add_access_token_to_user(
             user_id=user_id,
             token=access_token,
             device_id=device_id,
             valid_until_ms=valid_until_ms,
             puppets_user_id=puppets_user_id,
+            refresh_token_id=refresh_token_id,
         )
 
         # the device *should* have been registered before we got here; however,
@@ -915,7 +1036,7 @@ async def validate_login(
         self,
         login_submission: Dict[str, Any],
         ratelimit: bool = False,
-    ) -> Tuple[str, Optional[Callable[[Dict[str, str]], Awaitable[None]]]]:
+    ) -> Tuple[str, Optional[Callable[["LoginResponse"], Awaitable[None]]]]:
         """Authenticates the user for the /login API
 
         Also used by the user-interactive auth flow to validate auth types which don't
@@ -982,8 +1103,8 @@ async def validate_login(
             # We also apply account rate limiting using the 3PID as a key, as
             # otherwise using 3PID bypasses the ratelimiting based on user ID.
             if ratelimit:
-                self._failed_login_attempts_ratelimiter.ratelimit(
-                    (medium, address), update=False
+                await self._failed_login_attempts_ratelimiter.ratelimit(
+                    None, (medium, address), update=False
                 )
 
             # Check for login providers that support 3pid login types
@@ -1016,8 +1137,8 @@ async def validate_login(
                 # this code path, which is fine as then the per-user ratelimit
                 # will kick in below.
                 if ratelimit:
-                    self._failed_login_attempts_ratelimiter.can_do_action(
-                        (medium, address)
+                    await self._failed_login_attempts_ratelimiter.can_do_action(
+                        None, (medium, address)
                     )
                 raise LoginError(403, "", errcode=Codes.FORBIDDEN)
 
@@ -1039,8 +1160,8 @@ async def validate_login(
 
         # Check if we've hit the failed ratelimit (but don't update it)
         if ratelimit:
-            self._failed_login_attempts_ratelimiter.ratelimit(
-                qualified_user_id.lower(), update=False
+            await self._failed_login_attempts_ratelimiter.ratelimit(
+                None, qualified_user_id.lower(), update=False
             )
 
         try:
@@ -1051,8 +1172,8 @@ async def validate_login(
             # exception and masking the LoginError. The actual ratelimiting
             # should have happened above.
             if ratelimit:
-                self._failed_login_attempts_ratelimiter.can_do_action(
-                    qualified_user_id.lower()
+                await self._failed_login_attempts_ratelimiter.can_do_action(
+                    None, qualified_user_id.lower()
                 )
             raise
 
@@ -1060,7 +1181,7 @@ async def _validate_userid_login(
         self,
         username: str,
         login_submission: Dict[str, Any],
-    ) -> Tuple[str, Optional[Callable[[Dict[str, str]], Awaitable[None]]]]:
+    ) -> Tuple[str, Optional[Callable[["LoginResponse"], Awaitable[None]]]]:
         """Helper for validate_login
 
         Handles login, once we've mapped 3pids onto userids
@@ -1138,7 +1259,7 @@ async def _validate_userid_login(
 
     async def check_password_provider_3pid(
         self, medium: str, address: str, password: str
-    ) -> Tuple[Optional[str], Optional[Callable[[Dict[str, str]], Awaitable[None]]]]:
+    ) -> Tuple[Optional[str], Optional[Callable[["LoginResponse"], Awaitable[None]]]]:
         """Check if a password provider is able to validate a thirdparty login
 
         Args:
@@ -1189,6 +1310,32 @@ async def _check_local_password(self, user_id: str, password: str) -> Optional[s
             return None
         return user_id
 
+    def generate_access_token(self, for_user: UserID) -> str:
+        """Generates an opaque string, for use as an access token"""
+
+        # we use the following format for access tokens:
+        #    syt_<base64 local part>_<random string>_<base62 crc check>
+
+        b64local = unpaddedbase64.encode_base64(for_user.localpart.encode("utf-8"))
+        random_string = stringutils.random_string(20)
+        base = f"syt_{b64local}_{random_string}"
+
+        crc = base62_encode(crc32(base.encode("ascii")), minwidth=6)
+        return f"{base}_{crc}"
+
+    def generate_refresh_token(self, for_user: UserID) -> str:
+        """Generates an opaque string, for use as a refresh token"""
+
+        # we use the following format for refresh tokens:
+        #    syr_<base64 local part>_<random string>_<base62 crc check>
+
+        b64local = unpaddedbase64.encode_base64(for_user.localpart.encode("utf-8"))
+        random_string = stringutils.random_string(20)
+        base = f"syr_{b64local}_{random_string}"
+
+        crc = base62_encode(crc32(base.encode("ascii")), minwidth=6)
+        return f"{base}_{crc}"
+
     async def validate_short_term_login_token(
         self, login_token: str
     ) -> LoginTokenAttributes:
@@ -1245,7 +1392,7 @@ async def delete_access_tokens_for_user(
 
         # see if any of our auth providers want to know about this
         for provider in self.password_providers:
-            for token, token_id, device_id in tokens_and_devices:
+            for token, _, device_id in tokens_and_devices:
                 await provider.on_logged_out(
                     user_id=user_id, device_id=device_id, access_token=token
                 )
@@ -1383,9 +1530,9 @@ async def start_sso_ui_auth(self, request: SynapseRequest, session_id: str) -> s
         except StoreError:
             raise SynapseError(400, "Unknown session ID: %s" % (session_id,))
 
-        user_id_to_verify = await self.get_session_data(
+        user_id_to_verify: str = await self.get_session_data(
             session_id, UIAuthSessionDataConstants.REQUEST_USER_ID
-        )  # type: str
+        )
 
         idps = await self.hs.get_sso_handler().get_identity_providers_for_user(
             user_id_to_verify
@@ -1537,7 +1684,7 @@ def _complete_sso_login(
         )
         respond_with_html(request, 200, html)
 
-    async def _sso_login_callback(self, login_result: JsonDict) -> None:
+    async def _sso_login_callback(self, login_result: "LoginResponse") -> None:
         """
         A login callback which might add additional attributes to the login response.
 
@@ -1551,7 +1698,8 @@ async def _sso_login_callback(self, login_result: JsonDict) -> None:
 
         extra_attributes = self._extra_attributes.get(login_result["user_id"])
         if extra_attributes:
-            login_result.update(extra_attributes.extra_attributes)
+            login_result_dict = cast(Dict[str, Any], login_result)
+            login_result_dict.update(extra_attributes.extra_attributes)
 
     def _expire_sso_extra_attributes(self) -> None:
         """
@@ -1582,10 +1730,7 @@ class MacaroonGenerator:
 
     hs = attr.ib()
 
-    def generate_access_token(
-        self, user_id: str, extra_caveats: Optional[List[str]] = None
-    ) -> str:
-        extra_caveats = extra_caveats or []
+    def generate_guest_access_token(self, user_id: str) -> str:
         macaroon = self._generate_base_macaroon(user_id)
         macaroon.add_first_party_caveat("type = access")
         # Include a nonce, to make sure that each login gets a different
@@ -1593,8 +1738,7 @@ def generate_access_token(
         macaroon.add_first_party_caveat(
             "nonce = %s" % (stringutils.random_string_with_symbols(16),)
         )
-        for caveat in extra_caveats:
-            macaroon.add_first_party_caveat(caveat)
+        macaroon.add_first_party_caveat("guest = true")
         return macaroon.serialize()
 
     def generate_short_term_login_token(
diff --git a/synapse/handlers/cas_handler.py b/synapse/handlers/cas.py
similarity index 98%
rename from synapse/handlers/cas_handler.py
rename to synapse/handlers/cas.py
index 5060936f94..0325f86e20 100644
--- a/synapse/handlers/cas_handler.py
+++ b/synapse/handlers/cas.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -41,7 +40,7 @@ def __init__(self, error, error_description=None):
 
     def __str__(self):
         if self.error_description:
-            return "{}: {}".format(self.error, self.error_description)
+            return f"{self.error}: {self.error_description}"
         return self.error
 
 
@@ -172,7 +171,7 @@ def _parse_cas_response(self, cas_response_body: bytes) -> CasResponse:
 
         # Iterate through the nodes and pull out the user and any extra attributes.
         user = None
-        attributes = {}  # type: Dict[str, List[Optional[str]]]
+        attributes: Dict[str, List[Optional[str]]] = {}
         for child in root[0]:
             if child.tag.endswith("user"):
                 user = child.text
diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index 9167918257..d61da72efc 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017, 2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
@@ -50,7 +49,9 @@ def __init__(self, hs: "HomeServer"):
         if hs.config.run_background_tasks:
             hs.get_reactor().callWhenRunning(self._start_user_parting)
 
-        self._account_validity_enabled = hs.config.account_validity_enabled
+        self._account_validity_enabled = (
+            hs.config.account_validity.account_validity_enabled
+        )
 
     async def deactivate_account(
         self,
diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py
index 54293d0b9c..46ee834407 100644
--- a/synapse/handlers/device.py
+++ b/synapse/handlers/device.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 # Copyright 2019,2020 The Matrix.org Foundation C.I.C.
@@ -15,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Collection, Dict, Iterable, List, Optional, Set, Tuple
 
 from synapse.api import errors
 from synapse.api.constants import EventTypes
@@ -29,7 +28,6 @@
 from synapse.logging.opentracing import log_kv, set_tag, trace
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import (
-    Collection,
     JsonDict,
     StreamToken,
     UserID,
@@ -157,8 +155,7 @@ async def get_user_ids_changed(
             # The user may have left the room
             # TODO: Check if they actually did or if we were just invited.
             if room_id not in room_ids:
-                for key, event_id in current_state_ids.items():
-                    etype, state_key = key
+                for etype, state_key in current_state_ids.keys():
                     if etype != EventTypes.Member:
                         continue
                     possibly_left.add(state_key)
@@ -180,8 +177,7 @@ async def get_user_ids_changed(
                 log_kv(
                     {"event": "encountered empty previous state", "room_id": room_id}
                 )
-                for key, event_id in current_state_ids.items():
-                    etype, state_key = key
+                for etype, state_key in current_state_ids.keys():
                     if etype != EventTypes.Member:
                         continue
                     possibly_changed.add(state_key)
@@ -199,8 +195,7 @@ async def get_user_ids_changed(
             for state_dict in prev_state_ids.values():
                 member_event = state_dict.get((EventTypes.Member, user_id), None)
                 if not member_event or member_event != current_member_id:
-                    for key, event_id in current_state_ids.items():
-                        etype, state_key = key
+                    for etype, state_key in current_state_ids.keys():
                         if etype != EventTypes.Member:
                             continue
                         possibly_changed.add(state_key)
@@ -457,7 +452,7 @@ async def notify_device_update(
             user_id
         )
 
-        hosts = set()  # type: Set[str]
+        hosts: Set[str] = set()
         if self.hs.is_mine_id(user_id):
             hosts.update(get_domain_from_id(u) for u in users_who_share_room)
             hosts.discard(self.server_name)
@@ -618,14 +613,14 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceHandler):
         self._remote_edu_linearizer = Linearizer(name="remote_device_list")
 
         # user_id -> list of updates waiting to be handled.
-        self._pending_updates = (
-            {}
-        )  # type: Dict[str, List[Tuple[str, str, Iterable[str], JsonDict]]]
+        self._pending_updates: Dict[
+            str, List[Tuple[str, str, Iterable[str], JsonDict]]
+        ] = {}
 
         # Recently seen stream ids. We don't bother keeping these in the DB,
         # but they're useful to have them about to reduce the number of spurious
         # resyncs.
-        self._seen_updates = ExpiringCache(
+        self._seen_updates: ExpiringCache[str, Set[str]] = ExpiringCache(
             cache_name="device_update_edu",
             clock=self.clock,
             max_len=10000,
@@ -715,7 +710,7 @@ async def _handle_device_updates(self, user_id: str) -> None:
                 # This can happen since we batch updates
                 return
 
-            for device_id, stream_id, prev_ids, content in pending_updates:
+            for device_id, stream_id, prev_ids, _ in pending_updates:
                 logger.debug(
                     "Handling update %r/%r, ID: %r, prev: %r ",
                     user_id,
@@ -741,7 +736,7 @@ async def _handle_device_updates(self, user_id: str) -> None:
             else:
                 # Simply update the single device, since we know that is the only
                 # change (because of the single prev_id matching the current cache)
-                for device_id, stream_id, prev_ids, content in pending_updates:
+                for device_id, stream_id, _, content in pending_updates:
                     await self.store.update_remote_device_list_cache_entry(
                         user_id, device_id, content, stream_id
                     )
@@ -760,7 +755,7 @@ async def _need_to_do_resync(
         """Given a list of updates for a user figure out if we need to do a full
         resync, or whether we have enough data that we can just apply the delta.
         """
-        seen_updates = self._seen_updates.get(user_id, set())
+        seen_updates: Set[str] = self._seen_updates.get(user_id, set())
 
         extremity = await self.store.get_device_list_last_stream_id_for_remote(user_id)
 
@@ -930,6 +925,10 @@ async def user_device_resync(
         else:
             cached_devices = await self.store.get_cached_devices_for_user(user_id)
             if cached_devices == {d["device_id"]: d for d in devices}:
+                logging.info(
+                    "Skipping device list resync for %s, as our cache matches already",
+                    user_id,
+                )
                 devices = []
                 ignore_devices = True
 
@@ -945,6 +944,9 @@ async def user_device_resync(
             await self.store.update_remote_device_list_cache(
                 user_id, devices, stream_id
             )
+        # mark the cache as valid, whether or not we actually processed any device
+        # list updates.
+        await self.store.mark_remote_user_device_cache_as_valid(user_id)
         device_ids = [device["device_id"] for device in devices]
 
         # Handle cross-signing keys.
diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py
index eb547743be..679b47f081 100644
--- a/synapse/handlers/devicemessage.py
+++ b/synapse/handlers/devicemessage.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,15 +15,15 @@
 import logging
 from typing import TYPE_CHECKING, Any, Dict
 
-from synapse.api.constants import EduTypes
+from synapse.api.constants import ToDeviceEventTypes
 from synapse.api.errors import SynapseError
 from synapse.api.ratelimiting import Ratelimiter
 from synapse.logging.context import run_in_background
 from synapse.logging.opentracing import (
+    SynapseTags,
     get_active_span_text_map,
     log_kv,
     set_tag,
-    start_active_span,
 )
 from synapse.replication.http.devices import ReplicationUserDevicesResyncRestServlet
 from synapse.types import JsonDict, Requester, UserID, get_domain_from_id
@@ -80,7 +79,10 @@ def __init__(self, hs: "HomeServer"):
                 ReplicationUserDevicesResyncRestServlet.make_client(hs)
             )
 
+        # a rate limiter for room key requests.  The keys are
+        # (sending_user_id, sending_device_id).
         self._ratelimiter = Ratelimiter(
+            store=self.store,
             clock=hs.get_clock(),
             rate_hz=hs.config.rc_key_requests.per_second,
             burst_count=hs.config.rc_key_requests.burst_count,
@@ -100,12 +102,25 @@ async def on_direct_to_device_edu(self, origin: str, content: JsonDict) -> None:
         for user_id, by_device in content["messages"].items():
             # we use UserID.from_string to catch invalid user ids
             if not self.is_mine(UserID.from_string(user_id)):
-                logger.warning("Request for keys for non-local user %s", user_id)
+                logger.warning("To-device message to non-local user %s", user_id)
                 raise SynapseError(400, "Not a user here")
 
             if not by_device:
                 continue
 
+            # Ratelimit key requests by the sending user.
+            if message_type == ToDeviceEventTypes.RoomKeyRequest:
+                allowed, _ = await self._ratelimiter.can_do_action(
+                    None, (sender_user_id, None)
+                )
+                if not allowed:
+                    logger.info(
+                        "Dropping room_key_request from %s to %s due to rate limit",
+                        sender_user_id,
+                        user_id,
+                    )
+                    continue
+
             messages_by_device = {
                 device_id: {
                     "content": message_content,
@@ -182,20 +197,29 @@ async def send_device_message(
     ) -> None:
         sender_user_id = requester.user.to_string()
 
-        set_tag("number_of_messages", len(messages))
+        message_id = random_string(16)
+        set_tag(SynapseTags.TO_DEVICE_MESSAGE_ID, message_id)
+
+        log_kv({"number_of_to_device_messages": len(messages)})
         set_tag("sender", sender_user_id)
         local_messages = {}
-        remote_messages = {}  # type: Dict[str, Dict[str, Dict[str, JsonDict]]]
+        remote_messages: Dict[str, Dict[str, Dict[str, JsonDict]]] = {}
         for user_id, by_device in messages.items():
             # Ratelimit local cross-user key requests by the sending device.
             if (
-                message_type == EduTypes.RoomKeyRequest
+                message_type == ToDeviceEventTypes.RoomKeyRequest
                 and user_id != sender_user_id
-                and self._ratelimiter.can_do_action(
-                    (sender_user_id, requester.device_id)
-                )
             ):
-                continue
+                allowed, _ = await self._ratelimiter.can_do_action(
+                    requester, (sender_user_id, requester.device_id)
+                )
+                if not allowed:
+                    logger.info(
+                        "Dropping room_key_request from %s to %s due to rate limit",
+                        sender_user_id,
+                        user_id,
+                    )
+                    continue
 
             # we use UserID.from_string to catch invalid user ids
             if self.is_mine(UserID.from_string(user_id)):
@@ -204,32 +228,35 @@ async def send_device_message(
                         "content": message_content,
                         "type": message_type,
                         "sender": sender_user_id,
+                        "message_id": message_id,
                     }
                     for device_id, message_content in by_device.items()
                 }
                 if messages_by_device:
                     local_messages[user_id] = messages_by_device
+                    log_kv(
+                        {
+                            "user_id": user_id,
+                            "device_id": list(messages_by_device),
+                        }
+                    )
             else:
                 destination = get_domain_from_id(user_id)
                 remote_messages.setdefault(destination, {})[user_id] = by_device
 
-        message_id = random_string(16)
-
         context = get_active_span_text_map()
 
         remote_edu_contents = {}
         for destination, messages in remote_messages.items():
-            with start_active_span("to_device_for_user"):
-                set_tag("destination", destination)
-                remote_edu_contents[destination] = {
-                    "messages": messages,
-                    "sender": sender_user_id,
-                    "type": message_type,
-                    "message_id": message_id,
-                    "org.matrix.opentracing_context": json_encoder.encode(context),
-                }
+            log_kv({"destination": destination})
+            remote_edu_contents[destination] = {
+                "messages": messages,
+                "sender": sender_user_id,
+                "type": message_type,
+                "message_id": message_id,
+                "org.matrix.opentracing_context": json_encoder.encode(context),
+            }
 
-        log_kv({"local_messages": local_messages})
         stream_id = await self.store.add_messages_to_device_inbox(
             local_messages, remote_edu_contents
         )
@@ -238,7 +265,6 @@ async def send_device_message(
             "to_device_key", stream_id, users=local_messages.keys()
         )
 
-        log_kv({"remote_messages": remote_messages})
         if self.federation_sender:
             for destination in remote_messages.keys():
                 # Enqueue a new federation transaction to send the new
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index abcf86352d..d487fee627 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +14,7 @@
 
 import logging
 import string
-from typing import Iterable, List, Optional
+from typing import TYPE_CHECKING, Iterable, List, Optional
 
 from synapse.api.constants import MAX_ALIAS_LENGTH, EventTypes
 from synapse.api.errors import (
@@ -23,20 +22,25 @@
     CodeMessageException,
     Codes,
     NotFoundError,
+    RequestSendFailed,
     ShadowBanError,
     StoreError,
     SynapseError,
 )
 from synapse.appservice import ApplicationService
-from synapse.types import Requester, RoomAlias, UserID, get_domain_from_id
+from synapse.storage.databases.main.directory import RoomAliasMapping
+from synapse.types import JsonDict, Requester, RoomAlias, UserID, get_domain_from_id
 
 from ._base import BaseHandler
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
 class DirectoryHandler(BaseHandler):
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.state = hs.get_state_handler()
@@ -61,7 +65,7 @@ async def _create_association(
         room_id: str,
         servers: Optional[Iterable[str]] = None,
         creator: Optional[str] = None,
-    ):
+    ) -> None:
         # general association creation for both human users and app services
 
         for wchar in string.whitespace:
@@ -75,7 +79,7 @@ async def _create_association(
         # TODO(erikj): Add transactions.
         # TODO(erikj): Check if there is a current association.
         if not servers:
-            users = await self.state.get_current_users_in_room(room_id)
+            users = await self.store.get_users_in_room(room_id)
             servers = {get_domain_from_id(u) for u in users}
 
         if not servers:
@@ -105,8 +109,9 @@ async def create_association(
         """
 
         user_id = requester.user.to_string()
+        room_alias_str = room_alias.to_string()
 
-        if len(room_alias.to_string()) > MAX_ALIAS_LENGTH:
+        if len(room_alias_str) > MAX_ALIAS_LENGTH:
             raise SynapseError(
                 400,
                 "Can't create aliases longer than %s characters" % MAX_ALIAS_LENGTH,
@@ -115,7 +120,7 @@ async def create_association(
 
         service = requester.app_service
         if service:
-            if not service.is_interested_in_alias(room_alias.to_string()):
+            if not service.is_interested_in_alias(room_alias_str):
                 raise SynapseError(
                     400,
                     "This application service has not reserved this kind of alias.",
@@ -139,7 +144,7 @@ async def create_association(
                 raise AuthError(403, "This user is not permitted to create this alias")
 
             if not self.config.is_alias_creation_allowed(
-                user_id, room_id, room_alias.to_string()
+                user_id, room_id, room_alias_str
             ):
                 # Lets just return a generic message, as there may be all sorts of
                 # reasons why we said no. TODO: Allow configurable error messages
@@ -212,7 +217,7 @@ async def delete_association(
 
     async def delete_appservice_association(
         self, service: ApplicationService, room_alias: RoomAlias
-    ):
+    ) -> None:
         if not service.is_interested_in_alias(room_alias.to_string()):
             raise SynapseError(
                 400,
@@ -221,7 +226,7 @@ async def delete_appservice_association(
             )
         await self._delete_association(room_alias)
 
-    async def _delete_association(self, room_alias: RoomAlias):
+    async def _delete_association(self, room_alias: RoomAlias) -> str:
         if not self.hs.is_mine(room_alias):
             raise SynapseError(400, "Room alias must be local")
 
@@ -229,33 +234,37 @@ async def _delete_association(self, room_alias: RoomAlias):
 
         return room_id
 
-    async def get_association(self, room_alias: RoomAlias):
+    async def get_association(self, room_alias: RoomAlias) -> JsonDict:
         room_id = None
         if self.hs.is_mine(room_alias):
-            result = await self.get_association_from_room_alias(room_alias)
+            result: Optional[
+                RoomAliasMapping
+            ] = await self.get_association_from_room_alias(room_alias)
 
             if result:
                 room_id = result.room_id
                 servers = result.servers
         else:
             try:
-                result = await self.federation.make_query(
+                fed_result = await self.federation.make_query(
                     destination=room_alias.domain,
                     query_type="directory",
                     args={"room_alias": room_alias.to_string()},
                     retry_on_dns_fail=False,
                     ignore_backoff=True,
                 )
+            except RequestSendFailed:
+                raise SynapseError(502, "Failed to fetch alias")
             except CodeMessageException as e:
                 logging.warning("Error retrieving alias")
                 if e.code == 404:
-                    result = None
+                    fed_result = None
                 else:
-                    raise
+                    raise SynapseError(502, "Failed to fetch alias")
 
-            if result and "room_id" in result and "servers" in result:
-                room_id = result["room_id"]
-                servers = result["servers"]
+            if fed_result and "room_id" in fed_result and "servers" in fed_result:
+                room_id = fed_result["room_id"]
+                servers = fed_result["servers"]
 
         if not room_id:
             raise SynapseError(
@@ -264,7 +273,7 @@ async def get_association(self, room_alias: RoomAlias):
                 Codes.NOT_FOUND,
             )
 
-        users = await self.state.get_current_users_in_room(room_id)
+        users = await self.store.get_users_in_room(room_id)
         extra_servers = {get_domain_from_id(u) for u in users}
         servers = set(extra_servers) | set(servers)
 
@@ -276,7 +285,7 @@ async def get_association(self, room_alias: RoomAlias):
 
         return {"room_id": room_id, "servers": servers}
 
-    async def on_directory_query(self, args):
+    async def on_directory_query(self, args: JsonDict) -> JsonDict:
         room_alias = RoomAlias.from_string(args["room_alias"])
         if not self.hs.is_mine(room_alias):
             raise SynapseError(400, "Room Alias is not hosted on this homeserver")
@@ -294,7 +303,7 @@ async def on_directory_query(self, args):
 
     async def _update_canonical_alias(
         self, requester: Requester, user_id: str, room_id: str, room_alias: RoomAlias
-    ):
+    ) -> None:
         """
         Send an updated canonical alias event if the removed alias was set as
         the canonical alias or listed in the alt_aliases field.
@@ -345,7 +354,9 @@ async def _update_canonical_alias(
                 ratelimit=False,
             )
 
-    async def get_association_from_room_alias(self, room_alias: RoomAlias):
+    async def get_association_from_room_alias(
+        self, room_alias: RoomAlias
+    ) -> Optional[RoomAliasMapping]:
         result = await self.store.get_association_from_room_alias(room_alias)
         if not result:
             # Query AS to see if it exists
@@ -373,7 +384,7 @@ def can_modify_alias(self, alias: RoomAlias, user_id: Optional[str] = None) -> b
         # either no interested services, or no service with an exclusive lock
         return True
 
-    async def _user_can_delete_alias(self, alias: RoomAlias, user_id: str):
+    async def _user_can_delete_alias(self, alias: RoomAlias, user_id: str) -> bool:
         """Determine whether a user can delete an alias.
 
         One of the following must be true:
@@ -395,14 +406,13 @@ async def _user_can_delete_alias(self, alias: RoomAlias, user_id: str):
         if not room_id:
             return False
 
-        res = await self.auth.check_can_change_room_list(
+        return await self.auth.check_can_change_room_list(
             room_id, UserID.from_string(user_id)
         )
-        return res
 
     async def edit_published_room_list(
         self, requester: Requester, room_id: str, visibility: str
-    ):
+    ) -> None:
         """Edit the entry of the room in the published room list.
 
         requester
@@ -470,7 +480,7 @@ async def edit_published_room_list(
 
     async def edit_published_appservice_room_list(
         self, appservice_id: str, network_id: str, room_id: str, visibility: str
-    ):
+    ) -> None:
         """Add or remove a room from the appservice/network specific public
         room list.
 
@@ -500,5 +510,4 @@ async def get_aliases_for_room(
                 room_id, requester.user.to_string()
             )
 
-        aliases = await self.store.get_aliases_for_room(room_id)
-        return aliases
+        return await self.store.get_aliases_for_room(room_id)
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 2ad9b6d930..d92370859f 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2018-2019 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -38,7 +37,6 @@
 )
 from synapse.util import json_decoder, unwrapFirstError
 from synapse.util.async_helpers import Linearizer
-from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.retryutils import NotRetryingDestination
 
 if TYPE_CHECKING:
@@ -81,9 +79,15 @@ def __init__(self, hs: "HomeServer"):
             "client_keys", self.on_federation_query_client_keys
         )
 
+        # Limit the number of in-flight requests from a single device.
+        self._query_devices_linearizer = Linearizer(
+            name="query_devices",
+            max_count=10,
+        )
+
     @trace
     async def query_devices(
-        self, query_body: JsonDict, timeout: int, from_user_id: str
+        self, query_body: JsonDict, timeout: int, from_user_id: str, from_device_id: str
     ) -> JsonDict:
         """Handle a device key query from a client
 
@@ -107,191 +111,195 @@ async def query_devices(
             from_user_id: the user making the query.  This is used when
                 adding cross-signing signatures to limit what signatures users
                 can see.
+            from_device_id: the device making the query. This is used to limit
+                the number of in-flight queries at a time.
         """
+        with await self._query_devices_linearizer.queue((from_user_id, from_device_id)):
+            device_keys_query: Dict[str, Iterable[str]] = query_body.get(
+                "device_keys", {}
+            )
 
-        device_keys_query = query_body.get(
-            "device_keys", {}
-        )  # type: Dict[str, Iterable[str]]
-
-        # separate users by domain.
-        # make a map from domain to user_id to device_ids
-        local_query = {}
-        remote_queries = {}
-
-        for user_id, device_ids in device_keys_query.items():
-            # we use UserID.from_string to catch invalid user ids
-            if self.is_mine(UserID.from_string(user_id)):
-                local_query[user_id] = device_ids
-            else:
-                remote_queries[user_id] = device_ids
-
-        set_tag("local_key_query", local_query)
-        set_tag("remote_key_query", remote_queries)
-
-        # First get local devices.
-        # A map of destination -> failure response.
-        failures = {}  # type: Dict[str, JsonDict]
-        results = {}
-        if local_query:
-            local_result = await self.query_local_devices(local_query)
-            for user_id, keys in local_result.items():
-                if user_id in local_query:
-                    results[user_id] = keys
-
-        # Get cached cross-signing keys
-        cross_signing_keys = await self.get_cross_signing_keys_from_cache(
-            device_keys_query, from_user_id
-        )
+            # separate users by domain.
+            # make a map from domain to user_id to device_ids
+            local_query = {}
+            remote_queries = {}
 
-        # Now attempt to get any remote devices from our local cache.
-        # A map of destination -> user ID -> device IDs.
-        remote_queries_not_in_cache = {}  # type: Dict[str, Dict[str, Iterable[str]]]
-        if remote_queries:
-            query_list = []  # type: List[Tuple[str, Optional[str]]]
-            for user_id, device_ids in remote_queries.items():
-                if device_ids:
-                    query_list.extend((user_id, device_id) for device_id in device_ids)
+            for user_id, device_ids in device_keys_query.items():
+                # we use UserID.from_string to catch invalid user ids
+                if self.is_mine(UserID.from_string(user_id)):
+                    local_query[user_id] = device_ids
                 else:
-                    query_list.append((user_id, None))
-
-            (
-                user_ids_not_in_cache,
-                remote_results,
-            ) = await self.store.get_user_devices_from_cache(query_list)
-            for user_id, devices in remote_results.items():
-                user_devices = results.setdefault(user_id, {})
-                for device_id, device in devices.items():
-                    keys = device.get("keys", None)
-                    device_display_name = device.get("device_display_name", None)
-                    if keys:
-                        result = dict(keys)
-                        unsigned = result.setdefault("unsigned", {})
-                        if device_display_name:
-                            unsigned["device_display_name"] = device_display_name
-                        user_devices[device_id] = result
-
-            # check for missing cross-signing keys.
-            for user_id in remote_queries.keys():
-                cached_cross_master = user_id in cross_signing_keys["master_keys"]
-                cached_cross_selfsigning = (
-                    user_id in cross_signing_keys["self_signing_keys"]
-                )
-
-                # check if we are missing only one of cross-signing master or
-                # self-signing key, but the other one is cached.
-                # as we need both, this will issue a federation request.
-                # if we don't have any of the keys, either the user doesn't have
-                # cross-signing set up, or the cached device list
-                # is not (yet) updated.
-                if cached_cross_master ^ cached_cross_selfsigning:
-                    user_ids_not_in_cache.add(user_id)
-
-            # add those users to the list to fetch over federation.
-            for user_id in user_ids_not_in_cache:
-                domain = get_domain_from_id(user_id)
-                r = remote_queries_not_in_cache.setdefault(domain, {})
-                r[user_id] = remote_queries[user_id]
-
-        # Now fetch any devices that we don't have in our cache
-        @trace
-        async def do_remote_query(destination):
-            """This is called when we are querying the device list of a user on
-            a remote homeserver and their device list is not in the device list
-            cache. If we share a room with this user and we're not querying for
-            specific user we will update the cache with their device list.
-            """
-
-            destination_query = remote_queries_not_in_cache[destination]
-
-            # We first consider whether we wish to update the device list cache with
-            # the users device list. We want to track a user's devices when the
-            # authenticated user shares a room with the queried user and the query
-            # has not specified a particular device.
-            # If we update the cache for the queried user we remove them from further
-            # queries. We use the more efficient batched query_client_keys for all
-            # remaining users
-            user_ids_updated = []
-            for (user_id, device_list) in destination_query.items():
-                if user_id in user_ids_updated:
-                    continue
-
-                if device_list:
-                    continue
+                    remote_queries[user_id] = device_ids
+
+            set_tag("local_key_query", local_query)
+            set_tag("remote_key_query", remote_queries)
+
+            # First get local devices.
+            # A map of destination -> failure response.
+            failures: Dict[str, JsonDict] = {}
+            results = {}
+            if local_query:
+                local_result = await self.query_local_devices(local_query)
+                for user_id, keys in local_result.items():
+                    if user_id in local_query:
+                        results[user_id] = keys
 
-                room_ids = await self.store.get_rooms_for_user(user_id)
-                if not room_ids:
-                    continue
+            # Get cached cross-signing keys
+            cross_signing_keys = await self.get_cross_signing_keys_from_cache(
+                device_keys_query, from_user_id
+            )
 
-                # We've decided we're sharing a room with this user and should
-                # probably be tracking their device lists. However, we haven't
-                # done an initial sync on the device list so we do it now.
-                try:
-                    if self._is_master:
-                        user_devices = await self.device_handler.device_list_updater.user_device_resync(
-                            user_id
+            # Now attempt to get any remote devices from our local cache.
+            # A map of destination -> user ID -> device IDs.
+            remote_queries_not_in_cache: Dict[str, Dict[str, Iterable[str]]] = {}
+            if remote_queries:
+                query_list: List[Tuple[str, Optional[str]]] = []
+                for user_id, device_ids in remote_queries.items():
+                    if device_ids:
+                        query_list.extend(
+                            (user_id, device_id) for device_id in device_ids
                         )
                     else:
-                        user_devices = await self._user_device_resync_client(
-                            user_id=user_id
-                        )
+                        query_list.append((user_id, None))
 
-                    user_devices = user_devices["devices"]
-                    user_results = results.setdefault(user_id, {})
-                    for device in user_devices:
-                        user_results[device["device_id"]] = device["keys"]
-                    user_ids_updated.append(user_id)
-                except Exception as e:
-                    failures[destination] = _exception_to_failure(e)
-
-            if len(destination_query) == len(user_ids_updated):
-                # We've updated all the users in the query and we do not need to
-                # make any further remote calls.
-                return
-
-            # Remove all the users from the query which we have updated
-            for user_id in user_ids_updated:
-                destination_query.pop(user_id)
+                (
+                    user_ids_not_in_cache,
+                    remote_results,
+                ) = await self.store.get_user_devices_from_cache(query_list)
+                for user_id, devices in remote_results.items():
+                    user_devices = results.setdefault(user_id, {})
+                    for device_id, device in devices.items():
+                        keys = device.get("keys", None)
+                        device_display_name = device.get("device_display_name", None)
+                        if keys:
+                            result = dict(keys)
+                            unsigned = result.setdefault("unsigned", {})
+                            if device_display_name:
+                                unsigned["device_display_name"] = device_display_name
+                            user_devices[device_id] = result
+
+                # check for missing cross-signing keys.
+                for user_id in remote_queries.keys():
+                    cached_cross_master = user_id in cross_signing_keys["master_keys"]
+                    cached_cross_selfsigning = (
+                        user_id in cross_signing_keys["self_signing_keys"]
+                    )
 
-            try:
-                remote_result = await self.federation.query_client_keys(
-                    destination, {"device_keys": destination_query}, timeout=timeout
-                )
+                    # check if we are missing only one of cross-signing master or
+                    # self-signing key, but the other one is cached.
+                    # as we need both, this will issue a federation request.
+                    # if we don't have any of the keys, either the user doesn't have
+                    # cross-signing set up, or the cached device list
+                    # is not (yet) updated.
+                    if cached_cross_master ^ cached_cross_selfsigning:
+                        user_ids_not_in_cache.add(user_id)
+
+                # add those users to the list to fetch over federation.
+                for user_id in user_ids_not_in_cache:
+                    domain = get_domain_from_id(user_id)
+                    r = remote_queries_not_in_cache.setdefault(domain, {})
+                    r[user_id] = remote_queries[user_id]
+
+            # Now fetch any devices that we don't have in our cache
+            @trace
+            async def do_remote_query(destination):
+                """This is called when we are querying the device list of a user on
+                a remote homeserver and their device list is not in the device list
+                cache. If we share a room with this user and we're not querying for
+                specific user we will update the cache with their device list.
+                """
+
+                destination_query = remote_queries_not_in_cache[destination]
+
+                # We first consider whether we wish to update the device list cache with
+                # the users device list. We want to track a user's devices when the
+                # authenticated user shares a room with the queried user and the query
+                # has not specified a particular device.
+                # If we update the cache for the queried user we remove them from further
+                # queries. We use the more efficient batched query_client_keys for all
+                # remaining users
+                user_ids_updated = []
+                for (user_id, device_list) in destination_query.items():
+                    if user_id in user_ids_updated:
+                        continue
+
+                    if device_list:
+                        continue
+
+                    room_ids = await self.store.get_rooms_for_user(user_id)
+                    if not room_ids:
+                        continue
+
+                    # We've decided we're sharing a room with this user and should
+                    # probably be tracking their device lists. However, we haven't
+                    # done an initial sync on the device list so we do it now.
+                    try:
+                        if self._is_master:
+                            user_devices = await self.device_handler.device_list_updater.user_device_resync(
+                                user_id
+                            )
+                        else:
+                            user_devices = await self._user_device_resync_client(
+                                user_id=user_id
+                            )
+
+                        user_devices = user_devices["devices"]
+                        user_results = results.setdefault(user_id, {})
+                        for device in user_devices:
+                            user_results[device["device_id"]] = device["keys"]
+                        user_ids_updated.append(user_id)
+                    except Exception as e:
+                        failures[destination] = _exception_to_failure(e)
+
+                if len(destination_query) == len(user_ids_updated):
+                    # We've updated all the users in the query and we do not need to
+                    # make any further remote calls.
+                    return
+
+                # Remove all the users from the query which we have updated
+                for user_id in user_ids_updated:
+                    destination_query.pop(user_id)
 
-                for user_id, keys in remote_result["device_keys"].items():
-                    if user_id in destination_query:
-                        results[user_id] = keys
+                try:
+                    remote_result = await self.federation.query_client_keys(
+                        destination, {"device_keys": destination_query}, timeout=timeout
+                    )
 
-                if "master_keys" in remote_result:
-                    for user_id, key in remote_result["master_keys"].items():
+                    for user_id, keys in remote_result["device_keys"].items():
                         if user_id in destination_query:
-                            cross_signing_keys["master_keys"][user_id] = key
+                            results[user_id] = keys
 
-                if "self_signing_keys" in remote_result:
-                    for user_id, key in remote_result["self_signing_keys"].items():
-                        if user_id in destination_query:
-                            cross_signing_keys["self_signing_keys"][user_id] = key
+                    if "master_keys" in remote_result:
+                        for user_id, key in remote_result["master_keys"].items():
+                            if user_id in destination_query:
+                                cross_signing_keys["master_keys"][user_id] = key
 
-            except Exception as e:
-                failure = _exception_to_failure(e)
-                failures[destination] = failure
-                set_tag("error", True)
-                set_tag("reason", failure)
+                    if "self_signing_keys" in remote_result:
+                        for user_id, key in remote_result["self_signing_keys"].items():
+                            if user_id in destination_query:
+                                cross_signing_keys["self_signing_keys"][user_id] = key
 
-        await make_deferred_yieldable(
-            defer.gatherResults(
-                [
-                    run_in_background(do_remote_query, destination)
-                    for destination in remote_queries_not_in_cache
-                ],
-                consumeErrors=True,
-            ).addErrback(unwrapFirstError)
-        )
+                except Exception as e:
+                    failure = _exception_to_failure(e)
+                    failures[destination] = failure
+                    set_tag("error", True)
+                    set_tag("reason", failure)
+
+            await make_deferred_yieldable(
+                defer.gatherResults(
+                    [
+                        run_in_background(do_remote_query, destination)
+                        for destination in remote_queries_not_in_cache
+                    ],
+                    consumeErrors=True,
+                ).addErrback(unwrapFirstError)
+            )
 
-        ret = {"device_keys": results, "failures": failures}
+            ret = {"device_keys": results, "failures": failures}
 
-        ret.update(cross_signing_keys)
+            ret.update(cross_signing_keys)
 
-        return ret
+            return ret
 
     async def get_cross_signing_keys_from_cache(
         self, query: Iterable[str], from_user_id: Optional[str]
@@ -352,9 +360,9 @@ async def query_local_devices(
             A map from user_id -> device_id -> device details
         """
         set_tag("local_query", query)
-        local_query = []  # type: List[Tuple[str, Optional[str]]]
+        local_query: List[Tuple[str, Optional[str]]] = []
 
-        result_dict = {}  # type: Dict[str, Dict[str, dict]]
+        result_dict: Dict[str, Dict[str, dict]] = {}
         for user_id, device_ids in query.items():
             # we use UserID.from_string to catch invalid user ids
             if not self.is_mine(UserID.from_string(user_id)):
@@ -392,9 +400,9 @@ async def on_federation_query_client_keys(
         self, query_body: Dict[str, Dict[str, Optional[List[str]]]]
     ) -> JsonDict:
         """Handle a device key query from a federated server"""
-        device_keys_query = query_body.get(
+        device_keys_query: Dict[str, Optional[List[str]]] = query_body.get(
             "device_keys", {}
-        )  # type: Dict[str, Optional[List[str]]]
+        )
         res = await self.query_local_devices(device_keys_query)
         ret = {"device_keys": res}
 
@@ -411,8 +419,8 @@ async def on_federation_query_client_keys(
     async def claim_one_time_keys(
         self, query: Dict[str, Dict[str, Dict[str, str]]], timeout: int
     ) -> JsonDict:
-        local_query = []  # type: List[Tuple[str, str, str]]
-        remote_queries = {}  # type: Dict[str, Dict[str, Dict[str, str]]]
+        local_query: List[Tuple[str, str, str]] = []
+        remote_queries: Dict[str, Dict[str, Dict[str, str]]] = {}
 
         for user_id, one_time_keys in query.get("one_time_keys", {}).items():
             # we use UserID.from_string to catch invalid user ids
@@ -429,8 +437,8 @@ async def claim_one_time_keys(
         results = await self.store.claim_e2e_one_time_keys(local_query)
 
         # A map of user ID -> device ID -> key ID -> key.
-        json_result = {}  # type: Dict[str, Dict[str, Dict[str, JsonDict]]]
-        failures = {}  # type: Dict[str, JsonDict]
+        json_result: Dict[str, Dict[str, Dict[str, JsonDict]]] = {}
+        failures: Dict[str, JsonDict] = {}
         for user_id, device_keys in results.items():
             for device_id, keys in device_keys.items():
                 for key_id, json_str in keys.items():
@@ -758,8 +766,8 @@ async def _process_self_signatures(
         Raises:
             SynapseError: if the input is malformed
         """
-        signature_list = []  # type: List[SignatureListItem]
-        failures = {}  # type: Dict[str, Dict[str, JsonDict]]
+        signature_list: List["SignatureListItem"] = []
+        failures: Dict[str, Dict[str, JsonDict]] = {}
         if not signatures:
             return signature_list, failures
 
@@ -920,8 +928,8 @@ async def _process_other_signatures(
         Raises:
             SynapseError: if the input is malformed
         """
-        signature_list = []  # type: List[SignatureListItem]
-        failures = {}  # type: Dict[str, Dict[str, JsonDict]]
+        signature_list: List["SignatureListItem"] = []
+        failures: Dict[str, Dict[str, JsonDict]] = {}
         if not signatures:
             return signature_list, failures
 
@@ -1008,7 +1016,7 @@ async def _process_other_signatures(
         return signature_list, failures
 
     async def _get_e2e_cross_signing_verify_key(
-        self, user_id: str, key_type: str, from_user_id: str = None
+        self, user_id: str, key_type: str, from_user_id: Optional[str] = None
     ) -> Tuple[JsonDict, str, VerifyKey]:
         """Fetch locally or remotely query for a cross-signing public key.
 
@@ -1290,18 +1298,7 @@ def __init__(self, hs: "HomeServer", e2e_keys_handler: E2eKeysHandler):
         self._remote_edu_linearizer = Linearizer(name="remote_signing_key")
 
         # user_id -> list of updates waiting to be handled.
-        self._pending_updates = {}  # type: Dict[str, List[Tuple[JsonDict, JsonDict]]]
-
-        # Recently seen stream ids. We don't bother keeping these in the DB,
-        # but they're useful to have them about to reduce the number of spurious
-        # resyncs.
-        self._seen_updates = ExpiringCache(
-            cache_name="signing_key_update_edu",
-            clock=self.clock,
-            max_len=10000,
-            expiry_ms=30 * 60 * 1000,
-            iterable=True,
-        )
+        self._pending_updates: Dict[str, List[Tuple[JsonDict, JsonDict]]] = {}
 
     async def incoming_signing_key_update(
         self, origin: str, edu_content: JsonDict
@@ -1350,7 +1347,7 @@ async def _handle_signing_key_updates(self, user_id: str) -> None:
                 # This can happen since we batch updates
                 return
 
-            device_ids = []  # type: List[str]
+            device_ids: List[str] = []
 
             logger.info("pending updates: %r", pending_updates)
 
diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index a910d246d6..31742236a9 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017, 2018 New Vector Ltd
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py
new file mode 100644
index 0000000000..4288ffff09
--- /dev/null
+++ b/synapse/handlers/event_auth.py
@@ -0,0 +1,311 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from typing import TYPE_CHECKING, Collection, List, Optional, Union
+
+from synapse import event_auth
+from synapse.api.constants import (
+    EventTypes,
+    JoinRules,
+    Membership,
+    RestrictedJoinRuleTypes,
+)
+from synapse.api.errors import AuthError, Codes, SynapseError
+from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion
+from synapse.events import EventBase
+from synapse.events.builder import EventBuilder
+from synapse.types import StateMap, get_domain_from_id
+from synapse.util.metrics import Measure
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class EventAuthHandler:
+    """
+    This class contains methods for authenticating events added to room graphs.
+    """
+
+    def __init__(self, hs: "HomeServer"):
+        self._clock = hs.get_clock()
+        self._store = hs.get_datastore()
+        self._server_name = hs.hostname
+
+    async def check_from_context(
+        self, room_version: str, event, context, do_sig_check=True
+    ) -> None:
+        auth_event_ids = event.auth_event_ids()
+        auth_events_by_id = await self._store.get_events(auth_event_ids)
+        auth_events = {(e.type, e.state_key): e for e in auth_events_by_id.values()}
+
+        room_version_obj = KNOWN_ROOM_VERSIONS[room_version]
+        event_auth.check(
+            room_version_obj, event, auth_events=auth_events, do_sig_check=do_sig_check
+        )
+
+    def compute_auth_events(
+        self,
+        event: Union[EventBase, EventBuilder],
+        current_state_ids: StateMap[str],
+        for_verification: bool = False,
+    ) -> List[str]:
+        """Given an event and current state return the list of event IDs used
+        to auth an event.
+
+        If `for_verification` is False then only return auth events that
+        should be added to the event's `auth_events`.
+
+        Returns:
+            List of event IDs.
+        """
+
+        if event.type == EventTypes.Create:
+            return []
+
+        # Currently we ignore the `for_verification` flag even though there are
+        # some situations where we can drop particular auth events when adding
+        # to the event's `auth_events` (e.g. joins pointing to previous joins
+        # when room is publicly joinable). Dropping event IDs has the
+        # advantage that the auth chain for the room grows slower, but we use
+        # the auth chain in state resolution v2 to order events, which means
+        # care must be taken if dropping events to ensure that it doesn't
+        # introduce undesirable "state reset" behaviour.
+        #
+        # All of which sounds a bit tricky so we don't bother for now.
+        auth_ids = []
+        for etype, state_key in event_auth.auth_types_for_event(
+            event.room_version, event
+        ):
+            auth_ev_id = current_state_ids.get((etype, state_key))
+            if auth_ev_id:
+                auth_ids.append(auth_ev_id)
+
+        return auth_ids
+
+    async def get_user_which_could_invite(
+        self, room_id: str, current_state_ids: StateMap[str]
+    ) -> str:
+        """
+        Searches the room state for a local user who has the power level necessary
+        to invite other users.
+
+        Args:
+            room_id: The room ID under search.
+            current_state_ids: The current state of the room.
+
+        Returns:
+            The MXID of the user which could issue an invite.
+
+        Raises:
+            SynapseError if no appropriate user is found.
+        """
+        power_level_event_id = current_state_ids.get((EventTypes.PowerLevels, ""))
+        invite_level = 0
+        users_default_level = 0
+        if power_level_event_id:
+            power_level_event = await self._store.get_event(power_level_event_id)
+            invite_level = power_level_event.content.get("invite", invite_level)
+            users_default_level = power_level_event.content.get(
+                "users_default", users_default_level
+            )
+            users = power_level_event.content.get("users", {})
+        else:
+            users = {}
+
+        # Find the user with the highest power level.
+        users_in_room = await self._store.get_users_in_room(room_id)
+        # Only interested in local users.
+        local_users_in_room = [
+            u for u in users_in_room if get_domain_from_id(u) == self._server_name
+        ]
+        chosen_user = max(
+            local_users_in_room,
+            key=lambda user: users.get(user, users_default_level),
+            default=None,
+        )
+
+        # Return the chosen if they can issue invites.
+        user_power_level = users.get(chosen_user, users_default_level)
+        if chosen_user and user_power_level >= invite_level:
+            logger.debug(
+                "Found a user who can issue invites  %s with power level %d >= invite level %d",
+                chosen_user,
+                user_power_level,
+                invite_level,
+            )
+            return chosen_user
+
+        # No user was found.
+        raise SynapseError(
+            400,
+            "Unable to find a user which could issue an invite",
+            Codes.UNABLE_TO_GRANT_JOIN,
+        )
+
+    async def check_host_in_room(self, room_id: str, host: str) -> bool:
+        with Measure(self._clock, "check_host_in_room"):
+            return await self._store.is_host_joined(room_id, host)
+
+    async def check_restricted_join_rules(
+        self,
+        state_ids: StateMap[str],
+        room_version: RoomVersion,
+        user_id: str,
+        prev_member_event: Optional[EventBase],
+    ) -> None:
+        """
+        Check whether a user can join a room without an invite due to restricted join rules.
+
+        When joining a room with restricted joined rules (as defined in MSC3083),
+        the membership of rooms must be checked during a room join.
+
+        Args:
+            state_ids: The state of the room as it currently is.
+            room_version: The room version of the room being joined.
+            user_id: The user joining the room.
+            prev_member_event: The current membership event for this user.
+
+        Raises:
+            AuthError if the user cannot join the room.
+        """
+        # If the member is invited or currently joined, then nothing to do.
+        if prev_member_event and (
+            prev_member_event.membership in (Membership.JOIN, Membership.INVITE)
+        ):
+            return
+
+        # This is not a room with a restricted join rule, so we don't need to do the
+        # restricted room specific checks.
+        #
+        # Note: We'll be applying the standard join rule checks later, which will
+        # catch the cases of e.g. trying to join private rooms without an invite.
+        if not await self.has_restricted_join_rules(state_ids, room_version):
+            return
+
+        # Get the rooms which allow access to this room and check if the user is
+        # in any of them.
+        allowed_rooms = await self.get_rooms_that_allow_join(state_ids)
+        if not await self.is_user_in_rooms(allowed_rooms, user_id):
+
+            # If this is a remote request, the user might be in an allowed room
+            # that we do not know about.
+            if get_domain_from_id(user_id) != self._server_name:
+                for room_id in allowed_rooms:
+                    if not await self._store.is_host_joined(room_id, self._server_name):
+                        raise SynapseError(
+                            400,
+                            f"Unable to check if {user_id} is in allowed rooms.",
+                            Codes.UNABLE_AUTHORISE_JOIN,
+                        )
+
+            raise AuthError(
+                403,
+                "You do not belong to any of the required rooms/spaces to join this room.",
+            )
+
+    async def has_restricted_join_rules(
+        self, state_ids: StateMap[str], room_version: RoomVersion
+    ) -> bool:
+        """
+        Return if the room has the proper join rules set for access via rooms.
+
+        Args:
+            state_ids: The state of the room as it currently is.
+            room_version: The room version of the room to query.
+
+        Returns:
+            True if the proper room version and join rules are set for restricted access.
+        """
+        # This only applies to room versions which support the new join rule.
+        if not room_version.msc3083_join_rules:
+            return False
+
+        # If there's no join rule, then it defaults to invite (so this doesn't apply).
+        join_rules_event_id = state_ids.get((EventTypes.JoinRules, ""), None)
+        if not join_rules_event_id:
+            return False
+
+        # If the join rule is not restricted, this doesn't apply.
+        join_rules_event = await self._store.get_event(join_rules_event_id)
+        return join_rules_event.content.get("join_rule") == JoinRules.RESTRICTED
+
+    async def get_rooms_that_allow_join(
+        self, state_ids: StateMap[str]
+    ) -> Collection[str]:
+        """
+        Generate a list of rooms in which membership allows access to a room.
+
+        Args:
+            state_ids: The current state of the room the user wishes to join
+
+        Returns:
+            A collection of room IDs. Membership in any of the rooms in the list grants the ability to join the target room.
+        """
+        # If there's no join rule, then it defaults to invite (so this doesn't apply).
+        join_rules_event_id = state_ids.get((EventTypes.JoinRules, ""), None)
+        if not join_rules_event_id:
+            return ()
+
+        # If the join rule is not restricted, this doesn't apply.
+        join_rules_event = await self._store.get_event(join_rules_event_id)
+
+        # If allowed is of the wrong form, then only allow invited users.
+        allow_list = join_rules_event.content.get("allow", [])
+        if not isinstance(allow_list, list):
+            return ()
+
+        # Pull out the other room IDs, invalid data gets filtered.
+        result = []
+        for allow in allow_list:
+            if not isinstance(allow, dict):
+                continue
+
+            # If the type is unexpected, skip it.
+            if allow.get("type") != RestrictedJoinRuleTypes.ROOM_MEMBERSHIP:
+                continue
+
+            room_id = allow.get("room_id")
+            if not isinstance(room_id, str):
+                continue
+
+            result.append(room_id)
+
+        return result
+
+    async def is_user_in_rooms(self, room_ids: Collection[str], user_id: str) -> bool:
+        """
+        Check whether a user is a member of any of the provided rooms.
+
+        Args:
+            room_ids: The rooms to check for membership.
+            user_id: The user to check.
+
+        Returns:
+            True if the user is in any of the rooms, false otherwise.
+        """
+        if not room_ids:
+            return False
+
+        # Get the list of joined rooms and see if there's an overlap.
+        joined_rooms = await self._store.get_rooms_for_user(user_id)
+
+        # Check each room and see if the user is in it.
+        for room_id in room_ids:
+            if room_id in joined_rooms:
+                return True
+
+        # The user was not in any of the rooms.
+        return False
diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index f46cab7325..4b3f037072 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -94,7 +93,7 @@ async def get_stream(
 
             # When the user joins a new room, or another user joins a currently
             # joined room, we need to send down presence for those users.
-            to_add = []  # type: List[JsonDict]
+            to_add: List[JsonDict] = []
             for event in events:
                 if not isinstance(event, EventBase):
                     continue
@@ -104,9 +103,9 @@ async def get_stream(
                     # Send down presence.
                     if event.state_key == auth_user_id:
                         # Send down presence for everyone in the room.
-                        users = await self.state.get_current_users_in_room(
+                        users: Iterable[str] = await self.store.get_users_in_room(
                             event.room_id
-                        )  # type: Iterable[str]
+                        )
                     else:
                         users = [event.state_key]
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index ec2ce679c2..5f5ee1d300 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1,7 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2017-2018 New Vector Ltd
-# Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
 # Copyright 2020 Sorunome
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,9 +19,21 @@
 import logging
 from collections.abc import Container
 from http import HTTPStatus
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Collection,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Set,
+    Tuple,
+    Union,
+)
 
 import attr
+from prometheus_client import Counter
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import verify_signed_json
 from unpaddedbase64 import decode_base64
@@ -33,6 +42,7 @@
 
 from synapse import event_auth
 from synapse.api.constants import (
+    EventContentFields,
     EventTypes,
     Membership,
     RejectedReason,
@@ -83,6 +93,7 @@
     get_domain_from_id,
 )
 from synapse.util.async_helpers import Linearizer, concurrently_execute
+from synapse.util.iterutils import batch_iter
 from synapse.util.retryutils import NotRetryingDestination
 from synapse.util.stringutils import shortstr
 from synapse.visibility import filter_events_for_server
@@ -92,22 +103,39 @@
 
 logger = logging.getLogger(__name__)
 
+soft_failed_event_counter = Counter(
+    "synapse_federation_soft_failed_events_total",
+    "Events received over federation that we marked as soft_failed",
+)
+
 
-@attr.s(slots=True)
+@attr.s(slots=True, frozen=True, auto_attribs=True)
 class _NewEventInfo:
-    """Holds information about a received event, ready for passing to _handle_new_events
+    """Holds information about a received event, ready for passing to _auth_and_persist_events
 
     Attributes:
         event: the received event
 
-        state: the state at that event
+        state: the state at that event, according to /state_ids from a remote
+           homeserver. Only populated for backfilled events which are going to be a
+           new backwards extremity.
+
+        claimed_auth_event_map: a map of (type, state_key) => event for the event's
+            claimed auth_events.
+
+            This can include events which have not yet been persisted, in the case that
+            we are backfilling a batch of events.
+
+            Note: May be incomplete: if we were unable to find all of the claimed auth
+            events. Also, treat the contents with caution: the events might also have
+            been rejected, might not yet have been authorized themselves, or they might
+            be in the wrong room.
 
-        auth_events: the auth_event map for that event
     """
 
-    event = attr.ib(type=EventBase)
-    state = attr.ib(type=Optional[Sequence[EventBase]], default=None)
-    auth_events = attr.ib(type=Optional[MutableStateMap[EventBase]], default=None)
+    event: EventBase
+    state: Optional[Sequence[EventBase]]
+    claimed_auth_event_map: StateMap[EventBase]
 
 
 class FederationHandler(BaseHandler):
@@ -138,6 +166,7 @@ def __init__(self, hs: "HomeServer"):
         self.is_mine_id = hs.is_mine_id
         self.spam_checker = hs.get_spam_checker()
         self.event_creation_handler = hs.get_event_creation_handler()
+        self._event_auth_handler = hs.get_event_auth_handler()
         self._message_handler = hs.get_message_handler()
         self._server_notices_mxid = hs.config.server_notices_mxid
         self.config = hs.config
@@ -165,22 +194,26 @@ def __init__(self, hs: "HomeServer"):
 
         # When joining a room we need to queue any events for that room up.
         # For each room, a list of (pdu, origin) tuples.
-        self.room_queues = {}  # type: Dict[str, List[Tuple[EventBase, str]]]
+        self.room_queues: Dict[str, List[Tuple[EventBase, str]]] = {}
         self._room_pdu_linearizer = Linearizer("fed_room_pdu")
 
+        self._room_backfill = Linearizer("room_backfill")
+
         self.third_party_event_rules = hs.get_third_party_event_rules()
 
         self._ephemeral_messages_enabled = hs.config.enable_ephemeral_messages
 
-    async def on_receive_pdu(self, origin, pdu, sent_to_us_directly=False) -> None:
+    async def on_receive_pdu(
+        self, origin: str, pdu: EventBase, sent_to_us_directly: bool = False
+    ) -> None:
         """Process a PDU received via a federation /send/ transaction, or
         via backfill of missing prev_events
 
         Args:
-            origin (str): server which initiated the /send/ transaction. Will
+            origin: server which initiated the /send/ transaction. Will
                 be used to fetch missing events or state.
-            pdu (FrozenEvent): received PDU
-            sent_to_us_directly (bool): True if this event was pushed to us; False if
+            pdu: received PDU
+            sent_to_us_directly: True if this event was pushed to us; False if
                 we pulled it as the result of a missing prev_event.
         """
 
@@ -196,14 +229,19 @@ async def on_receive_pdu(self, origin, pdu, sent_to_us_directly=False) -> None:
 
         # FIXME: Currently we fetch an event again when we already have it
         # if it has been marked as an outlier.
-
-        already_seen = existing and (
-            not existing.internal_metadata.is_outlier()
-            or pdu.internal_metadata.is_outlier()
-        )
-        if already_seen:
-            logger.debug("Already seen pdu")
-            return
+        if existing:
+            if not existing.internal_metadata.is_outlier():
+                logger.info(
+                    "Ignoring received event %s which we have already seen", event_id
+                )
+                return
+            if pdu.internal_metadata.is_outlier():
+                logger.info(
+                    "Ignoring received outlier %s which we already have as an outlier",
+                    event_id,
+                )
+                return
+            logger.info("De-outliering event %s", event_id)
 
         # do some initial sanity-checking of the event. In particular, make
         # sure it doesn't have hundreds of prev_events or auth_events, which
@@ -230,7 +268,9 @@ async def on_receive_pdu(self, origin, pdu, sent_to_us_directly=False) -> None:
         #
         # Note that if we were never in the room then we would have already
         # dropped the event, since we wouldn't know the room version.
-        is_in_room = await self.auth.check_host_in_room(room_id, self.server_name)
+        is_in_room = await self._event_auth_handler.check_host_in_room(
+            room_id, self.server_name
+        )
         if not is_in_room:
             logger.info(
                 "Ignoring PDU from %s as we're not in the room",
@@ -240,7 +280,12 @@ async def on_receive_pdu(self, origin, pdu, sent_to_us_directly=False) -> None:
 
         state = None
 
-        # Get missing pdus if necessary.
+        # Check that the event passes auth based on the state at the event. This is
+        # done for events that are to be added to the timeline (non-outliers).
+        #
+        # Get missing pdus if necessary:
+        #  - Fetching any missing prev events to fill in gaps in the graph
+        #  - Fetching state if we have a hole in the graph
         if not pdu.internal_metadata.is_outlier():
             # We only backfill backwards to the min depth.
             min_depth = await self.get_min_depth_for_context(pdu.room_id)
@@ -299,7 +344,8 @@ async def on_receive_pdu(self, origin, pdu, sent_to_us_directly=False) -> None:
                         shortstr(missing_prevs),
                     )
 
-            if prevs - seen:
+            missing_prevs = prevs - seen
+            if missing_prevs:
                 # We've still not been able to get all of the prev_events for this event.
                 #
                 # In this case, we need to fall back to asking another server in the
@@ -327,8 +373,8 @@ async def on_receive_pdu(self, origin, pdu, sent_to_us_directly=False) -> None:
                 if sent_to_us_directly:
                     logger.warning(
                         "Rejecting: failed to fetch %d prev events: %s",
-                        len(prevs - seen),
-                        shortstr(prevs - seen),
+                        len(missing_prevs),
+                        shortstr(missing_prevs),
                     )
                     raise FederationError(
                         "ERROR",
@@ -340,6 +386,13 @@ async def on_receive_pdu(self, origin, pdu, sent_to_us_directly=False) -> None:
                         affected=pdu.event_id,
                     )
 
+                logger.info(
+                    "Event %s is missing prev_events %s: calculating state for a "
+                    "backwards extremity",
+                    event_id,
+                    shortstr(missing_prevs),
+                )
+
                 # Calculate the state after each of the previous events, and
                 # resolve them to find the correct state at the current event.
                 event_map = {event_id: pdu}
@@ -348,14 +401,14 @@ async def on_receive_pdu(self, origin, pdu, sent_to_us_directly=False) -> None:
                     ours = await self.state_store.get_state_groups_ids(room_id, seen)
 
                     # state_maps is a list of mappings from (type, state_key) to event_id
-                    state_maps = list(ours.values())  # type: List[StateMap[str]]
+                    state_maps: List[StateMap[str]] = list(ours.values())
 
                     # we don't need this any more, let's delete it.
                     del ours
 
                     # Ask the remote server for the states we don't
                     # know about
-                    for p in prevs - seen:
+                    for p in missing_prevs:
                         logger.info("Requesting state after missing prev_event %s", p)
 
                         with nested_logging_context(p):
@@ -411,15 +464,24 @@ async def on_receive_pdu(self, origin, pdu, sent_to_us_directly=False) -> None:
                         affected=event_id,
                     )
 
+        # A second round of checks for all events. Check that the event passes auth
+        # based on `auth_events`, this allows us to assert that the event would
+        # have been allowed at some point. If an event passes this check its OK
+        # for it to be used as part of a returned `/state` request, as either
+        # a) we received the event as part of the original join and so trust it, or
+        # b) we'll do a state resolution with existing state before it becomes
+        # part of the "current state", which adds more protection.
         await self._process_received_pdu(origin, pdu, state=state)
 
-    async def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
+    async def _get_missing_events_for_pdu(
+        self, origin: str, pdu: EventBase, prevs: Set[str], min_depth: int
+    ) -> None:
         """
         Args:
-            origin (str): Origin of the pdu. Will be called to get the missing events
+            origin: Origin of the pdu. Will be called to get the missing events
             pdu: received pdu
-            prevs (set(str)): List of event ids which we are missing
-            min_depth (int): Minimum depth of events to return.
+            prevs: List of event ids which we are missing
+            min_depth: Minimum depth of events to return.
         """
 
         room_id = pdu.room_id
@@ -508,21 +570,14 @@ async def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
             logger.warning("Failed to get prev_events: %s", e)
             return
 
-        logger.info(
-            "Got %d prev_events: %s",
-            len(missing_events),
-            shortstr(missing_events),
-        )
+        logger.info("Got %d prev_events", len(missing_events))
 
         # We want to sort these by depth so we process them and
         # tell clients about them in order.
         missing_events.sort(key=lambda x: x.depth)
 
         for ev in missing_events:
-            logger.info(
-                "Handling received prev_event %s",
-                ev.event_id,
-            )
+            logger.info("Handling received prev_event %s", ev)
             with nested_logging_context(ev.event_id):
                 try:
                     await self.on_receive_pdu(origin, ev, sent_to_us_directly=False)
@@ -540,8 +595,12 @@ async def _get_state_for_room(
         destination: str,
         room_id: str,
         event_id: str,
-    ) -> Tuple[List[EventBase], List[EventBase]]:
-        """Requests all of the room state at a given event from a remote homeserver.
+    ) -> List[EventBase]:
+        """Requests all of the room state at a given event from a remote
+        homeserver.
+
+        Will also fetch any missing events reported in the `auth_chain_ids`
+        section of `/state_ids`.
 
         Args:
             destination: The remote homeserver to query for the state.
@@ -549,8 +608,7 @@ async def _get_state_for_room(
             event_id: The id of the event we want the state at.
 
         Returns:
-            A list of events in the state, not including the event itself, and
-            a list of events in the auth chain for the given event.
+            A list of events in the state, not including the event itself.
         """
         (
             state_event_ids,
@@ -559,68 +617,55 @@ async def _get_state_for_room(
             destination, room_id, event_id=event_id
         )
 
-        desired_events = set(state_event_ids + auth_event_ids)
-
-        event_map = await self._get_events_from_store_or_dest(
-            destination, room_id, desired_events
+        # Fetch the state events from the DB, and check we have the auth events.
+        event_map = await self.store.get_events(state_event_ids, allow_rejected=True)
+        auth_events_in_store = await self.store.have_seen_events(
+            room_id, auth_event_ids
         )
 
-        failed_to_fetch = desired_events - event_map.keys()
-        if failed_to_fetch:
-            logger.warning(
-                "Failed to fetch missing state/auth events for %s %s",
-                event_id,
-                failed_to_fetch,
+        # Check for missing events. We handle state and auth event seperately,
+        # as we want to pull the state from the DB, but we don't for the auth
+        # events. (Note: we likely won't use the majority of the auth chain, and
+        # it can be *huge* for large rooms, so it's worth ensuring that we don't
+        # unnecessarily pull it from the DB).
+        missing_state_events = set(state_event_ids) - set(event_map)
+        missing_auth_events = set(auth_event_ids) - set(auth_events_in_store)
+        if missing_state_events or missing_auth_events:
+            await self._get_events_and_persist(
+                destination=destination,
+                room_id=room_id,
+                events=missing_state_events | missing_auth_events,
             )
 
-        remote_state = [
-            event_map[e_id] for e_id in state_event_ids if e_id in event_map
-        ]
-
-        auth_chain = [event_map[e_id] for e_id in auth_event_ids if e_id in event_map]
-        auth_chain.sort(key=lambda e: e.depth)
-
-        return remote_state, auth_chain
-
-    async def _get_events_from_store_or_dest(
-        self, destination: str, room_id: str, event_ids: Iterable[str]
-    ) -> Dict[str, EventBase]:
-        """Fetch events from a remote destination, checking if we already have them.
-
-        Persists any events we don't already have as outliers.
-
-        If we fail to fetch any of the events, a warning will be logged, and the event
-        will be omitted from the result. Likewise, any events which turn out not to
-        be in the given room.
-
-        This function *does not* automatically get missing auth events of the
-        newly fetched events. Callers must include the full auth chain of
-        of the missing events in the `event_ids` argument, to ensure that any
-        missing auth events are correctly fetched.
+            if missing_state_events:
+                new_events = await self.store.get_events(
+                    missing_state_events, allow_rejected=True
+                )
+                event_map.update(new_events)
 
-        Returns:
-            map from event_id to event
-        """
-        fetched_events = await self.store.get_events(event_ids, allow_rejected=True)
+                missing_state_events.difference_update(new_events)
 
-        missing_events = set(event_ids) - fetched_events.keys()
+                if missing_state_events:
+                    logger.warning(
+                        "Failed to fetch missing state events for %s %s",
+                        event_id,
+                        missing_state_events,
+                    )
 
-        if missing_events:
-            logger.debug(
-                "Fetching unknown state/auth events %s for room %s",
-                missing_events,
-                room_id,
-            )
+            if missing_auth_events:
+                auth_events_in_store = await self.store.have_seen_events(
+                    room_id, missing_auth_events
+                )
+                missing_auth_events.difference_update(auth_events_in_store)
 
-            await self._get_events_and_persist(
-                destination=destination, room_id=room_id, events=missing_events
-            )
+                if missing_auth_events:
+                    logger.warning(
+                        "Failed to fetch missing auth events for %s %s",
+                        event_id,
+                        missing_auth_events,
+                    )
 
-            # we need to make sure we re-load from the database to get the rejected
-            # state correct.
-            fetched_events.update(
-                (await self.store.get_events(missing_events, allow_rejected=True))
-            )
+        remote_state = list(event_map.values())
 
         # check for events which were in the wrong room.
         #
@@ -628,8 +673,8 @@ async def _get_events_from_store_or_dest(
         # auth_events at an event in room A are actually events in room B
 
         bad_events = [
-            (event_id, event.room_id)
-            for event_id, event in fetched_events.items()
+            (event.event_id, event.room_id)
+            for event in remote_state
             if event.room_id != room_id
         ]
 
@@ -646,9 +691,10 @@ async def _get_events_from_store_or_dest(
                 room_id,
             )
 
-            del fetched_events[bad_event_id]
+        if bad_events:
+            remote_state = [e for e in remote_state if e.room_id == room_id]
 
-        return fetched_events
+        return remote_state
 
     async def _get_state_after_missing_prev_event(
         self,
@@ -708,7 +754,7 @@ async def _get_state_after_missing_prev_event(
 
         missing_auth_events = set(auth_event_ids) - fetched_events.keys()
         missing_auth_events.difference_update(
-            await self.store.have_seen_events(missing_auth_events)
+            await self.store.have_seen_events(room_id, missing_auth_events)
         )
         logger.debug("We are also missing %i auth events", len(missing_auth_events))
 
@@ -721,7 +767,7 @@ async def _get_state_after_missing_prev_event(
         # we need to make sure we re-load from the database to get the rejected
         # state correct.
         fetched_events.update(
-            (await self.store.get_events(missing_desired_events, allow_rejected=True))
+            await self.store.get_events(missing_desired_events, allow_rejected=True)
         )
 
         # check for events which were in the wrong room.
@@ -780,7 +826,7 @@ async def _process_received_pdu(
         origin: str,
         event: EventBase,
         state: Optional[Iterable[EventBase]],
-    ):
+    ) -> None:
         """Called when we have a new pdu. We need to do auth checks and put it
         through the StateHandler.
 
@@ -796,7 +842,10 @@ async def _process_received_pdu(
         logger.debug("Processing event: %s", event)
 
         try:
-            await self._handle_new_event(origin, event, state=state)
+            context = await self.state_handler.compute_event_context(
+                event, old_state=state
+            )
+            await self._auth_and_persist_event(origin, event, context, state=state)
         except AuthError as e:
             raise FederationError("ERROR", e.code, e.msg, affected=event.event_id)
 
@@ -828,7 +877,7 @@ async def _process_received_pdu(
                 # exact key to expect. Otherwise check it matches any key we
                 # have for that device.
 
-                current_keys = []  # type: Container[str]
+                current_keys: Container[str] = []
 
                 if device:
                     keys = device.get("keys", {}).get("keys", {})
@@ -872,6 +921,79 @@ async def _process_received_pdu(
                     "resync_device_due_to_pdu", self._resync_device, event.sender
                 )
 
+        await self._handle_marker_event(origin, event)
+
+    async def _handle_marker_event(self, origin: str, marker_event: EventBase):
+        """Handles backfilling the insertion event when we receive a marker
+        event that points to one.
+
+        Args:
+            origin: Origin of the event. Will be called to get the insertion event
+            marker_event: The event to process
+        """
+
+        if marker_event.type != EventTypes.MSC2716_MARKER:
+            # Not a marker event
+            return
+
+        if marker_event.rejected_reason is not None:
+            # Rejected event
+            return
+
+        # Skip processing a marker event if the room version doesn't
+        # support it.
+        room_version = await self.store.get_room_version(marker_event.room_id)
+        if not room_version.msc2716_historical:
+            return
+
+        logger.debug("_handle_marker_event: received %s", marker_event)
+
+        insertion_event_id = marker_event.content.get(
+            EventContentFields.MSC2716_MARKER_INSERTION
+        )
+
+        if insertion_event_id is None:
+            # Nothing to retrieve then (invalid marker)
+            return
+
+        logger.debug(
+            "_handle_marker_event: backfilling insertion event %s", insertion_event_id
+        )
+
+        await self._get_events_and_persist(
+            origin,
+            marker_event.room_id,
+            [insertion_event_id],
+        )
+
+        insertion_event = await self.store.get_event(
+            insertion_event_id, allow_none=True
+        )
+        if insertion_event is None:
+            logger.warning(
+                "_handle_marker_event: server %s didn't return insertion event %s for marker %s",
+                origin,
+                insertion_event_id,
+                marker_event.event_id,
+            )
+            return
+
+        logger.debug(
+            "_handle_marker_event: succesfully backfilled insertion event %s from marker event %s",
+            insertion_event,
+            marker_event,
+        )
+
+        await self.store.insert_insertion_extremity(
+            insertion_event_id, marker_event.room_id
+        )
+
+        logger.debug(
+            "_handle_marker_event: insertion extremity added for %s from marker event %s",
+            insertion_event,
+            marker_event,
+        )
+
     async def _resync_device(self, sender: str) -> None:
         """We have detected that the device list for the given user may be out
         of sync, so we try and resync them.
@@ -889,7 +1011,9 @@ async def _resync_device(self, sender: str) -> None:
             logger.exception("Failed to resync device for %s", sender)
 
     @log_function
-    async def backfill(self, dest, room_id, limit, extremities):
+    async def backfill(
+        self, dest: str, room_id: str, limit: int, extremities: List[str]
+    ) -> List[EventBase]:
         """Trigger a backfill request to `dest` for the given `room_id`
 
         This will attempt to get more events from the remote. If the other side
@@ -946,27 +1070,23 @@ async def backfill(self, dest, room_id, limit, extremities):
 
         # For each edge get the current state.
 
-        auth_events = {}
         state_events = {}
         events_to_state = {}
         for e_id in edges:
-            state, auth = await self._get_state_for_room(
+            state = await self._get_state_for_room(
                 destination=dest,
                 room_id=room_id,
                 event_id=e_id,
             )
-            auth_events.update({a.event_id: a for a in auth})
-            auth_events.update({s.event_id: s for s in state})
             state_events.update({s.event_id: s for s in state})
             events_to_state[e_id] = state
 
         required_auth = {
             a_id
-            for event in events
-            + list(state_events.values())
-            + list(auth_events.values())
+            for event in events + list(state_events.values())
             for a_id in event.auth_event_ids()
         }
+        auth_events = await self.store.get_events(required_auth, allow_rejected=True)
         auth_events.update(
             {e_id: event_map[e_id] for e_id in required_auth if e_id in event_map}
         )
@@ -985,7 +1105,7 @@ async def backfill(self, dest, room_id, limit, extremities):
                 _NewEventInfo(
                     event=ev,
                     state=events_to_state[e_id],
-                    auth_events={
+                    claimed_auth_event_map={
                         (
                             auth_events[a_id].type,
                             auth_events[a_id].state_key,
@@ -997,7 +1117,9 @@ async def backfill(self, dest, room_id, limit, extremities):
             )
 
         if ev_infos:
-            await self._handle_new_events(dest, room_id, ev_infos, backfilled=True)
+            await self._auth_and_persist_events(
+                dest, room_id, ev_infos, backfilled=True
+            )
 
         # Step 2: Persist the rest of the events in the chunk one by one
         events.sort(key=lambda e: e.depth)
@@ -1010,10 +1132,12 @@ async def backfill(self, dest, room_id, limit, extremities):
             # non-outliers
             assert not event.internal_metadata.is_outlier()
 
+            context = await self.state_handler.compute_event_context(event)
+
             # We store these one at a time since each event depends on the
             # previous to work out the state.
             # TODO: We can probably do something more clever here.
-            await self._handle_new_event(dest, event, backfilled=True)
+            await self._auth_and_persist_event(dest, event, context, backfilled=True)
 
         return events
 
@@ -1032,9 +1156,25 @@ async def maybe_backfill(
                 return. This is used as part of the heuristic to decide if we
                 should back paginate.
         """
-        extremities = await self.store.get_oldest_events_with_depth_in_room(room_id)
+        with (await self._room_backfill.queue(room_id)):
+            return await self._maybe_backfill_inner(room_id, current_depth, limit)
+
+    async def _maybe_backfill_inner(
+        self, room_id: str, current_depth: int, limit: int
+    ) -> bool:
+        oldest_events_with_depth = (
+            await self.store.get_oldest_event_ids_with_depth_in_room(room_id)
+        )
+        insertion_events_to_be_backfilled = (
+            await self.store.get_insertion_event_backwards_extremities_in_room(room_id)
+        )
+        logger.debug(
+            "_maybe_backfill_inner: extremities oldest_events_with_depth=%s insertion_events_to_be_backfilled=%s",
+            oldest_events_with_depth,
+            insertion_events_to_be_backfilled,
+        )
 
-        if not extremities:
+        if not oldest_events_with_depth and not insertion_events_to_be_backfilled:
             logger.debug("Not backfilling as no extremeties found.")
             return False
 
@@ -1064,10 +1204,12 @@ async def maybe_backfill(
         #   state *before* the event, ignoring the special casing certain event
         #   types have.
 
-        forward_events = await self.store.get_successor_events(list(extremities))
+        forward_event_ids = await self.store.get_successor_events(
+            list(oldest_events_with_depth)
+        )
 
         extremities_events = await self.store.get_events(
-            forward_events,
+            forward_event_ids,
             redact_behaviour=EventRedactBehaviour.AS_IS,
             get_prev_content=False,
         )
@@ -1081,10 +1223,19 @@ async def maybe_backfill(
             redact=False,
             check_history_visibility_only=True,
         )
+        logger.debug(
+            "_maybe_backfill_inner: filtered_extremities %s", filtered_extremities
+        )
 
-        if not filtered_extremities:
+        if not filtered_extremities and not insertion_events_to_be_backfilled:
             return False
 
+        extremities = {
+            **oldest_events_with_depth,
+            # TODO: insertion_events_to_be_backfilled is currently skipping the filtered_extremities checks
+            **insertion_events_to_be_backfilled,
+        }
+
         # Check if we reached a point where we should start backfilling.
         sorted_extremeties_tuple = sorted(extremities.items(), key=lambda e: -int(e[1]))
         max_depth = sorted_extremeties_tuple[0][1]
@@ -1144,16 +1295,15 @@ async def maybe_backfill(
 
         curr_state = await self.state_handler.get_current_state(room_id)
 
-        def get_domains_from_state(state):
+        def get_domains_from_state(state: StateMap[EventBase]) -> List[Tuple[str, int]]:
             """Get joined domains from state
 
             Args:
-                state (dict[tuple, FrozenEvent]): State map from type/state
-                    key to event.
+                state: State map from type/state key to event.
 
             Returns:
-                list[tuple[str, int]]: Returns a list of servers with the
-                lowest depth of their joins. Sorted by lowest depth first.
+                Returns a list of servers with the lowest depth of their joins.
+                 Sorted by lowest depth first.
             """
             joined_users = [
                 (state_key, int(event.depth))
@@ -1161,7 +1311,7 @@ def get_domains_from_state(state):
                 if e_type == EventTypes.Member and event.membership == Membership.JOIN
             ]
 
-            joined_domains = {}  # type: Dict[str, int]
+            joined_domains: Dict[str, int] = {}
             for u, d in joined_users:
                 try:
                     dom = get_domain_from_id(u)
@@ -1181,7 +1331,7 @@ def get_domains_from_state(state):
             domain for domain, depth in curr_domains if domain != self.server_name
         ]
 
-        async def try_backfill(domains):
+        async def try_backfill(domains: List[str]) -> bool:
             # TODO: Should we try multiple of these at a time?
             for dom in domains:
                 try:
@@ -1260,21 +1410,25 @@ async def try_backfill(domains):
         }
 
         for e_id, _ in sorted_extremeties_tuple:
-            likely_domains = get_domains_from_state(states[e_id])
+            likely_extremeties_domains = get_domains_from_state(states[e_id])
 
             success = await try_backfill(
-                [dom for dom, _ in likely_domains if dom not in tried_domains]
+                [
+                    dom
+                    for dom, _ in likely_extremeties_domains
+                    if dom not in tried_domains
+                ]
             )
             if success:
                 return True
 
-            tried_domains.update(dom for dom, _ in likely_domains)
+            tried_domains.update(dom for dom, _ in likely_extremeties_domains)
 
         return False
 
     async def _get_events_and_persist(
         self, destination: str, room_id: str, events: Iterable[str]
-    ):
+    ) -> None:
         """Fetch the given events from a server, and persist them as outliers.
 
         This function *does not* recursively get missing auth events of the
@@ -1286,7 +1440,7 @@ async def _get_events_and_persist(
 
         room_version = await self.store.get_room_version(room_id)
 
-        event_map = {}  # type: Dict[str, EventBase]
+        event_map: Dict[str, EventBase] = {}
 
         async def get_event(event_id: str):
             with nested_logging_context(event_id):
@@ -1344,13 +1498,14 @@ async def get_event(event_id: str):
 
             event_infos.append(_NewEventInfo(event, None, auth))
 
-        await self._handle_new_events(
-            destination,
-            room_id,
-            event_infos,
-        )
+        if event_infos:
+            await self._auth_and_persist_events(
+                destination,
+                room_id,
+                event_infos,
+            )
 
-    def _sanity_check_event(self, ev):
+    def _sanity_check_event(self, ev: EventBase) -> None:
         """
         Do some early sanity checks of a received event
 
@@ -1359,9 +1514,7 @@ def _sanity_check_event(self, ev):
         or cascade of event fetches.
 
         Args:
-            ev (synapse.events.EventBase): event to be checked
-
-        Returns: None
+            ev: event to be checked
 
         Raises:
             SynapseError if the event does not pass muster
@@ -1382,17 +1535,20 @@ def _sanity_check_event(self, ev):
             )
             raise SynapseError(HTTPStatus.BAD_REQUEST, "Too many auth_events")
 
-    async def send_invite(self, target_host, event):
+    async def send_invite(self, target_host: str, event: EventBase) -> EventBase:
         """Sends the invite to the remote server for signing.
 
         Invites must be signed by the invitee's server before distribution.
         """
-        pdu = await self.federation_client.send_invite(
-            destination=target_host,
-            room_id=event.room_id,
-            event_id=event.event_id,
-            pdu=event,
-        )
+        try:
+            pdu = await self.federation_client.send_invite(
+                destination=target_host,
+                room_id=event.room_id,
+                event_id=event.event_id,
+                pdu=event,
+            )
+        except RequestSendFailed:
+            raise SynapseError(502, f"Can't connect to server {target_host}")
 
         return pdu
 
@@ -1464,9 +1620,10 @@ async def do_invite_join(
                 host_list, event, room_version_obj
             )
 
-            origin = ret["origin"]
-            state = ret["state"]
-            auth_chain = ret["auth_chain"]
+            event = ret.event
+            origin = ret.origin
+            state = ret.state
+            auth_chain = ret.auth_chain
             auth_chain.sort(key=lambda e: e.depth)
 
             logger.debug("do_invite_join auth_chain: %s", auth_chain)
@@ -1566,7 +1723,7 @@ async def do_knock(
 
         # Ask the remote server to create a valid knock event for us. Once received,
         # we sign the event
-        params = {"ver": supported_room_versions}  # type: Dict[str, Iterable[str]]
+        params: Dict[str, Iterable[str]] = {"ver": supported_room_versions}
         origin, event, event_format_version = await self._make_and_verify_event(
             target_hosts, room_id, knockee, Membership.KNOCK, content, params=params
         )
@@ -1601,20 +1758,19 @@ async def do_knock(
         )
         return event.event_id, stream_id
 
-    async def _handle_queued_pdus(self, room_queue):
+    async def _handle_queued_pdus(
+        self, room_queue: List[Tuple[EventBase, str]]
+    ) -> None:
         """Process PDUs which got queued up while we were busy send_joining.
 
         Args:
-            room_queue (list[FrozenEvent, str]): list of PDUs to be processed
-                and the servers that sent them
+            room_queue: list of PDUs to be processed and the servers that sent them
         """
         for p, origin in room_queue:
             try:
                 logger.info(
-                    "Processing queued PDU %s which was received "
-                    "while we were joining %s",
-                    p.event_id,
-                    p.room_id,
+                    "Processing queued PDU %s which was received while we were joining",
+                    p,
                 )
                 with nested_logging_context(p.event_id):
                     await self.on_receive_pdu(origin, p, sent_to_us_directly=True)
@@ -1645,10 +1801,12 @@ async def on_make_join_request(
 
         # checking the room version will check that we've actually heard of the room
         # (and return a 404 otherwise)
-        room_version = await self.store.get_room_version_id(room_id)
+        room_version = await self.store.get_room_version(room_id)
 
         # now check that we are *still* in the room
-        is_in_room = await self.auth.check_host_in_room(room_id, self.server_name)
+        is_in_room = await self._event_auth_handler.check_host_in_room(
+            room_id, self.server_name
+        )
         if not is_in_room:
             logger.info(
                 "Got /make_join request for room %s we are no longer in",
@@ -1658,8 +1816,38 @@ async def on_make_join_request(
 
         event_content = {"membership": Membership.JOIN}
 
+        # If the current room is using restricted join rules, additional information
+        # may need to be included in the event content in order to efficiently
+        # validate the event.
+        #
+        # Note that this requires the /send_join request to come back to the
+        # same server.
+        if room_version.msc3083_join_rules:
+            state_ids = await self.store.get_current_state_ids(room_id)
+            if await self._event_auth_handler.has_restricted_join_rules(
+                state_ids, room_version
+            ):
+                prev_member_event_id = state_ids.get((EventTypes.Member, user_id), None)
+                # If the user is invited or joined to the room already, then
+                # no additional info is needed.
+                include_auth_user_id = True
+                if prev_member_event_id:
+                    prev_member_event = await self.store.get_event(prev_member_event_id)
+                    include_auth_user_id = prev_member_event.membership not in (
+                        Membership.JOIN,
+                        Membership.INVITE,
+                    )
+
+                if include_auth_user_id:
+                    event_content[
+                        "join_authorised_via_users_server"
+                    ] = await self._event_auth_handler.get_user_which_could_invite(
+                        room_id,
+                        state_ids,
+                    )
+
         builder = self.event_builder_factory.new(
-            room_version,
+            room_version.identifier,
             {
                 "type": EventTypes.Member,
                 "content": event_content,
@@ -1677,71 +1865,20 @@ async def on_make_join_request(
             logger.warning("Failed to create join to %s because %s", room_id, e)
             raise
 
+        # Ensure the user can even join the room.
+        await self._check_join_restrictions(context, event)
+
         # The remote hasn't signed it yet, obviously. We'll do the full checks
         # when we get the event back in `on_send_join_request`
-        await self.auth.check_from_context(
-            room_version, event, context, do_sig_check=False
+        await self._event_auth_handler.check_from_context(
+            room_version.identifier, event, context, do_sig_check=False
         )
 
         return event
 
-    async def on_send_join_request(self, origin, pdu):
-        """We have received a join event for a room. Fully process it and
-        respond with the current state and auth chains.
-        """
-        event = pdu
-
-        logger.debug(
-            "on_send_join_request from %s: Got event: %s, signatures: %s",
-            origin,
-            event.event_id,
-            event.signatures,
-        )
-
-        if get_domain_from_id(event.sender) != origin:
-            logger.info(
-                "Got /send_join request for user %r from different origin %s",
-                event.sender,
-                origin,
-            )
-            raise SynapseError(403, "User not from origin", Codes.FORBIDDEN)
-
-        event.internal_metadata.outlier = False
-        # Send this event on behalf of the origin server.
-        #
-        # The reasons we have the destination server rather than the origin
-        # server send it are slightly mysterious: the origin server should have
-        # all the necessary state once it gets the response to the send_join,
-        # so it could send the event itself if it wanted to. It may be that
-        # doing it this way reduces failure modes, or avoids certain attacks
-        # where a new server selectively tells a subset of the federation that
-        # it has joined.
-        #
-        # The fact is that, as of the current writing, Synapse doesn't send out
-        # the join event over federation after joining, and changing it now
-        # would introduce the danger of backwards-compatibility problems.
-        event.internal_metadata.send_on_behalf_of = origin
-
-        context = await self._handle_new_event(origin, event)
-
-        logger.debug(
-            "on_send_join_request: After _handle_new_event: %s, sigs: %s",
-            event.event_id,
-            event.signatures,
-        )
-
-        prev_state_ids = await context.get_prev_state_ids()
-
-        state_ids = list(prev_state_ids.values())
-        auth_chain = await self.store.get_auth_chain(event.room_id, state_ids)
-
-        state = await self.store.get_events(list(prev_state_ids.values()))
-
-        return {"state": list(state.values()), "auth_chain": auth_chain}
-
     async def on_invite_request(
         self, origin: str, event: EventBase, room_version: RoomVersion
-    ):
+    ) -> EventBase:
         """We've got an invite event. Process and persist it. Sign it.
 
         Respond with the now signed event.
@@ -1791,7 +1928,7 @@ async def on_invite_request(
         member_handler = self.hs.get_room_member_handler()
         # We don't rate limit based on room ID, as that should be done by
         # sending server.
-        member_handler.ratelimit_invite(None, event.state_key)
+        await member_handler.ratelimit_invite(None, None, event.state_key)
 
         # keep a record of the room version, if we don't yet know it.
         # (this may get overwritten if we later get a different room version in a
@@ -1852,7 +1989,7 @@ async def _make_and_verify_event(
         room_id: str,
         user_id: str,
         membership: str,
-        content: JsonDict = {},
+        content: JsonDict,
         params: Optional[Dict[str, Union[str, Iterable[str]]]] = None,
     ) -> Tuple[str, EventBase, RoomVersion]:
         (
@@ -1912,7 +2049,7 @@ async def on_make_leave_request(
         try:
             # The remote hasn't signed it yet, obviously. We'll do the full checks
             # when we get the event back in `on_send_leave_request`
-            await self.auth.check_from_context(
+            await self._event_auth_handler.check_from_context(
                 room_version, event, context, do_sig_check=False
             )
         except AuthError as e:
@@ -1921,36 +2058,6 @@ async def on_make_leave_request(
 
         return event
 
-    async def on_send_leave_request(self, origin, pdu):
-        """ We have received a leave event for a room. Fully process it."""
-        event = pdu
-
-        logger.debug(
-            "on_send_leave_request: Got event: %s, signatures: %s",
-            event.event_id,
-            event.signatures,
-        )
-
-        if get_domain_from_id(event.sender) != origin:
-            logger.info(
-                "Got /send_leave request for user %r from different origin %s",
-                event.sender,
-                origin,
-            )
-            raise SynapseError(403, "User not from origin", Codes.FORBIDDEN)
-
-        event.internal_metadata.outlier = False
-
-        await self._handle_new_event(origin, event)
-
-        logger.debug(
-            "on_send_leave_request: After _handle_new_event: %s, sigs: %s",
-            event.event_id,
-            event.signatures,
-        )
-
-        return None
-
     @log_function
     async def on_make_knock_request(
         self, origin: str, room_id: str, user_id: str
@@ -1969,8 +2076,7 @@ async def on_make_knock_request(
         """
         if get_domain_from_id(user_id) != origin:
             logger.info(
-                "Get /xyz.amorgan.knock/make_knock request for user %r"
-                "from different origin %s, ignoring",
+                "Get /make_knock request for user %r from different origin %s, ignoring",
                 user_id,
                 origin,
             )
@@ -1993,7 +2099,7 @@ async def on_make_knock_request(
             builder=builder
         )
 
-        event_allowed = await self.third_party_event_rules.check_event_allowed(
+        event_allowed, _ = await self.third_party_event_rules.check_event_allowed(
             event, context
         )
         if not event_allowed:
@@ -2005,7 +2111,7 @@ async def on_make_knock_request(
         try:
             # The remote hasn't signed it yet, obviously. We'll do the full checks
             # when we get the event back in `on_send_knock_request`
-            await self.auth.check_from_context(
+            await self._event_auth_handler.check_from_context(
                 room_version, event, context, do_sig_check=False
             )
         except AuthError as e:
@@ -2015,55 +2121,114 @@ async def on_make_knock_request(
         return event
 
     @log_function
-    async def on_send_knock_request(
+    async def on_send_membership_event(
         self, origin: str, event: EventBase
-    ) -> EventContext:
+    ) -> Tuple[EventBase, EventContext]:
         """
-        We have received a knock event for a room. Verify that event and send it into the room
-        on the knocking homeserver's behalf.
+        We have received a join/leave/knock event for a room via send_join/leave/knock.
+
+        Verify that event and send it into the room on the remote homeserver's behalf.
+
+        This is quite similar to on_receive_pdu, with the following principal
+        differences:
+          * only membership events are permitted (and only events with
+            sender==state_key -- ie, no kicks or bans)
+          * *We* send out the event on behalf of the remote server.
+          * We enforce the membership restrictions of restricted rooms.
+          * Rejected events result in an exception rather than being stored.
+
+        There are also other differences, however it is not clear if these are by
+        design or omission. In particular, we do not attempt to backfill any missing
+        prev_events.
 
         Args:
-            origin: The remote homeserver of the knocking user.
-            event: The knocking member event that has been signed by the remote homeserver.
+            origin: The homeserver of the remote (joining/invited/knocking) user.
+            event: The member event that has been signed by the remote homeserver.
 
         Returns:
-            The context of the event after inserting it into the room graph.
+            The event and context of the event after inserting it into the room graph.
+
+        Raises:
+            SynapseError if the event is not accepted into the room
         """
         logger.debug(
-            "on_send_knock_request: Got event: %s, signatures: %s",
+            "on_send_membership_event: Got event: %s, signatures: %s",
             event.event_id,
             event.signatures,
         )
 
         if get_domain_from_id(event.sender) != origin:
             logger.info(
-                "Got /xyz.amorgan.knock/send_knock request for user %r "
-                "from different origin %s",
+                "Got send_membership request for user %r from different origin %s",
                 event.sender,
                 origin,
             )
             raise SynapseError(403, "User not from origin", Codes.FORBIDDEN)
 
-        event.internal_metadata.outlier = False
+        if event.sender != event.state_key:
+            raise SynapseError(400, "state_key and sender must match", Codes.BAD_JSON)
 
-        context = await self._handle_new_event(origin, event)
+        assert not event.internal_metadata.outlier
 
-        event_allowed = await self.third_party_event_rules.check_event_allowed(
-            event, context
-        )
-        if not event_allowed:
-            logger.info("Sending of knock %s forbidden by third-party rules", event)
+        # Send this event on behalf of the other server.
+        #
+        # The remote server isn't a full participant in the room at this point, so
+        # may not have an up-to-date list of the other homeservers participating in
+        # the room, so we send it on their behalf.
+        event.internal_metadata.send_on_behalf_of = origin
+
+        context = await self.state_handler.compute_event_context(event)
+        context = await self._check_event_auth(origin, event, context)
+        if context.rejected:
             raise SynapseError(
-                403, "This event is not allowed in this context", Codes.FORBIDDEN
+                403, f"{event.membership} event was rejected", Codes.FORBIDDEN
             )
 
-        logger.debug(
-            "on_send_knock_request: After _handle_new_event: %s, sigs: %s",
-            event.event_id,
-            event.signatures,
-        )
+        # for joins, we need to check the restrictions of restricted rooms
+        if event.membership == Membership.JOIN:
+            await self._check_join_restrictions(context, event)
 
-        return context
+        # for knock events, we run the third-party event rules. It's not entirely clear
+        # why we don't do this for other sorts of membership events.
+        if event.membership == Membership.KNOCK:
+            event_allowed, _ = await self.third_party_event_rules.check_event_allowed(
+                event, context
+            )
+            if not event_allowed:
+                logger.info("Sending of knock %s forbidden by third-party rules", event)
+                raise SynapseError(
+                    403, "This event is not allowed in this context", Codes.FORBIDDEN
+                )
+
+        # all looks good, we can persist the event.
+        await self._run_push_actions_and_persist_event(event, context)
+        return event, context
+
+    async def _check_join_restrictions(
+        self, context: EventContext, event: EventBase
+    ) -> None:
+        """Check that restrictions in restricted join rules are matched
+
+        Called when we receive a join event via send_join.
+
+        Raises an auth error if the restrictions are not matched.
+        """
+        prev_state_ids = await context.get_prev_state_ids()
+
+        # Check if the user is already in the room or invited to the room.
+        user_id = event.state_key
+        prev_member_event_id = prev_state_ids.get((EventTypes.Member, user_id), None)
+        prev_member_event = None
+        if prev_member_event_id:
+            prev_member_event = await self.store.get_event(prev_member_event_id)
+
+        # Check if the member should be allowed access via membership in a space.
+        await self._event_auth_handler.check_restricted_join_rules(
+            prev_state_ids,
+            event.room_version,
+            user_id,
+            prev_member_event,
+        )
 
     async def get_state_for_pdu(self, room_id: str, event_id: str) -> List[EventBase]:
         """Returns the state at the event. i.e. not including said event."""
@@ -2118,7 +2283,7 @@ async def get_state_ids_for_pdu(self, room_id: str, event_id: str) -> List[str]:
     async def on_backfill_request(
         self, origin: str, room_id: str, pdu_list: List[str], limit: int
     ) -> List[EventBase]:
-        in_room = await self.auth.check_host_in_room(room_id, origin)
+        in_room = await self._event_auth_handler.check_host_in_room(room_id, origin)
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
@@ -2153,7 +2318,9 @@ async def get_persisted_pdu(
         )
 
         if event:
-            in_room = await self.auth.check_host_in_room(event.room_id, origin)
+            in_room = await self._event_auth_handler.check_host_in_room(
+                event.room_id, origin
+            )
             if not in_room:
                 raise AuthError(403, "Host not in room.")
 
@@ -2163,16 +2330,62 @@ async def get_persisted_pdu(
         else:
             return None
 
-    async def get_min_depth_for_context(self, context):
+    async def get_min_depth_for_context(self, context: str) -> int:
         return await self.store.get_min_depth(context)
 
-    async def _handle_new_event(
-        self, origin, event, state=None, auth_events=None, backfilled=False
-    ):
-        context = await self._prep_event(
-            origin, event, state=state, auth_events=auth_events, backfilled=backfilled
+    async def _auth_and_persist_event(
+        self,
+        origin: str,
+        event: EventBase,
+        context: EventContext,
+        state: Optional[Iterable[EventBase]] = None,
+        claimed_auth_event_map: Optional[StateMap[EventBase]] = None,
+        backfilled: bool = False,
+    ) -> None:
+        """
+        Process an event by performing auth checks and then persisting to the database.
+
+        Args:
+            origin: The host the event originates from.
+            event: The event itself.
+            context:
+                The event context.
+
+            state:
+                The state events used to check the event for soft-fail. If this is
+                not provided the current state events will be used.
+
+            claimed_auth_event_map:
+                A map of (type, state_key) => event for the event's claimed auth_events.
+                Possibly incomplete, and possibly including events that are not yet
+                persisted, or authed, or in the right room.
+
+                Only populated where we may not already have persisted these events -
+                for example, when populating outliers.
+
+            backfilled: True if the event was backfilled.
+        """
+        context = await self._check_event_auth(
+            origin,
+            event,
+            context,
+            state=state,
+            claimed_auth_event_map=claimed_auth_event_map,
+            backfilled=backfilled,
         )
 
+        await self._run_push_actions_and_persist_event(event, context, backfilled)
+
+    async def _run_push_actions_and_persist_event(
+        self, event: EventBase, context: EventContext, backfilled: bool = False
+    ):
+        """Run the push actions for a received event, and persist it.
+
+        Args:
+            event: The event itself.
+            context: The event context.
+            backfilled: True if the event was backfilled.
+        """
         try:
             if (
                 not event.internal_metadata.is_outlier()
@@ -2192,13 +2405,11 @@ async def _handle_new_event(
             )
             raise
 
-        return context
-
-    async def _handle_new_events(
+    async def _auth_and_persist_events(
         self,
         origin: str,
         room_id: str,
-        event_infos: Iterable[_NewEventInfo],
+        event_infos: Collection[_NewEventInfo],
         backfilled: bool = False,
     ) -> None:
         """Creates the appropriate contexts and persists events. The events
@@ -2209,14 +2420,21 @@ async def _handle_new_events(
         Notifies about the events where appropriate.
         """
 
+        if not event_infos:
+            return
+
         async def prep(ev_info: _NewEventInfo):
             event = ev_info.event
             with nested_logging_context(suffix=event.event_id):
-                res = await self._prep_event(
+                res = await self.state_handler.compute_event_context(
+                    event, old_state=ev_info.state
+                )
+                res = await self._check_event_auth(
                     origin,
                     event,
+                    res,
                     state=ev_info.state,
-                    auth_events=ev_info.auth_events,
+                    claimed_auth_event_map=ev_info.claimed_auth_event_map,
                     backfilled=backfilled,
                 )
             return res
@@ -2333,13 +2551,14 @@ async def _persist_auth_tree(
                     raise
                 events_to_context[e.event_id].rejected = RejectedReason.AUTH_ERROR
 
-        await self.persist_events_and_notify(
-            room_id,
-            [
-                (e, events_to_context[e.event_id])
-                for e in itertools.chain(auth_events, state)
-            ],
-        )
+        if auth_events or state:
+            await self.persist_events_and_notify(
+                room_id,
+                [
+                    (e, events_to_context[e.event_id])
+                    for e in itertools.chain(auth_events, state)
+                ],
+            )
 
         new_event_context = await self.state_handler.compute_event_context(
             event, old_state=state
@@ -2349,51 +2568,12 @@ async def _persist_auth_tree(
             room_id, [(event, new_event_context)]
         )
 
-    async def _prep_event(
+    async def _check_for_soft_fail(
         self,
-        origin: str,
         event: EventBase,
         state: Optional[Iterable[EventBase]],
-        auth_events: Optional[MutableStateMap[EventBase]],
         backfilled: bool,
-    ) -> EventContext:
-        context = await self.state_handler.compute_event_context(event, old_state=state)
-
-        if not auth_events:
-            prev_state_ids = await context.get_prev_state_ids()
-            auth_events_ids = self.auth.compute_auth_events(
-                event, prev_state_ids, for_verification=True
-            )
-            auth_events_x = await self.store.get_events(auth_events_ids)
-            auth_events = {(e.type, e.state_key): e for e in auth_events_x.values()}
-
-        # This is a hack to fix some old rooms where the initial join event
-        # didn't reference the create event in its auth events.
-        if event.type == EventTypes.Member and not event.auth_event_ids():
-            if len(event.prev_event_ids()) == 1 and event.depth < 5:
-                c = await self.store.get_event(
-                    event.prev_event_ids()[0], allow_none=True
-                )
-                if c and c.type == EventTypes.Create:
-                    auth_events[(c.type, c.state_key)] = c
-
-        context = await self.do_auth(origin, event, context, auth_events=auth_events)
-
-        if not context.rejected:
-            await self._check_for_soft_fail(event, state, backfilled)
-
-        if event.type == EventTypes.GuestAccess and not context.rejected:
-            await self.maybe_kick_guest_users(event)
-
-        # If we are going to send this event over federation we precaclculate
-        # the joined hosts.
-        if event.internal_metadata.get_send_on_behalf_of():
-            await self.event_creation_handler.cache_joined_hosts_for_event(event)
-
-        return context
-
-    async def _check_for_soft_fail(
-        self, event: EventBase, state: Optional[Iterable[EventBase]], backfilled: bool
+        origin: str,
     ) -> None:
         """Checks if we should soft fail the event; if so, marks the event as
         such.
@@ -2402,6 +2582,7 @@ async def _check_for_soft_fail(
             event
             state: The state at the event if we don't have all the event's prev events
             backfilled: Whether the event is from backfill
+            origin: The host the event originates from.
         """
         # For new (non-backfilled and non-outlier) events we check if the event
         # passes auth based on the current state. If it doesn't then we
@@ -2438,14 +2619,14 @@ async def _check_for_soft_fail(
             state_sets_d = await self.state_store.get_state_groups(
                 event.room_id, extrem_ids
             )
-            state_sets = list(state_sets_d.values())  # type: List[Iterable[EventBase]]
+            state_sets: List[Iterable[EventBase]] = list(state_sets_d.values())
             state_sets.append(state)
             current_states = await self.state_handler.resolve_events(
                 room_version, state_sets, event
             )
-            current_state_ids = {
+            current_state_ids: StateMap[str] = {
                 k: e.event_id for k, e in current_states.items()
-            }  # type: StateMap[str]
+            }
         else:
             current_state_ids = await self.state_handler.get_current_state_ids(
                 event.room_id, latest_event_ids=extrem_ids
@@ -2458,7 +2639,7 @@ async def _check_for_soft_fail(
         )
 
         # Now check if event pass auth against said current state
-        auth_types = auth_types_for_event(event)
+        auth_types = auth_types_for_event(room_version_obj, event)
         current_state_ids_list = [
             e for k, e in current_state_ids.items() if k in auth_types
         ]
@@ -2471,44 +2652,29 @@ async def _check_for_soft_fail(
         try:
             event_auth.check(room_version_obj, event, auth_events=current_auth_events)
         except AuthError as e:
-            logger.warning("Soft-failing %r because %s", event, e)
+            logger.warning(
+                "Soft-failing %r (from %s) because %s",
+                event,
+                e,
+                origin,
+                extra={
+                    "room_id": event.room_id,
+                    "mxid": event.sender,
+                    "hs": origin,
+                },
+            )
+            soft_failed_event_counter.inc()
             event.internal_metadata.soft_failed = True
 
-    async def on_query_auth(
-        self, origin, event_id, room_id, remote_auth_chain, rejects, missing
-    ):
-        in_room = await self.auth.check_host_in_room(room_id, origin)
-        if not in_room:
-            raise AuthError(403, "Host not in room.")
-
-        event = await self.store.get_event(event_id, check_room_id=room_id)
-
-        # Just go through and process each event in `remote_auth_chain`. We
-        # don't want to fall into the trap of `missing` being wrong.
-        for e in remote_auth_chain:
-            try:
-                await self._handle_new_event(origin, e)
-            except AuthError:
-                pass
-
-        # Now get the current auth_chain for the event.
-        local_auth_chain = await self.store.get_auth_chain(
-            room_id, list(event.auth_event_ids()), include_given=True
-        )
-
-        # TODO: Check if we would now reject event_id. If so we need to tell
-        # everyone.
-
-        ret = await self.construct_auth_difference(local_auth_chain, remote_auth_chain)
-
-        logger.debug("on_query_auth returning: %s", ret)
-
-        return ret
-
     async def on_get_missing_events(
-        self, origin, room_id, earliest_events, latest_events, limit
-    ):
-        in_room = await self.auth.check_host_in_room(room_id, origin)
+        self,
+        origin: str,
+        room_id: str,
+        earliest_events: List[str],
+        latest_events: List[str],
+        limit: int,
+    ) -> List[EventBase]:
+        in_room = await self._event_auth_handler.check_host_in_room(room_id, origin)
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
@@ -2528,36 +2694,63 @@ async def on_get_missing_events(
 
         return missing_events
 
-    async def do_auth(
+    async def _check_event_auth(
         self,
         origin: str,
         event: EventBase,
         context: EventContext,
-        auth_events: MutableStateMap[EventBase],
+        state: Optional[Iterable[EventBase]] = None,
+        claimed_auth_event_map: Optional[StateMap[EventBase]] = None,
+        backfilled: bool = False,
     ) -> EventContext:
         """
+        Checks whether an event should be rejected (for failing auth checks).
 
         Args:
-            origin:
-            event:
+            origin: The host the event originates from.
+            event: The event itself.
             context:
-            auth_events:
-                Map from (event_type, state_key) to event
+                The event context.
 
-                Normally, our calculated auth_events based on the state of the room
-                at the event's position in the DAG, though occasionally (eg if the
-                event is an outlier), may be the auth events claimed by the remote
-                server.
+            state:
+                The state events used to check the event for soft-fail. If this is
+                not provided the current state events will be used.
+
+            claimed_auth_event_map:
+                A map of (type, state_key) => event for the event's claimed auth_events.
+                Possibly incomplete, and possibly including events that are not yet
+                persisted, or authed, or in the right room.
+
+                Only populated where we may not already have persisted these events -
+                for example, when populating outliers, or the state for a backwards
+                extremity.
+
+            backfilled: True if the event was backfilled.
 
-                Also NB that this function adds entries to it.
         Returns:
-            updated context object
+            The updated context object.
         """
         room_version = await self.store.get_room_version_id(event.room_id)
         room_version_obj = KNOWN_ROOM_VERSIONS[room_version]
 
+        if claimed_auth_event_map:
+            # if we have a copy of the auth events from the event, use that as the
+            # basis for auth.
+            auth_events = claimed_auth_event_map
+        else:
+            # otherwise, we calculate what the auth events *should* be, and use that
+            prev_state_ids = await context.get_prev_state_ids()
+            auth_events_ids = self._event_auth_handler.compute_auth_events(
+                event, prev_state_ids, for_verification=True
+            )
+            auth_events_x = await self.store.get_events(auth_events_ids)
+            auth_events = {(e.type, e.state_key): e for e in auth_events_x.values()}
+
         try:
-            context = await self._update_auth_events_and_context_for_auth(
+            (
+                context,
+                auth_events_for_auth,
+            ) = await self._update_auth_events_and_context_for_auth(
                 origin, event, context, auth_events
             )
         except Exception:
@@ -2570,13 +2763,27 @@ async def do_auth(
                 "Ignoring failure and continuing processing of event.",
                 event.event_id,
             )
+            auth_events_for_auth = auth_events
 
         try:
-            event_auth.check(room_version_obj, event, auth_events=auth_events)
+            event_auth.check(room_version_obj, event, auth_events=auth_events_for_auth)
         except AuthError as e:
             logger.warning("Failed auth resolution for %r because %s", event, e)
             context.rejected = RejectedReason.AUTH_ERROR
 
+        if not context.rejected:
+            await self._check_for_soft_fail(event, state, backfilled, origin=origin)
+
+        if event.type == EventTypes.GuestAccess and not context.rejected:
+            await self.maybe_kick_guest_users(event)
+
+        # If we are going to send this event over federation we precaclculate
+        # the joined hosts.
+        if event.internal_metadata.get_send_on_behalf_of():
+            await self.event_creation_handler.cache_joined_hosts_for_event(
+                event, context
+            )
+
         return context
 
     async def _update_auth_events_and_context_for_auth(
@@ -2584,9 +2791,9 @@ async def _update_auth_events_and_context_for_auth(
         origin: str,
         event: EventBase,
         context: EventContext,
-        auth_events: MutableStateMap[EventBase],
-    ) -> EventContext:
-        """Helper for do_auth. See there for docs.
+        input_auth_events: StateMap[EventBase],
+    ) -> Tuple[EventContext, StateMap[EventBase]]:
+        """Helper for _check_event_auth. See there for docs.
 
         Checks whether a given event has the expected auth events. If it
         doesn't then we talk to the remote server to compare state to see if
@@ -2602,7 +2809,7 @@ async def _update_auth_events_and_context_for_auth(
             event:
             context:
 
-            auth_events:
+            input_auth_events:
                 Map from (event_type, state_key) to event
 
                 Normally, our calculated auth_events based on the state of the room
@@ -2610,11 +2817,12 @@ async def _update_auth_events_and_context_for_auth(
                 event is an outlier), may be the auth events claimed by the remote
                 server.
 
-                Also NB that this function adds entries to it.
-
         Returns:
-            updated context
+            updated context, updated auth event map
         """
+        # take a copy of input_auth_events before we modify it.
+        auth_events: MutableStateMap[EventBase] = dict(input_auth_events)
+
         event_auth_events = set(event.auth_event_ids())
 
         # missing_auth is the set of the event's auth_events which we don't yet have
@@ -2627,7 +2835,7 @@ async def _update_auth_events_and_context_for_auth(
         #
         # we start by checking if they are in the store, and then try calling /event_auth/.
         if missing_auth:
-            have_events = await self.store.have_seen_events(missing_auth)
+            have_events = await self.store.have_seen_events(event.room_id, missing_auth)
             logger.debug("Events %s are in the store", have_events)
             missing_auth.difference_update(have_events)
 
@@ -2643,10 +2851,10 @@ async def _update_auth_events_and_context_for_auth(
                     # The other side isn't around or doesn't implement the
                     # endpoint, so lets just bail out.
                     logger.info("Failed to get event auth from remote: %s", e1)
-                    return context
+                    return context, auth_events
 
                 seen_remotes = await self.store.have_seen_events(
-                    [e.event_id for e in remote_auth_chain]
+                    event.room_id, [e.event_id for e in remote_auth_chain]
                 )
 
                 for e in remote_auth_chain:
@@ -2666,9 +2874,19 @@ async def _update_auth_events_and_context_for_auth(
                         e.internal_metadata.outlier = True
 
                         logger.debug(
-                            "do_auth %s missing_auth: %s", event.event_id, e.event_id
+                            "_check_event_auth %s missing_auth: %s",
+                            event.event_id,
+                            e.event_id,
+                        )
+                        missing_auth_event_context = (
+                            await self.state_handler.compute_event_context(e)
+                        )
+                        await self._auth_and_persist_event(
+                            origin,
+                            e,
+                            missing_auth_event_context,
+                            claimed_auth_event_map=auth,
                         )
-                        await self._handle_new_event(origin, e, auth_events=auth)
 
                         if e.event_id in event_auth_events:
                             auth_events[(e.type, e.state_key)] = e
@@ -2685,14 +2903,14 @@ async def _update_auth_events_and_context_for_auth(
             # obviously be empty
             # (b) alternatively, why don't we do it earlier?
             logger.info("Skipping auth_event fetch for outlier")
-            return context
+            return context, auth_events
 
         different_auth = event_auth_events.difference(
             e.event_id for e in auth_events.values()
         )
 
         if not different_auth:
-            return context
+            return context, auth_events
 
         logger.info(
             "auth_events refers to events which are not in our calculated auth "
@@ -2718,7 +2936,7 @@ async def _update_auth_events_and_context_for_auth(
                 # XXX: should we reject the event in this case? It feels like we should,
                 # but then shouldn't we also do so if we've failed to fetch any of the
                 # auth events?
-                return context
+                return context, auth_events
 
         # now we state-resolve between our own idea of the auth events, and the remote's
         # idea of them.
@@ -2748,7 +2966,7 @@ async def _update_auth_events_and_context_for_auth(
             event, context, auth_events
         )
 
-        return context
+        return context, auth_events
 
     async def _update_context_for_auth_events(
         self, event: EventBase, context: EventContext, auth_events: StateMap[EventBase]
@@ -2768,7 +2986,7 @@ async def _update_context_for_auth_events(
         """
         # exclude the state key of the new event from the current_state in the context.
         if event.is_state():
-            event_key = (event.type, event.state_key)  # type: Optional[Tuple[str, str]]
+            event_key: Optional[Tuple[str, str]] = (event.type, event.state_key)
         else:
             event_key = None
         state_updates = {
@@ -2811,8 +3029,8 @@ async def construct_auth_difference(
         assumes that we have already processed all events in remote_auth
 
         Params:
-            local_auth (list)
-            remote_auth (list)
+            local_auth
+            remote_auth
 
         Returns:
             dict
@@ -2936,8 +3154,8 @@ def get_next(it, opt=None):
 
     @log_function
     async def exchange_third_party_invite(
-        self, sender_user_id, target_user_id, room_id, signed
-    ):
+        self, sender_user_id: str, target_user_id: str, room_id: str, signed: JsonDict
+    ) -> None:
         third_party_invite = {"signed": signed}
 
         event_dict = {
@@ -2951,7 +3169,7 @@ async def exchange_third_party_invite(
             "state_key": target_user_id,
         }
 
-        if await self.auth.check_host_in_room(room_id, self.hs.hostname):
+        if await self._event_auth_handler.check_host_in_room(room_id, self.hs.hostname):
             room_version = await self.store.get_room_version_id(room_id)
             builder = self.event_builder_factory.new(room_version, event_dict)
 
@@ -2971,7 +3189,9 @@ async def exchange_third_party_invite(
             event.internal_metadata.send_on_behalf_of = self.hs.hostname
 
             try:
-                await self.auth.check_from_context(room_version, event, context)
+                await self._event_auth_handler.check_from_context(
+                    room_version, event, context
+                )
             except AuthError as e:
                 logger.warning("Denying new third party invite %r because %s", event, e)
                 raise e
@@ -2983,9 +3203,13 @@ async def exchange_third_party_invite(
             await member_handler.send_membership_event(None, event, context)
         else:
             destinations = {x.split(":", 1)[-1] for x in (sender_user_id, room_id)}
-            await self.federation_client.forward_third_party_invite(
-                destinations, room_id, event_dict
-            )
+
+            try:
+                await self.federation_client.forward_third_party_invite(
+                    destinations, room_id, event_dict
+                )
+            except (RequestSendFailed, HttpResponseException):
+                raise SynapseError(502, "Failed to forward third party invite")
 
     async def on_exchange_third_party_invite_request(
         self, event_dict: JsonDict
@@ -3014,7 +3238,9 @@ async def on_exchange_third_party_invite_request(
         )
 
         try:
-            await self.auth.check_from_context(room_version, event, context)
+            await self._event_auth_handler.check_from_context(
+                room_version, event, context
+            )
         except AuthError as e:
             logger.warning("Denying third party invite %r because %s", event, e)
             raise e
@@ -3029,8 +3255,12 @@ async def on_exchange_third_party_invite_request(
         await member_handler.send_membership_event(None, event, context)
 
     async def add_display_name_to_third_party_invite(
-        self, room_version, event_dict, event, context
-    ):
+        self,
+        room_version: str,
+        event_dict: JsonDict,
+        event: EventBase,
+        context: EventContext,
+    ) -> Tuple[EventBase, EventContext]:
         key = (
             EventTypes.ThirdPartyInvite,
             event.content["third_party_invite"]["signed"]["token"],
@@ -3066,13 +3296,13 @@ async def add_display_name_to_third_party_invite(
         EventValidator().validate_new(event, self.config)
         return (event, context)
 
-    async def _check_signature(self, event, context):
+    async def _check_signature(self, event: EventBase, context: EventContext) -> None:
         """
         Checks that the signature in the event is consistent with its invite.
 
         Args:
-            event (Event): The m.room.member event to check
-            context (EventContext):
+            event: The m.room.member event to check
+            context:
 
         Raises:
             AuthError: if signature didn't match any keys, or key has been
@@ -3095,14 +3325,14 @@ async def _check_signature(self, event, context):
 
         logger.debug("Checking auth on event %r", event.content)
 
-        last_exception = None  # type: Optional[Exception]
+        last_exception: Optional[Exception] = None
 
         # for each public key in the 3pid invite event
-        for public_key_object in self.hs.get_auth().get_public_keys(invite_event):
+        for public_key_object in event_auth.get_public_keys(invite_event):
             try:
                 # for each sig on the third_party_invite block of the actual invite
                 for server, signature_block in signed["signatures"].items():
-                    for key_name, encoded_signature in signature_block.items():
+                    for key_name in signature_block.keys():
                         if not key_name.startswith("ed25519:"):
                             continue
 
@@ -3158,13 +3388,13 @@ async def _check_signature(self, event, context):
 
         raise last_exception
 
-    async def _check_key_revocation(self, public_key, url):
+    async def _check_key_revocation(self, public_key: str, url: str) -> None:
         """
         Checks whether public_key has been revoked.
 
         Args:
-            public_key (str): base-64 encoded public key.
-            url (str): Key revocation URL.
+            public_key: base-64 encoded public key.
+            url: Key revocation URL.
 
         Raises:
             AuthError: if they key has been revoked.
@@ -3194,16 +3424,25 @@ async def persist_events_and_notify(
                 the same room.
             backfilled: Whether these events are a result of
                 backfilling or not
+
+        Returns:
+            The stream ID after which all events have been persisted.
         """
+        if not event_and_contexts:
+            return self.store.get_current_events_token()
+
         instance = self.config.worker.events_shard_config.get_instance(room_id)
         if instance != self._instance_name:
-            result = await self._send_events(
-                instance_name=instance,
-                store=self.store,
-                room_id=room_id,
-                event_and_contexts=event_and_contexts,
-                backfilled=backfilled,
-            )
+            # Limit the number of events sent over replication. We choose 200
+            # here as that is what we default to in `max_request_body_size(..)`
+            for batch in batch_iter(event_and_contexts, 200):
+                result = await self._send_events(
+                    instance_name=instance,
+                    store=self.store,
+                    room_id=room_id,
+                    event_and_contexts=batch,
+                    backfilled=backfilled,
+                )
             return result["max_stream_id"]
         else:
             assert self.storage.persistence
diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index a41ca5df9c..1a6c5c64a2 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -215,7 +214,7 @@ async def get_publicised_groups_for_user(self, user_id: str) -> JsonDict:
     async def bulk_get_publicised_groups(
         self, user_ids: Iterable[str], proxy: bool = True
     ) -> JsonDict:
-        destinations = {}  # type: Dict[str, Set[str]]
+        destinations: Dict[str, Set[str]] = {}
         local_users = set()
 
         for user_id in user_ids:
@@ -228,7 +227,7 @@ async def bulk_get_publicised_groups(
             raise SynapseError(400, "Some user_ids are not local")
 
         results = {}
-        failed_results = []  # type: List[str]
+        failed_results: List[str] = []
         for destination, dest_user_ids in destinations.items():
             try:
                 r = await self.transport_client.bulk_get_publicised_groups(
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index f4c1265b43..6242b6b323 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018, 2019 New Vector Ltd
@@ -16,10 +15,9 @@
 # limitations under the License.
 
 """Utilities for interacting with Identity Servers"""
-
 import logging
 import urllib.parse
-from typing import Awaitable, Callable, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Awaitable, Callable, Dict, List, Optional, Tuple
 
 from synapse.api.errors import (
     AuthError,
@@ -37,15 +35,22 @@
 from synapse.types import JsonDict, Requester
 from synapse.util import json_decoder
 from synapse.util.hash import sha256_and_url_safe_base64
-from synapse.util.stringutils import assert_valid_client_secret, random_string
+from synapse.util.stringutils import (
+    assert_valid_client_secret,
+    random_string,
+    valid_id_server_location,
+)
 
 from ._base import BaseHandler
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
 class IdentityHandler(BaseHandler):
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         # An HTTP client for contacting trusted URLs.
@@ -64,22 +69,24 @@ def __init__(self, hs):
 
         # Ratelimiters for `/requestToken` endpoints.
         self._3pid_validation_ratelimiter_ip = Ratelimiter(
+            store=self.store,
             clock=hs.get_clock(),
             rate_hz=hs.config.ratelimiting.rc_3pid_validation.per_second,
             burst_count=hs.config.ratelimiting.rc_3pid_validation.burst_count,
         )
         self._3pid_validation_ratelimiter_address = Ratelimiter(
+            store=self.store,
             clock=hs.get_clock(),
             rate_hz=hs.config.ratelimiting.rc_3pid_validation.per_second,
             burst_count=hs.config.ratelimiting.rc_3pid_validation.burst_count,
         )
 
-    def ratelimit_request_token_requests(
+    async def ratelimit_request_token_requests(
         self,
         request: SynapseRequest,
         medium: str,
         address: str,
-    ):
+    ) -> None:
         """Used to ratelimit requests to `/requestToken` by IP and address.
 
         Args:
@@ -88,8 +95,12 @@ def ratelimit_request_token_requests(
             address: The actual threepid ID, e.g. the phone number or email address
         """
 
-        self._3pid_validation_ratelimiter_ip.ratelimit((medium, request.getClientIP()))
-        self._3pid_validation_ratelimiter_address.ratelimit((medium, address))
+        await self._3pid_validation_ratelimiter_ip.ratelimit(
+            None, (medium, request.getClientIP())
+        )
+        await self._3pid_validation_ratelimiter_address.ratelimit(
+            None, (medium, address)
+        )
 
     async def threepid_from_creds(
         self, id_server_url: str, creds: Dict[str, str]
@@ -177,6 +188,11 @@ async def bind_threepid(
                 server with, if necessary. Required if use_v2 is true
             use_v2: Whether to use v2 Identity Service API endpoints. Defaults to True
 
+        Raises:
+            SynapseError: On any of the following conditions
+                - the supplied id_server is not a valid identity server name
+                - we failed to contact the supplied identity server
+
         Returns:
             The response from the identity server
         """
@@ -186,6 +202,12 @@ async def bind_threepid(
         if id_access_token is None:
             use_v2 = False
 
+        if not valid_id_server_location(id_server):
+            raise SynapseError(
+                400,
+                "id_server must be a valid hostname with optional port and path components",
+            )
+
         # if we have a rewrite rule set for the identity server,
         # apply it now, but only for sending the request (not
         # storing in the database).
@@ -279,12 +301,24 @@ async def try_unbind_threepid_with_id_server(
             id_server: Identity server to unbind from
 
         Raises:
-            SynapseError: If we failed to contact the identity server
+            SynapseError: On any of the following conditions
+                - the supplied id_server is not a valid identity server name
+                - we failed to contact the supplied identity server
 
         Returns:
             True on success, otherwise False if the identity
             server doesn't support unbinding
         """
+
+        if not valid_id_server_location(id_server):
+            raise SynapseError(
+                400,
+                "id_server must be a valid hostname with optional port and path components",
+            )
+
+        url = "https://%s/_matrix/identity/api/v1/3pid/unbind" % (id_server,)
+        url_bytes = b"/_matrix/identity/api/v1/3pid/unbind"
+
         content = {
             "mxid": mxid,
             "threepid": {"medium": threepid["medium"], "address": threepid["address"]},
@@ -802,7 +836,7 @@ async def _lookup_3pid_v1(
                 return data["mxid"]
         except RequestTimedOutError:
             raise SynapseError(500, "Timed out contacting identity server")
-        except IOError as e:
+        except OSError as e:
             logger.warning("Error from v1 identity server lookup: %s" % (e,))
 
         return None
@@ -929,6 +963,7 @@ async def ask_id_server_for_third_party_invite(
         room_avatar_url: str,
         room_join_rules: str,
         room_name: str,
+        room_type: Optional[str],
         inviter_display_name: str,
         inviter_avatar_url: str,
         id_access_token: Optional[str] = None,
@@ -948,6 +983,7 @@ async def ask_id_server_for_third_party_invite(
                 notifications.
             room_join_rules: The join rules of the email (e.g. "public").
             room_name: The m.room.name of the room.
+            room_type: The type of the room from its m.room.create event (e.g "m.space").
             inviter_display_name: The current display name of the
                 inviter.
             inviter_avatar_url: The URL of the inviter's avatar.
@@ -974,6 +1010,10 @@ async def ask_id_server_for_third_party_invite(
             "sender_display_name": inviter_display_name,
             "sender_avatar_url": inviter_avatar_url,
         }
+
+        if room_type is not None:
+            invite_config["org.matrix.msc3288.room_type"] = room_type
+
         # If a custom web client location is available, include it in the request.
         if self._web_client_location:
             invite_config["org.matrix.web_client_location"] = self._web_client_location
diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index 13f8152283..e1c544a3c9 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,6 +21,7 @@
 from synapse.api.errors import SynapseError
 from synapse.events.validator import EventValidator
 from synapse.handlers.presence import format_user_presence_state
+from synapse.handlers.receipts import ReceiptEventSource
 from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.storage.roommember import RoomsForUser
 from synapse.streams.config import PaginationConfig
@@ -47,9 +47,17 @@ def __init__(self, hs: "HomeServer"):
         self.state = hs.get_state_handler()
         self.clock = hs.get_clock()
         self.validator = EventValidator()
-        self.snapshot_cache = ResponseCache(
-            hs.get_clock(), "initial_sync_cache"
-        )  # type: ResponseCache[Tuple[str, Optional[StreamToken], Optional[StreamToken], str, Optional[int], bool, bool]]
+        self.snapshot_cache: ResponseCache[
+            Tuple[
+                str,
+                Optional[StreamToken],
+                Optional[StreamToken],
+                str,
+                Optional[int],
+                bool,
+                bool,
+            ]
+        ] = ResponseCache(hs.get_clock(), "initial_sync_cache")
         self._event_serializer = hs.get_event_client_serializer()
         self.storage = hs.get_storage()
         self.state_store = self.storage.state
@@ -127,6 +135,8 @@ async def _snapshot_all_rooms(
             joined_rooms,
             to_key=int(now_token.receipt_key),
         )
+        if self.hs.config.experimental.msc2285_enabled:
+            receipt = ReceiptEventSource.filter_out_hidden(receipt, user_id)
 
         tags_by_room = await self.store.get_tags_for_user(user_id)
 
@@ -423,7 +433,9 @@ async def get_receipts():
                 room_id, to_key=now_token.receipt_key
             )
             if not receipts:
-                receipts = []
+                return []
+            if self.hs.config.experimental.msc2285_enabled:
+                receipts = ReceiptEventSource.filter_out_hidden(receipts, user_id)
             return receipts
 
         presence, receipts, (messages, token) = await make_deferred_yieldable(
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 67a8410276..101a29c6d3 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019-2020 The Matrix.org Foundation C.I.C.
@@ -17,10 +16,11 @@
 # limitations under the License.
 import logging
 import random
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple
 
 from canonicaljson import encode_canonical_json
 
+from twisted.internet import defer
 from twisted.internet.interfaces import IDelayedCall
 
 from synapse import event_auth
@@ -41,19 +41,19 @@
 )
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions
 from synapse.api.urls import ConsentURIBuilder
-from synapse.config.api import DEFAULT_ROOM_STATE_TYPES
 from synapse.events import EventBase
 from synapse.events.builder import EventBuilder
 from synapse.events.snapshot import EventContext
 from synapse.events.validator import EventValidator
-from synapse.logging.context import run_in_background
+from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.http.send_event import ReplicationSendEventRestServlet
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.storage.state import StateFilter
 from synapse.types import Requester, RoomAlias, StreamToken, UserID, create_requester
-from synapse.util import json_decoder, json_encoder
-from synapse.util.async_helpers import Linearizer
+from synapse.util import json_decoder, json_encoder, log_failure
+from synapse.util.async_helpers import Linearizer, unwrapFirstError
+from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.metrics import measure_func
 from synapse.visibility import filter_events_for_client
 
@@ -69,7 +69,7 @@
 class MessageHandler:
     """Contains some read only APIs to get state about a room"""
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.auth = hs.get_auth()
         self.clock = hs.get_clock()
         self.state = hs.get_state_handler()
@@ -81,7 +81,7 @@ def __init__(self, hs):
 
         # The scheduled call to self._expire_event. None if no call is currently
         # scheduled.
-        self._scheduled_expiry = None  # type: Optional[IDelayedCall]
+        self._scheduled_expiry: Optional[IDelayedCall] = None
 
         if not hs.config.worker_app:
             run_as_background_process(
@@ -94,7 +94,7 @@ async def get_room_data(
         room_id: str,
         event_type: str,
         state_key: str,
-    ) -> dict:
+    ) -> Optional[EventBase]:
         """Get data from a room.
 
         Args:
@@ -118,6 +118,10 @@ async def get_room_data(
             data = await self.state.get_current_state(room_id, event_type, state_key)
         elif membership == Membership.LEAVE:
             key = (event_type, state_key)
+            # If the membership is not JOIN, then the event ID should exist.
+            assert (
+                membership_event_id is not None
+            ), "check_user_in_room_or_world_readable returned invalid data"
             room_state = await self.state_store.get_state_for_events(
                 [membership_event_id], StateFilter.from_types([key])
             )
@@ -139,7 +143,7 @@ async def get_state_events(
         self,
         user_id: str,
         room_id: str,
-        state_filter: StateFilter = StateFilter.all(),
+        state_filter: Optional[StateFilter] = None,
         at_token: Optional[StreamToken] = None,
         is_guest: bool = False,
     ) -> List[dict]:
@@ -166,6 +170,8 @@ async def get_state_events(
             AuthError (403) if the user doesn't have permission to view
             members of this room.
         """
+        state_filter = state_filter or StateFilter.all()
+
         if at_token:
             # FIXME this claims to get the state at a stream position, but
             # get_recent_events_for_room operates by topo ordering. This therefore
@@ -177,20 +183,37 @@ async def get_state_events(
 
             if not last_events:
                 raise NotFoundError("Can't find event for token %s" % (at_token,))
+            last_event = last_events[0]
+
+            # check whether the user is in the room at that time to determine
+            # whether they should be treated as peeking.
+            state_map = await self.state_store.get_state_for_event(
+                last_event.event_id,
+                StateFilter.from_types([(EventTypes.Member, user_id)]),
+            )
+
+            joined = False
+            membership_event = state_map.get((EventTypes.Member, user_id))
+            if membership_event:
+                joined = membership_event.membership == Membership.JOIN
+
+            is_peeking = not joined
 
             visible_events = await filter_events_for_client(
                 self.storage,
                 user_id,
                 last_events,
                 filter_send_to_client=False,
+                is_peeking=is_peeking,
             )
 
-            event = last_events[0]
             if visible_events:
-                room_state = await self.state_store.get_state_for_events(
-                    [event.event_id], state_filter=state_filter
+                room_state_events = await self.state_store.get_state_for_events(
+                    [last_event.event_id], state_filter=state_filter
                 )
-                room_state = room_state[event.event_id]
+                room_state: Mapping[Any, EventBase] = room_state_events[
+                    last_event.event_id
+                ]
             else:
                 raise AuthError(
                     403,
@@ -211,10 +234,14 @@ async def get_state_events(
                 )
                 room_state = await self.store.get_events(state_ids.values())
             elif membership == Membership.LEAVE:
-                room_state = await self.state_store.get_state_for_events(
+                # If the membership is not JOIN, then the event ID should exist.
+                assert (
+                    membership_event_id is not None
+                ), "check_user_in_room_or_world_readable returned invalid data"
+                room_state_events = await self.state_store.get_state_for_events(
                     [membership_event_id], state_filter=state_filter
                 )
-                room_state = room_state[membership_event_id]
+                room_state = room_state_events[membership_event_id]
 
         now = self.clock.time_msec()
         events = await self._event_serializer.serialize_events(
@@ -249,7 +276,7 @@ async def get_joined_members(self, requester: Requester, room_id: str) -> dict:
                     "Getting joined members after leaving is not implemented"
                 )
 
-        users_with_profile = await self.state.get_current_users_in_room(room_id)
+        users_with_profile = await self.store.get_users_in_room_with_profiles(room_id)
 
         # If this is an AS, double check that they are allowed to see the members.
         # This can either be because the AS user is in the room or because there
@@ -373,6 +400,7 @@ class EventCreationHandler:
     def __init__(self, hs: "HomeServer"):
         self.hs = hs
         self.auth = hs.get_auth()
+        self._event_auth_handler = hs.get_event_auth_handler()
         self.store = hs.get_datastore()
         self.storage = hs.get_storage()
         self.state = hs.get_state_handler()
@@ -387,13 +415,14 @@ def __init__(self, hs: "HomeServer"):
         self._events_shard_config = self.config.worker.events_shard_config
         self._instance_name = hs.get_instance_name()
 
-        self.room_invite_state_types = self.hs.config.room_invite_state_types
+        self.room_prejoin_state_types = self.hs.config.api.room_prejoin_state
 
-        self.membership_types_to_include_profile_data_in = (
-            {Membership.JOIN, Membership.INVITE, Membership.KNOCK}
-            if self.hs.config.include_profile_data_on_invite
-            else {Membership.JOIN, Membership.KNOCK}
-        )
+        self.membership_types_to_include_profile_data_in = {
+            Membership.JOIN,
+            Membership.KNOCK,
+        }
+        if self.hs.config.include_profile_data_on_invite:
+            self.membership_types_to_include_profile_data_in.add(Membership.INVITE)
 
         self.send_event = ReplicationSendEventRestServlet.make_client(hs)
 
@@ -407,9 +436,9 @@ def __init__(self, hs: "HomeServer"):
         self.action_generator = hs.get_action_generator()
 
         self.spam_checker = hs.get_spam_checker()
-        self.third_party_event_rules = (
+        self.third_party_event_rules: "ThirdPartyEventRules" = (
             self.hs.get_third_party_event_rules()
-        )  # type: ThirdPartyEventRules
+        )
 
         self._block_events_without_consent_error = (
             self.config.block_events_without_consent_error
@@ -426,7 +455,7 @@ def __init__(self, hs: "HomeServer"):
         #
         # map from room id to time-of-last-attempt.
         #
-        self._rooms_to_exclude_from_dummy_event_insertion = {}  # type: Dict[str, int]
+        self._rooms_to_exclude_from_dummy_event_insertion: Dict[str, int] = {}
         # The number of forward extremeities before a dummy event is sent.
         self._dummy_events_threshold = hs.config.dummy_events_threshold
 
@@ -448,6 +477,17 @@ def __init__(self, hs: "HomeServer"):
 
         self._external_cache = hs.get_external_cache()
 
+        # Stores the state groups we've recently added to the joined hosts
+        # external cache. Note that the timeout must be significantly less than
+        # the TTL on the external cache.
+        self._external_cache_joined_hosts_updates: Optional[ExpiringCache] = None
+        if self._external_cache.is_enabled():
+            self._external_cache_joined_hosts_updates = ExpiringCache(
+                "_external_cache_joined_hosts_updates",
+                self.clock,
+                expiry_ms=30 * 60 * 1000,
+            )
+
     async def create_event(
         self,
         requester: Requester,
@@ -456,6 +496,9 @@ async def create_event(
         prev_event_ids: Optional[List[str]] = None,
         auth_event_ids: Optional[List[str]] = None,
         require_consent: bool = True,
+        outlier: bool = False,
+        historical: bool = False,
+        depth: Optional[int] = None,
     ) -> Tuple[EventBase, EventContext]:
         """
         Given a dict from a client, create a new event.
@@ -480,8 +523,21 @@ async def create_event(
                 Should normally be left as None, which will cause them to be calculated
                 based on the room state at the prev_events.
 
+                If non-None, prev_event_ids must also be provided.
+
             require_consent: Whether to check if the requester has
                 consented to the privacy policy.
+
+            outlier: Indicates whether the event is an `outlier`, i.e. if
+                it's from an arbitrary point and floating in the DAG as
+                opposed to being inline with the current DAG.
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
+            depth: Override the depth used to order the event in the DAG.
+                Should normally be set to None, which will cause the depth to be calculated
+                based on the prev_events.
+
         Raises:
             ResourceLimitError if server is blocked to some resource being
             exceeded
@@ -537,11 +593,39 @@ async def create_event(
         if txn_id is not None:
             builder.internal_metadata.txn_id = txn_id
 
+        builder.internal_metadata.outlier = outlier
+
+        builder.internal_metadata.historical = historical
+
+        # Strip down the auth_event_ids to only what we need to auth the event.
+        # For example, we don't need extra m.room.member that don't match event.sender
+        if auth_event_ids is not None:
+            # If auth events are provided, prev events must be also.
+            assert prev_event_ids is not None
+
+            temp_event = await builder.build(
+                prev_event_ids=prev_event_ids,
+                auth_event_ids=auth_event_ids,
+                depth=depth,
+            )
+            auth_events = await self.store.get_events_as_list(auth_event_ids)
+            # Create a StateMap[str]
+            auth_event_state_map = {
+                (e.type, e.state_key): e.event_id for e in auth_events
+            }
+            # Actually strip down and use the necessary auth events
+            auth_event_ids = self._event_auth_handler.compute_auth_events(
+                event=temp_event,
+                current_state_ids=auth_event_state_map,
+                for_verification=False,
+            )
+
         event, context = await self.create_new_client_event(
             builder=builder,
             requester=requester,
             prev_event_ids=prev_event_ids,
             auth_event_ids=auth_event_ids,
+            depth=depth,
         )
 
         # In an ideal world we wouldn't need the second part of this condition. However,
@@ -698,9 +782,14 @@ async def create_and_send_nonmember_event(
         self,
         requester: Requester,
         event_dict: dict,
+        prev_event_ids: Optional[List[str]] = None,
+        auth_event_ids: Optional[List[str]] = None,
         ratelimit: bool = True,
         txn_id: Optional[str] = None,
         ignore_shadow_ban: bool = False,
+        outlier: bool = False,
+        historical: bool = False,
+        depth: Optional[int] = None,
     ) -> Tuple[EventBase, int]:
         """
         Creates an event, then sends it.
@@ -710,10 +799,29 @@ async def create_and_send_nonmember_event(
         Args:
             requester: The requester sending the event.
             event_dict: An entire event.
+            prev_event_ids:
+                The event IDs to use as the prev events.
+                Should normally be left as None to automatically request them
+                from the database.
+            auth_event_ids:
+                The event ids to use as the auth_events for the new event.
+                Should normally be left as None, which will cause them to be calculated
+                based on the room state at the prev_events.
+
+                If non-None, prev_event_ids must also be provided.
             ratelimit: Whether to rate limit this send.
             txn_id: The transaction ID.
             ignore_shadow_ban: True if shadow-banned users should be allowed to
                 send this event.
+            outlier: Indicates whether the event is an `outlier`, i.e. if
+                it's from an arbitrary point and floating in the DAG as
+                opposed to being inline with the current DAG.
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
+            depth: Override the depth used to order the event in the DAG.
+                Should normally be set to None, which will cause the depth to be calculated
+                based on the prev_events.
 
         Returns:
             The event, and its stream ordering (if deduplication happened,
@@ -753,7 +861,14 @@ async def create_and_send_nonmember_event(
                     return event, event.internal_metadata.stream_ordering
 
             event, context = await self.create_event(
-                requester, event_dict, txn_id=txn_id
+                requester,
+                event_dict,
+                txn_id=txn_id,
+                prev_event_ids=prev_event_ids,
+                auth_event_ids=auth_event_ids,
+                outlier=outlier,
+                historical=historical,
+                depth=depth,
             )
 
             assert self.hs.is_mine_id(event.sender), "User must be our own: %s" % (
@@ -785,6 +900,7 @@ async def create_new_client_event(
         requester: Optional[Requester] = None,
         prev_event_ids: Optional[List[str]] = None,
         auth_event_ids: Optional[List[str]] = None,
+        depth: Optional[int] = None,
     ) -> Tuple[EventBase, EventContext]:
         """Create a new event for a local client
 
@@ -802,6 +918,10 @@ async def create_new_client_event(
                 Should normally be left as None, which will cause them to be calculated
                 based on the room state at the prev_events.
 
+            depth: Override the depth used to order the event in the DAG.
+                Should normally be set to None, which will cause the depth to be calculated
+                based on the prev_events.
+
         Returns:
             Tuple of created event, context
         """
@@ -825,16 +945,31 @@ async def create_new_client_event(
         ), "Attempting to create an event with no prev_events"
 
         event = await builder.build(
-            prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids
+            prev_event_ids=prev_event_ids,
+            auth_event_ids=auth_event_ids,
+            depth=depth,
         )
-        context = await self.state.compute_event_context(event)
+
+        old_state = None
+
+        # Pass on the outlier property from the builder to the event
+        # after it is created
+        if builder.internal_metadata.outlier:
+            event.internal_metadata.outlier = builder.internal_metadata.outlier
+
+            # Calculate the state for outliers that pass in their own `auth_event_ids`
+            if auth_event_ids:
+                old_state = await self.store.get_events_as_list(auth_event_ids)
+
+        context = await self.state.compute_event_context(event, old_state=old_state)
+
         if requester:
             context.app_service = requester.app_service
 
-        third_party_result = await self.third_party_event_rules.check_event_allowed(
+        res, new_content = await self.third_party_event_rules.check_event_allowed(
             event, context
         )
-        if not third_party_result:
+        if res is False:
             logger.info(
                 "Event %s forbidden by third-party rules",
                 event,
@@ -842,11 +977,11 @@ async def create_new_client_event(
             raise SynapseError(
                 403, "This event is not allowed in this context", Codes.FORBIDDEN
             )
-        elif isinstance(third_party_result, dict):
+        elif new_content is not None:
             # the third-party rules want to replace the event. We'll need to build a new
             # event.
             event, context = await self._rebuild_event_after_third_party_rules(
-                third_party_result, event
+                new_content, event
             )
 
         self.validator.validate_new(event, self.config)
@@ -876,7 +1011,7 @@ async def handle_new_client_event(
         event: EventBase,
         context: EventContext,
         ratelimit: bool = True,
-        extra_users: List[UserID] = [],
+        extra_users: Optional[List[UserID]] = None,
         ignore_shadow_ban: bool = False,
     ) -> EventBase:
         """Processes a new event.
@@ -904,6 +1039,7 @@ async def handle_new_client_event(
         Raises:
             ShadowBanError if the requester has been shadow-banned.
         """
+        extra_users = extra_users or []
 
         # we don't apply shadow-banning to membership events here. Invites are blocked
         # higher up the stack, and we allow shadow-banned users to send join and leave
@@ -942,7 +1078,9 @@ async def handle_new_client_event(
             assert event.content["membership"] == Membership.LEAVE
         else:
             try:
-                await self.auth.check_from_context(room_version, event, context)
+                await self._event_auth_handler.check_from_context(
+                    room_version, event, context
+                )
             except AuthError as err:
                 logger.warning("Denying new event %r because %s", event, err)
                 raise err
@@ -955,9 +1093,49 @@ async def handle_new_client_event(
             logger.exception("Failed to encode content: %r", event.content)
             raise
 
-        await self.action_generator.handle_push_actions_for_event(event, context)
+        # We now persist the event (and update the cache in parallel, since we
+        # don't want to block on it).
+        result = await make_deferred_yieldable(
+            defer.gatherResults(
+                [
+                    run_in_background(
+                        self._persist_event,
+                        requester=requester,
+                        event=event,
+                        context=context,
+                        ratelimit=ratelimit,
+                        extra_users=extra_users,
+                    ),
+                    run_in_background(
+                        self.cache_joined_hosts_for_event, event, context
+                    ).addErrback(log_failure, "cache_joined_hosts_for_event failed"),
+                ],
+                consumeErrors=True,
+            )
+        ).addErrback(unwrapFirstError)
 
-        await self.cache_joined_hosts_for_event(event)
+        return result[0]
+
+    async def _persist_event(
+        self,
+        requester: Requester,
+        event: EventBase,
+        context: EventContext,
+        ratelimit: bool = True,
+        extra_users: Optional[List[UserID]] = None,
+    ) -> EventBase:
+        """Actually persists the event. Should only be called by
+        `handle_new_client_event`, and see its docstring for documentation of
+        the arguments.
+        """
+
+        # Skip push notification actions for historical messages
+        # because we don't want to notify people about old history back in time.
+        # The historical messages also do not have the proper `context.current_state_ids`
+        # and `state_groups` because they have `prev_events` that aren't persisted yet
+        # (historical messages persisted in reverse-chronological order).
+        if not event.internal_metadata.is_historical():
+            await self.action_generator.handle_push_actions_for_event(event, context)
 
         try:
             # If we're a worker we need to hit out to the master.
@@ -998,7 +1176,9 @@ async def handle_new_client_event(
             await self.store.remove_push_actions_from_staging(event.event_id)
             raise
 
-    async def cache_joined_hosts_for_event(self, event: EventBase) -> None:
+    async def cache_joined_hosts_for_event(
+        self, event: EventBase, context: EventContext
+    ) -> None:
         """Precalculate the joined hosts at the event, when using Redis, so that
         external federation senders don't have to recalculate it themselves.
         """
@@ -1006,6 +1186,9 @@ async def cache_joined_hosts_for_event(self, event: EventBase) -> None:
         if not self._external_cache.is_enabled():
             return
 
+        # If external cache is enabled we should always have this.
+        assert self._external_cache_joined_hosts_updates is not None
+
         # We actually store two mappings, event ID -> prev state group,
         # state group -> joined hosts, which is much more space efficient
         # than event ID -> joined hosts.
@@ -1013,22 +1196,28 @@ async def cache_joined_hosts_for_event(self, event: EventBase) -> None:
         # Note: We have to cache event ID -> prev state group, as we don't
         # store that in the DB.
         #
-        # Note: We always set the state group -> joined hosts cache, even if
-        # we already set it, so that the expiry time is reset.
+        # Note: We set the state group -> joined hosts cache if it hasn't been
+        # set for a while, so that the expiry time is reset.
 
         state_entry = await self.state.resolve_state_groups_for_events(
             event.room_id, event_ids=event.prev_event_ids()
         )
 
         if state_entry.state_group:
-            joined_hosts = await self.store.get_joined_hosts(event.room_id, state_entry)
-
             await self._external_cache.set(
                 "event_to_prev_state_group",
                 event.event_id,
                 state_entry.state_group,
                 expiry_ms=60 * 60 * 1000,
             )
+
+            if state_entry.state_group in self._external_cache_joined_hosts_updates:
+                return
+
+            joined_hosts = await self.store.get_joined_hosts(event.room_id, state_entry)
+
+            # Note that the expiry times must be larger than the expiry time in
+            # _external_cache_joined_hosts_updates.
             await self._external_cache.set(
                 "get_joined_hosts",
                 str(state_entry.state_group),
@@ -1036,6 +1225,8 @@ async def cache_joined_hosts_for_event(self, event: EventBase) -> None:
                 expiry_ms=60 * 60 * 1000,
             )
 
+            self._external_cache_joined_hosts_updates[state_entry.state_group] = None
+
     async def _validate_canonical_alias(
         self, directory_handler, room_alias_str: str, expected_room_id: str
     ) -> None:
@@ -1073,7 +1264,7 @@ async def persist_and_notify_client_event(
         event: EventBase,
         context: EventContext,
         ratelimit: bool = True,
-        extra_users: List[UserID] = [],
+        extra_users: Optional[List[UserID]] = None,
     ) -> EventBase:
         """Called when we have fully built the event, have already
         calculated the push actions for the event, and checked auth.
@@ -1085,6 +1276,8 @@ async def persist_and_notify_client_event(
             it was de-duplicated (e.g. because we had already persisted an
             event with the same transaction ID.)
         """
+        extra_users = extra_users or []
+
         assert self.storage.persistence is not None
         assert self._events_shard_config.should_handle(
             self._instance_name, event.room_id
@@ -1119,7 +1312,7 @@ async def persist_and_notify_client_event(
             # Validate a newly added alias or newly added alt_aliases.
 
             original_alias = None
-            original_alt_aliases = []  # type: List[str]
+            original_alt_aliases: List[str] = []
 
             original_event_id = event.unsigned.get("replaces_state")
             if original_event_id:
@@ -1165,7 +1358,7 @@ async def persist_and_notify_client_event(
                     "invite_room_state"
                 ] = await self.store.get_stripped_room_state_from_event_context(
                     context,
-                    self.room_invite_state_types,
+                    self.room_prejoin_state_types,
                     membership_user_id=event.sender,
                 )
 
@@ -1188,7 +1381,7 @@ async def persist_and_notify_client_event(
                     "knock_room_state"
                 ] = await self.store.get_stripped_room_state_from_event_context(
                     context,
-                    DEFAULT_ROOM_STATE_TYPES,
+                    self.room_prejoin_state_types,
                 )
 
         if event.type == EventTypes.Redaction:
@@ -1212,7 +1405,7 @@ async def persist_and_notify_client_event(
                     raise AuthError(403, "Redacting server ACL events is not permitted")
 
             prev_state_ids = await context.get_prev_state_ids()
-            auth_events_ids = self.auth.compute_auth_events(
+            auth_events_ids = self._event_auth_handler.compute_auth_events(
                 event, prev_state_ids, for_verification=True
             )
             auth_events_map = await self.store.get_events(auth_events_ids)
@@ -1241,13 +1434,21 @@ async def persist_and_notify_client_event(
             if prev_state_ids:
                 raise AuthError(403, "Changing the room create event is forbidden")
 
+        # Mark any `m.historical` messages as backfilled so they don't appear
+        # in `/sync` and have the proper decrementing `stream_ordering` as we import
+        backfilled = False
+        if event.internal_metadata.is_historical():
+            backfilled = True
+
         # Note that this returns the event that was persisted, which may not be
         # the same as we passed in if it was deduplicated due transaction IDs.
         (
             event,
             event_pos,
             max_stream_token,
-        ) = await self.storage.persistence.persist_event(event, context=context)
+        ) = await self.storage.persistence.persist_event(
+            event, context=context, backfilled=backfilled
+        )
 
         if self._ephemeral_events_enabled:
             # If there's an expiry timestamp on the event, schedule its expiry.
diff --git a/synapse/handlers/oidc_handler.py b/synapse/handlers/oidc.py
similarity index 96%
rename from synapse/handlers/oidc_handler.py
rename to synapse/handlers/oidc.py
index 6624212d6f..eca8f16040 100644
--- a/synapse/handlers/oidc_handler.py
+++ b/synapse/handlers/oidc.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Quentin Gliech
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
@@ -16,7 +15,7 @@
 import inspect
 import logging
 from typing import TYPE_CHECKING, Dict, Generic, List, Optional, TypeVar, Union
-from urllib.parse import urlencode
+from urllib.parse import urlencode, urlparse
 
 import attr
 import pymacaroons
@@ -38,10 +37,7 @@
 from twisted.web.http_headers import Headers
 
 from synapse.config import ConfigError
-from synapse.config.oidc_config import (
-    OidcProviderClientSecretJwtKey,
-    OidcProviderConfig,
-)
+from synapse.config.oidc import OidcProviderClientSecretJwtKey, OidcProviderConfig
 from synapse.handlers.sso import MappingException, UserAttributes
 from synapse.http.site import SynapseRequest
 from synapse.logging.context import make_deferred_yieldable
@@ -72,30 +68,30 @@
 #
 # Here we have the names of the cookies, and the options we use to set them.
 _SESSION_COOKIES = [
-    (b"oidc_session", b"Path=/_synapse/client/oidc; HttpOnly; Secure; SameSite=None"),
-    (b"oidc_session_no_samesite", b"Path=/_synapse/client/oidc; HttpOnly"),
+    (b"oidc_session", b"HttpOnly; Secure; SameSite=None"),
+    (b"oidc_session_no_samesite", b"HttpOnly"),
 ]
 
+
 #: A token exchanged from the token endpoint, as per RFC6749 sec 5.1. and
 #: OpenID.Core sec 3.1.3.3.
-Token = TypedDict(
-    "Token",
-    {
-        "access_token": str,
-        "token_type": str,
-        "id_token": Optional[str],
-        "refresh_token": Optional[str],
-        "expires_in": int,
-        "scope": Optional[str],
-    },
-)
+class Token(TypedDict):
+    access_token: str
+    token_type: str
+    id_token: Optional[str]
+    refresh_token: Optional[str]
+    expires_in: int
+    scope: Optional[str]
+
 
 #: A JWK, as per RFC7517 sec 4. The type could be more precise than that, but
 #: there is no real point of doing this in our case.
 JWK = Dict[str, str]
 
+
 #: A JWK Set, as per RFC7517 sec 5.
-JWKS = TypedDict("JWKS", {"keys": List[JWK]})
+class JWKS(TypedDict):
+    keys: List[JWK]
 
 
 class OidcHandler:
@@ -109,9 +105,9 @@ def __init__(self, hs: "HomeServer"):
         assert provider_confs
 
         self._token_generator = OidcSessionTokenGenerator(hs)
-        self._providers = {
+        self._providers: Dict[str, "OidcProvider"] = {
             p.idp_id: OidcProvider(hs, self._token_generator, p) for p in provider_confs
-        }  # type: Dict[str, OidcProvider]
+        }
 
     async def load_metadata(self) -> None:
         """Validate the config and load the metadata from the remote endpoint.
@@ -182,7 +178,7 @@ async def handle_oidc_callback(self, request: SynapseRequest) -> None:
         # are two.
 
         for cookie_name, _ in _SESSION_COOKIES:
-            session = request.getCookie(cookie_name)  # type: Optional[bytes]
+            session: Optional[bytes] = request.getCookie(cookie_name)
             if session is not None:
                 break
         else:
@@ -259,7 +255,7 @@ def __init__(self, error, error_description=None):
 
     def __str__(self):
         if self.error_description:
-            return "{}: {}".format(self.error, self.error_description)
+            return f"{self.error}: {self.error_description}"
         return self.error
 
 
@@ -281,13 +277,20 @@ def __init__(
         self._token_generator = token_generator
 
         self._config = provider
-        self._callback_url = hs.config.oidc_callback_url  # type: str
+        self._callback_url: str = hs.config.oidc_callback_url
+
+        # Calculate the prefix for OIDC callback paths based on the public_baseurl.
+        # We'll insert this into the Path= parameter of any session cookies we set.
+        public_baseurl_path = urlparse(hs.config.server.public_baseurl).path
+        self._callback_path_prefix = (
+            public_baseurl_path.encode("utf-8") + b"_synapse/client/oidc"
+        )
 
         self._oidc_attribute_requirements = provider.attribute_requirements
         self._scopes = provider.scopes
         self._user_profile_method = provider.user_profile_method
 
-        client_secret = None  # type: Union[None, str, JwtClientSecret]
+        client_secret: Optional[Union[str, JwtClientSecret]] = None
         if provider.client_secret:
             client_secret = provider.client_secret
         elif provider.client_secret_jwt_key:
@@ -302,7 +305,7 @@ def __init__(
             provider.client_id,
             client_secret,
             provider.client_auth_method,
-        )  # type: ClientAuth
+        )
         self._client_auth_method = provider.client_auth_method
 
         # cache of metadata for the identity provider (endpoint uris, mostly). This is
@@ -321,7 +324,7 @@ def __init__(
         self._allow_existing_users = provider.allow_existing_users
 
         self._http_client = hs.get_proxied_http_client()
-        self._server_name = hs.config.server_name  # type: str
+        self._server_name: str = hs.config.server_name
 
         # identifier for the external_ids table
         self.idp_id = provider.idp_id
@@ -636,7 +639,7 @@ async def _exchange_code(self, code: str) -> Token:
             )
             logger.warning(description)
             # Body was still valid JSON. Might be useful to log it for debugging.
-            logger.warning("Code exchange response: {resp!r}".format(resp=resp))
+            logger.warning("Code exchange response: %r", resp)
             raise OidcError("server_error", description)
 
         return resp
@@ -783,8 +786,13 @@ async def handle_redirect_request(
 
         for cookie_name, options in _SESSION_COOKIES:
             request.cookies.append(
-                b"%s=%s; Max-Age=3600; %s"
-                % (cookie_name, cookie.encode("utf-8"), options)
+                b"%s=%s; Max-Age=3600; Path=%s; %s"
+                % (
+                    cookie_name,
+                    cookie.encode("utf-8"),
+                    self._callback_path_prefix,
+                    options,
+                )
             )
 
         metadata = await self.load_metadata()
@@ -961,6 +969,11 @@ async def grandfather_existing_users() -> Optional[str]:
                 # and attempt to match it.
                 attributes = await oidc_response_to_user_attributes(failures=0)
 
+                if attributes.localpart is None:
+                    # If no localpart is returned then we will generate one, so
+                    # there is no need to search for existing users.
+                    return None
+
                 user_id = UserID(attributes.localpart, self._server_name).to_string()
                 users = await self._store.get_users_by_id_case_insensitive(user_id)
                 if users:
@@ -1204,10 +1217,12 @@ class OidcSessionData:
     ui_auth_session_id = attr.ib(type=str)
 
 
-UserAttributeDict = TypedDict(
-    "UserAttributeDict",
-    {"localpart": Optional[str], "display_name": Optional[str], "emails": List[str]},
-)
+class UserAttributeDict(TypedDict):
+    localpart: Optional[str]
+    display_name: Optional[str]
+    emails: List[str]
+
+
 C = TypeVar("C")
 
 
@@ -1368,7 +1383,7 @@ def render_template_field(template: Optional[Template]) -> Optional[str]:
         if display_name == "":
             display_name = None
 
-        emails = []  # type: List[str]
+        emails: List[str] = []
         email = render_template_field(self._config.email_template)
         if email:
             emails.append(email)
@@ -1378,7 +1393,7 @@ def render_template_field(template: Optional[Template]) -> Optional[str]:
         )
 
     async def get_extra_attributes(self, userinfo: UserInfo, token: Token) -> JsonDict:
-        extras = {}  # type: Dict[str, str]
+        extras: Dict[str, str] = {}
         for key, template in self._config.extra_attributes.items():
             try:
                 extras[key] = template.render(user=userinfo).strip()
diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py
index 66dc886c81..1dbafd253d 100644
--- a/synapse/handlers/pagination.py
+++ b/synapse/handlers/pagination.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 # Copyright 2017 - 2018 New Vector Ltd
 #
@@ -82,9 +81,9 @@ def __init__(self, hs: "HomeServer"):
         self._server_name = hs.hostname
 
         self.pagination_lock = ReadWriteLock()
-        self._purges_in_progress_by_room = set()  # type: Set[str]
+        self._purges_in_progress_by_room: Set[str] = set()
         # map from purge id to PurgeStatus
-        self._purges_by_id = {}  # type: Dict[str, PurgeStatus]
+        self._purges_by_id: Dict[str, PurgeStatus] = {}
         self._event_serializer = hs.get_event_client_serializer()
 
         self._retention_default_max_lifetime = hs.config.retention_default_max_lifetime
diff --git a/synapse/handlers/password_policy.py b/synapse/handlers/password_policy.py
index 92cefa11aa..cd21efdcc6 100644
--- a/synapse/handlers/password_policy.py
+++ b/synapse/handlers/password_policy.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index da92feacc9..7ca14e1d84 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -23,9 +22,23 @@
     - should_notify
 """
 import abc
+import contextlib
 import logging
+from bisect import bisect
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Dict, Iterable, List, Set, Tuple
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Collection,
+    Dict,
+    FrozenSet,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
 
 from prometheus_client import Counter
 from typing_extensions import ContextManager
@@ -34,15 +47,22 @@
 from synapse.api.constants import EventTypes, Membership, PresenceState
 from synapse.api.errors import SynapseError
 from synapse.api.presence import UserPresenceState
+from synapse.events.presence_router import PresenceRouter
 from synapse.logging.context import run_in_background
 from synapse.logging.utils import log_function
 from synapse.metrics import LaterGauge
 from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.state import StateHandler
+from synapse.replication.http.presence import (
+    ReplicationBumpPresenceActiveTime,
+    ReplicationPresenceSetState,
+)
+from synapse.replication.http.streams import ReplicationGetStreamUpdates
+from synapse.replication.tcp.commands import ClearUserSyncsCommand
+from synapse.replication.tcp.streams import PresenceFederationStream, PresenceStream
 from synapse.storage.databases.main import DataStore
-from synapse.types import Collection, JsonDict, UserID, get_domain_from_id
+from synapse.types import JsonDict, UserID, get_domain_from_id
 from synapse.util.async_helpers import Linearizer
-from synapse.util.caches.descriptors import cached
+from synapse.util.caches.descriptors import _CacheContext, cached
 from synapse.util.metrics import Measure
 from synapse.util.wheel_timer import WheelTimer
 
@@ -94,15 +114,29 @@
 # are dead.
 EXTERNAL_PROCESS_EXPIRY = 5 * 60 * 1000
 
+# Delay before a worker tells the presence handler that a user has stopped
+# syncing.
+UPDATE_SYNCING_USERS_MS = 10 * 1000
+
 assert LAST_ACTIVE_GRANULARITY < IDLE_TIMER
 
 
 class BasePresenceHandler(abc.ABC):
-    """Parts of the PresenceHandler that are shared between workers and master"""
+    """Parts of the PresenceHandler that are shared between workers and presence
+    writer"""
 
     def __init__(self, hs: "HomeServer"):
         self.clock = hs.get_clock()
         self.store = hs.get_datastore()
+        self.presence_router = hs.get_presence_router()
+        self.state = hs.get_state_handler()
+        self.is_mine_id = hs.is_mine_id
+
+        self._federation = None
+        if hs.should_send_federation():
+            self._federation = hs.get_federation_sender()
+
+        self._federation_queue = PresenceFederationQueue(hs, self)
 
         self._busy_presence_enabled = hs.config.experimental.msc3026_enabled
 
@@ -188,9 +222,21 @@ async def current_state_for_users(
 
     @abc.abstractmethod
     async def set_state(
-        self, target_user: UserID, state: JsonDict, ignore_status_msg: bool = False
+        self,
+        target_user: UserID,
+        state: JsonDict,
+        ignore_status_msg: bool = False,
+        force_notify: bool = False,
     ) -> None:
-        """Set the presence state of the user. """
+        """Set the presence state of the user.
+
+        Args:
+            target_user: The ID of the user to set the presence state of.
+            state: The presence state as a JSON dictionary.
+            ignore_status_msg: True to ignore the "status_msg" field of the `state` dict.
+                If False, the user's current status will be updated.
+            force_notify: Whether to force notification of the update to clients.
+        """
 
     @abc.abstractmethod
     async def bump_presence_active_time(self, user: UserID):
@@ -198,17 +244,374 @@ async def bump_presence_active_time(self, user: UserID):
         with the app.
         """
 
+    async def update_external_syncs_row(
+        self, process_id: str, user_id: str, is_syncing: bool, sync_time_msec: int
+    ) -> None:
+        """Update the syncing users for an external process as a delta.
+
+        This is a no-op when presence is handled by a different worker.
+
+        Args:
+            process_id: An identifier for the process the users are
+                syncing against. This allows synapse to process updates
+                as user start and stop syncing against a given process.
+            user_id: The user who has started or stopped syncing
+            is_syncing: Whether or not the user is now syncing
+            sync_time_msec: Time in ms when the user was last syncing
+        """
+        pass
+
+    async def update_external_syncs_clear(self, process_id: str) -> None:
+        """Marks all users that had been marked as syncing by a given process
+        as offline.
+
+        Used when the process has stopped/disappeared.
+
+        This is a no-op when presence is handled by a different worker.
+        """
+        pass
+
+    async def process_replication_rows(
+        self, stream_name: str, instance_name: str, token: int, rows: list
+    ):
+        """Process streams received over replication."""
+        await self._federation_queue.process_replication_rows(
+            stream_name, instance_name, token, rows
+        )
+
+    def get_federation_queue(self) -> "PresenceFederationQueue":
+        """Get the presence federation queue."""
+        return self._federation_queue
+
+    async def maybe_send_presence_to_interested_destinations(
+        self, states: List[UserPresenceState]
+    ):
+        """If this instance is a federation sender, send the states to all
+        destinations that are interested. Filters out any states for remote
+        users.
+        """
+
+        if not self._federation:
+            return
+
+        states = [s for s in states if self.is_mine_id(s.user_id)]
+
+        if not states:
+            return
+
+        hosts_to_states = await get_interested_remotes(
+            self.store,
+            self.presence_router,
+            states,
+        )
+
+        for destination, host_states in hosts_to_states.items():
+            self._federation.send_presence_to_destinations(host_states, [destination])
+
+    async def send_full_presence_to_users(self, user_ids: Collection[str]):
+        """
+        Adds to the list of users who should receive a full snapshot of presence
+        upon their next sync. Note that this only works for local users.
+
+        Then, grabs the current presence state for a given set of users and adds it
+        to the top of the presence stream.
+
+        Args:
+            user_ids: The IDs of the local users to send full presence to.
+        """
+        # Retrieve one of the users from the given set
+        if not user_ids:
+            raise Exception(
+                "send_full_presence_to_users must be called with at least one user"
+            )
+        user_id = next(iter(user_ids))
+
+        # Mark all users as receiving full presence on their next sync
+        await self.store.add_users_to_send_full_presence_to(user_ids)
+
+        # Add a new entry to the presence stream. Since we use stream tokens to determine whether a
+        # local user should receive a full snapshot of presence when they sync, we need to bump the
+        # presence stream so that subsequent syncs with no presence activity in between won't result
+        # in the client receiving multiple full snapshots of presence.
+        #
+        # If we bump the stream ID, then the user will get a higher stream token next sync, and thus
+        # correctly won't receive a second snapshot.
+
+        # Get the current presence state for one of the users (defaults to offline if not found)
+        current_presence_state = await self.get_state(UserID.from_string(user_id))
+
+        # Convert the UserPresenceState object into a serializable dict
+        state = {
+            "presence": current_presence_state.state,
+            "status_message": current_presence_state.status_msg,
+        }
+
+        # Copy the presence state to the tip of the presence stream.
+
+        # We set force_notify=True here so that this presence update is guaranteed to
+        # increment the presence stream ID (which resending the current user's presence
+        # otherwise would not do).
+        await self.set_state(UserID.from_string(user_id), state, force_notify=True)
+
+
+class _NullContextManager(ContextManager[None]):
+    """A context manager which does nothing."""
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+
+class WorkerPresenceHandler(BasePresenceHandler):
+    def __init__(self, hs: "HomeServer"):
+        super().__init__(hs)
+        self.hs = hs
+
+        self._presence_writer_instance = hs.config.worker.writers.presence[0]
+
+        self._presence_enabled = hs.config.use_presence
+
+        # Route presence EDUs to the right worker
+        hs.get_federation_registry().register_instances_for_edu(
+            "m.presence",
+            hs.config.worker.writers.presence,
+        )
+
+        # The number of ongoing syncs on this process, by user id.
+        # Empty if _presence_enabled is false.
+        self._user_to_num_current_syncs: Dict[str, int] = {}
+
+        self.notifier = hs.get_notifier()
+        self.instance_id = hs.get_instance_id()
+
+        # user_id -> last_sync_ms. Lists the users that have stopped syncing but
+        # we haven't notified the presence writer of that yet
+        self.users_going_offline: Dict[str, int] = {}
+
+        self._bump_active_client = ReplicationBumpPresenceActiveTime.make_client(hs)
+        self._set_state_client = ReplicationPresenceSetState.make_client(hs)
+
+        self._send_stop_syncing_loop = self.clock.looping_call(
+            self.send_stop_syncing, UPDATE_SYNCING_USERS_MS
+        )
+
+        self._busy_presence_enabled = hs.config.experimental.msc3026_enabled
+
+        hs.get_reactor().addSystemEventTrigger(
+            "before",
+            "shutdown",
+            run_as_background_process,
+            "generic_presence.on_shutdown",
+            self._on_shutdown,
+        )
+
+    def _on_shutdown(self) -> None:
+        if self._presence_enabled:
+            self.hs.get_tcp_replication().send_command(
+                ClearUserSyncsCommand(self.instance_id)
+            )
+
+    def send_user_sync(self, user_id: str, is_syncing: bool, last_sync_ms: int) -> None:
+        if self._presence_enabled:
+            self.hs.get_tcp_replication().send_user_sync(
+                self.instance_id, user_id, is_syncing, last_sync_ms
+            )
+
+    def mark_as_coming_online(self, user_id: str) -> None:
+        """A user has started syncing. Send a UserSync to the presence writer,
+        unless they had recently stopped syncing.
+        """
+        going_offline = self.users_going_offline.pop(user_id, None)
+        if not going_offline:
+            # Safe to skip because we haven't yet told the presence writer they
+            # were offline
+            self.send_user_sync(user_id, True, self.clock.time_msec())
+
+    def mark_as_going_offline(self, user_id: str) -> None:
+        """A user has stopped syncing. We wait before notifying the presence
+        writer as its likely they'll come back soon. This allows us to avoid
+        sending a stopped syncing immediately followed by a started syncing
+        notification to the presence writer
+        """
+        self.users_going_offline[user_id] = self.clock.time_msec()
+
+    def send_stop_syncing(self) -> None:
+        """Check if there are any users who have stopped syncing a while ago and
+        haven't come back yet. If there are poke the presence writer about them.
+        """
+        now = self.clock.time_msec()
+        for user_id, last_sync_ms in list(self.users_going_offline.items()):
+            if now - last_sync_ms > UPDATE_SYNCING_USERS_MS:
+                self.users_going_offline.pop(user_id, None)
+                self.send_user_sync(user_id, False, last_sync_ms)
+
+    async def user_syncing(
+        self, user_id: str, affect_presence: bool
+    ) -> ContextManager[None]:
+        """Record that a user is syncing.
+
+        Called by the sync and events servlets to record that a user has connected to
+        this worker and is waiting for some events.
+        """
+        if not affect_presence or not self._presence_enabled:
+            return _NullContextManager()
+
+        curr_sync = self._user_to_num_current_syncs.get(user_id, 0)
+        self._user_to_num_current_syncs[user_id] = curr_sync + 1
+
+        # If we went from no in flight sync to some, notify replication
+        if self._user_to_num_current_syncs[user_id] == 1:
+            self.mark_as_coming_online(user_id)
+
+        def _end():
+            # We check that the user_id is in user_to_num_current_syncs because
+            # user_to_num_current_syncs may have been cleared if we are
+            # shutting down.
+            if user_id in self._user_to_num_current_syncs:
+                self._user_to_num_current_syncs[user_id] -= 1
+
+                # If we went from one in flight sync to non, notify replication
+                if self._user_to_num_current_syncs[user_id] == 0:
+                    self.mark_as_going_offline(user_id)
+
+        @contextlib.contextmanager
+        def _user_syncing():
+            try:
+                yield
+            finally:
+                _end()
+
+        return _user_syncing()
+
+    async def notify_from_replication(
+        self, states: List[UserPresenceState], stream_id: int
+    ) -> None:
+        parties = await get_interested_parties(self.store, self.presence_router, states)
+        room_ids_to_states, users_to_states = parties
+
+        self.notifier.on_new_event(
+            "presence_key",
+            stream_id,
+            rooms=room_ids_to_states.keys(),
+            users=users_to_states.keys(),
+        )
+
+    async def process_replication_rows(
+        self, stream_name: str, instance_name: str, token: int, rows: list
+    ):
+        await super().process_replication_rows(stream_name, instance_name, token, rows)
+
+        if stream_name != PresenceStream.NAME:
+            return
+
+        states = [
+            UserPresenceState(
+                row.user_id,
+                row.state,
+                row.last_active_ts,
+                row.last_federation_update_ts,
+                row.last_user_sync_ts,
+                row.status_msg,
+                row.currently_active,
+            )
+            for row in rows
+        ]
+
+        # The list of states to notify sync streams and remote servers about.
+        # This is calculated by comparing the old and new states for each user
+        # using `should_notify(..)`.
+        #
+        # Note that this is necessary as the presence writer will periodically
+        # flush presence state changes that should not be notified about to the
+        # DB, and so will be sent over the replication stream.
+        state_to_notify = []
+
+        for new_state in states:
+            old_state = self.user_to_current_state.get(new_state.user_id)
+            self.user_to_current_state[new_state.user_id] = new_state
+
+            if not old_state or should_notify(old_state, new_state):
+                state_to_notify.append(new_state)
+
+        stream_id = token
+        await self.notify_from_replication(state_to_notify, stream_id)
+
+        # If this is a federation sender, notify about presence updates.
+        await self.maybe_send_presence_to_interested_destinations(state_to_notify)
+
+    def get_currently_syncing_users_for_replication(self) -> Iterable[str]:
+        return [
+            user_id
+            for user_id, count in self._user_to_num_current_syncs.items()
+            if count > 0
+        ]
+
+    async def set_state(
+        self,
+        target_user: UserID,
+        state: JsonDict,
+        ignore_status_msg: bool = False,
+        force_notify: bool = False,
+    ) -> None:
+        """Set the presence state of the user.
+
+        Args:
+            target_user: The ID of the user to set the presence state of.
+            state: The presence state as a JSON dictionary.
+            ignore_status_msg: True to ignore the "status_msg" field of the `state` dict.
+                If False, the user's current status will be updated.
+            force_notify: Whether to force notification of the update to clients.
+        """
+        presence = state["presence"]
+
+        valid_presence = (
+            PresenceState.ONLINE,
+            PresenceState.UNAVAILABLE,
+            PresenceState.OFFLINE,
+            PresenceState.BUSY,
+        )
+
+        if presence not in valid_presence or (
+            presence == PresenceState.BUSY and not self._busy_presence_enabled
+        ):
+            raise SynapseError(400, "Invalid presence state")
+
+        user_id = target_user.to_string()
+
+        # If presence is disabled, no-op
+        if not self.hs.config.use_presence:
+            return
+
+        # Proxy request to instance that writes presence
+        await self._set_state_client(
+            instance_name=self._presence_writer_instance,
+            user_id=user_id,
+            state=state,
+            ignore_status_msg=ignore_status_msg,
+            force_notify=force_notify,
+        )
+
+    async def bump_presence_active_time(self, user: UserID) -> None:
+        """We've seen the user do something that indicates they're interacting
+        with the app.
+        """
+        # If presence is disabled, no-op
+        if not self.hs.config.use_presence:
+            return
+
+        # Proxy request to instance that writes presence
+        user_id = user.to_string()
+        await self._bump_active_client(
+            instance_name=self._presence_writer_instance, user_id=user_id
+        )
+
 
 class PresenceHandler(BasePresenceHandler):
     def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
         self.hs = hs
-        self.is_mine_id = hs.is_mine_id
         self.server_name = hs.hostname
         self.wheel_timer = WheelTimer()
         self.notifier = hs.get_notifier()
-        self.federation = hs.get_federation_sender()
-        self.state = hs.get_state_handler()
         self._presence_enabled = hs.config.use_presence
 
         federation_registry = hs.get_federation_registry()
@@ -247,7 +650,7 @@ def __init__(self, hs: "HomeServer"):
 
         # Set of users who have presence in the `user_to_current_state` that
         # have not yet been persisted
-        self.unpersisted_users_changes = set()  # type: Set[str]
+        self.unpersisted_users_changes: Set[str] = set()
 
         hs.get_reactor().addSystemEventTrigger(
             "before",
@@ -261,7 +664,7 @@ def __init__(self, hs: "HomeServer"):
 
         # Keeps track of the number of *ongoing* syncs on this process. While
         # this is non zero a user will never go offline.
-        self.user_to_num_current_syncs = {}  # type: Dict[str, int]
+        self.user_to_num_current_syncs: Dict[str, int] = {}
 
         # Keeps track of the number of *ongoing* syncs on other processes.
         # While any sync is ongoing on another process the user will never
@@ -271,8 +674,8 @@ def __init__(self, hs: "HomeServer"):
         # we assume that all the sync requests on that process have stopped.
         # Stored as a dict from process_id to set of user_id, and a dict of
         # process_id to millisecond timestamp last updated.
-        self.external_process_to_current_syncs = {}  # type: Dict[int, Set[str]]
-        self.external_process_last_updated_ms = {}  # type: Dict[int, int]
+        self.external_process_to_current_syncs: Dict[str, Set[str]] = {}
+        self.external_process_last_updated_ms: Dict[str, int] = {}
 
         self.external_sync_linearizer = Linearizer(name="external_sync_linearizer")
 
@@ -312,7 +715,7 @@ def run_persister():
         self._event_pos = self.store.get_current_events_token()
         self._event_processing = False
 
-    async def _on_shutdown(self):
+    async def _on_shutdown(self) -> None:
         """Gets called when shutting down. This lets us persist any updates that
         we haven't yet persisted, e.g. updates that only changes some internal
         timers. This allows changes to persist across startup without having to
@@ -341,7 +744,7 @@ async def _on_shutdown(self):
             )
         logger.info("Finished _on_shutdown")
 
-    async def _persist_unpersisted_changes(self):
+    async def _persist_unpersisted_changes(self) -> None:
         """We periodically persist the unpersisted changes, as otherwise they
         may stack up and slow down shutdown times.
         """
@@ -354,13 +757,19 @@ async def _persist_unpersisted_changes(self):
                 [self.user_to_current_state[user_id] for user_id in unpersisted]
             )
 
-    async def _update_states(self, new_states: Iterable[UserPresenceState]) -> None:
+    async def _update_states(
+        self, new_states: Iterable[UserPresenceState], force_notify: bool = False
+    ) -> None:
         """Updates presence of users. Sets the appropriate timeouts. Pokes
         the notifier and federation if and only if the changed presence state
         should be sent to clients/servers.
 
         Args:
             new_states: The new user presence state updates to process.
+            force_notify: Whether to force notifying clients of this presence state update,
+                even if it doesn't change the state of a user's presence (e.g online -> online).
+                This is currently used to bump the max presence stream ID without changing any
+                user's presence (see PresenceHandler.add_users_to_send_full_presence_to).
         """
         now = self.clock.time_msec()
 
@@ -397,6 +806,9 @@ async def _update_states(self, new_states: Iterable[UserPresenceState]) -> None:
                     now=now,
                 )
 
+                if force_notify:
+                    should_notify = True
+
                 self.user_to_current_state[user_id] = new_state
 
                 if should_notify:
@@ -415,6 +827,13 @@ async def _update_states(self, new_states: Iterable[UserPresenceState]) -> None:
             self.unpersisted_users_changes |= {s.user_id for s in new_states}
             self.unpersisted_users_changes -= set(to_notify.keys())
 
+            # Check if we need to resend any presence states to remote hosts. We
+            # only do this for states that haven't been updated in a while to
+            # ensure that the remote host doesn't time the presence state out.
+            #
+            # Note that since these are states that have *not* been updated,
+            # they won't get sent down the normal presence replication stream,
+            # and so we have to explicitly send them via the federation stream.
             to_federation_ping = {
                 user_id: state
                 for user_id, state in to_federation_ping.items()
@@ -423,9 +842,18 @@ async def _update_states(self, new_states: Iterable[UserPresenceState]) -> None:
             if to_federation_ping:
                 federation_presence_out_counter.inc(len(to_federation_ping))
 
-                self._push_to_remotes(to_federation_ping.values())
+                hosts_to_states = await get_interested_remotes(
+                    self.store,
+                    self.presence_router,
+                    list(to_federation_ping.values()),
+                )
+
+                for destination, states in hosts_to_states.items():
+                    self._federation_queue.send_presence_to_destinations(
+                        states, [destination]
+                    )
 
-    async def _handle_timeouts(self):
+    async def _handle_timeouts(self) -> None:
         """Checks the presence of users that have timed out and updates as
         appropriate.
         """
@@ -477,7 +905,7 @@ async def _handle_timeouts(self):
 
         return await self._update_states(changes)
 
-    async def bump_presence_active_time(self, user):
+    async def bump_presence_active_time(self, user: UserID) -> None:
         """We've seen the user do something that indicates they're interacting
         with the app.
         """
@@ -574,17 +1002,17 @@ def get_currently_syncing_users_for_replication(self) -> Iterable[str]:
         return []
 
     async def update_external_syncs_row(
-        self, process_id, user_id, is_syncing, sync_time_msec
-    ):
+        self, process_id: str, user_id: str, is_syncing: bool, sync_time_msec: int
+    ) -> None:
         """Update the syncing users for an external process as a delta.
 
         Args:
-            process_id (str): An identifier for the process the users are
+            process_id: An identifier for the process the users are
                 syncing against. This allows synapse to process updates
                 as user start and stop syncing against a given process.
-            user_id (str): The user who has started or stopped syncing
-            is_syncing (bool): Whether or not the user is now syncing
-            sync_time_msec(int): Time in ms when the user was last syncing
+            user_id: The user who has started or stopped syncing
+            is_syncing: Whether or not the user is now syncing
+            sync_time_msec: Time in ms when the user was last syncing
         """
         with (await self.external_sync_linearizer.queue(process_id)):
             prev_state = await self.current_state_for_user(user_id)
@@ -621,7 +1049,7 @@ async def update_external_syncs_row(
 
             self.external_process_last_updated_ms[process_id] = self.clock.time_msec()
 
-    async def update_external_syncs_clear(self, process_id):
+    async def update_external_syncs_clear(self, process_id: str) -> None:
         """Marks all users that had been marked as syncing by a given process
         as offline.
 
@@ -642,18 +1070,18 @@ async def update_external_syncs_clear(self, process_id):
             )
             self.external_process_last_updated_ms.pop(process_id, None)
 
-    async def current_state_for_user(self, user_id):
+    async def current_state_for_user(self, user_id: str) -> UserPresenceState:
         """Get the current presence state for a user."""
         res = await self.current_state_for_users([user_id])
         return res[user_id]
 
-    async def _persist_and_notify(self, states):
+    async def _persist_and_notify(self, states: List[UserPresenceState]) -> None:
         """Persist states in the database, poke the notifier and send to
         interested remote servers
         """
         stream_id, max_token = await self.store.update_presence(states)
 
-        parties = await get_interested_parties(self.store, states)
+        parties = await get_interested_parties(self.store, self.presence_router, states)
         room_ids_to_states, users_to_states = parties
 
         self.notifier.on_new_event(
@@ -663,17 +1091,12 @@ async def _persist_and_notify(self, states):
             users=[UserID.from_string(u) for u in users_to_states],
         )
 
-        self._push_to_remotes(states)
-
-    def _push_to_remotes(self, states):
-        """Sends state updates to remote servers.
+        # We only want to poke the local federation sender, if any, as other
+        # workers will receive the presence updates via the presence replication
+        # stream (which is updated by `store.update_presence`).
+        await self.maybe_send_presence_to_interested_destinations(states)
 
-        Args:
-            states (list(UserPresenceState))
-        """
-        self.federation.send_presence(states)
-
-    async def incoming_presence(self, origin, content):
+    async def incoming_presence(self, origin: str, content: JsonDict) -> None:
         """Called when we receive a `m.presence` EDU from a remote server."""
         if not self._presence_enabled:
             return
@@ -723,8 +1146,22 @@ async def incoming_presence(self, origin, content):
             federation_presence_counter.inc(len(updates))
             await self._update_states(updates)
 
-    async def set_state(self, target_user, state, ignore_status_msg=False):
-        """Set the presence state of the user."""
+    async def set_state(
+        self,
+        target_user: UserID,
+        state: JsonDict,
+        ignore_status_msg: bool = False,
+        force_notify: bool = False,
+    ) -> None:
+        """Set the presence state of the user.
+
+        Args:
+            target_user: The ID of the user to set the presence state of.
+            state: The presence state as a JSON dictionary.
+            ignore_status_msg: True to ignore the "status_msg" field of the `state` dict.
+                If False, the user's current status will be updated.
+            force_notify: Whether to force notification of the update to clients.
+        """
         status_msg = state.get("status_msg", None)
         presence = state["presence"]
 
@@ -747,17 +1184,18 @@ async def set_state(self, target_user, state, ignore_status_msg=False):
         new_fields = {"state": presence}
 
         if not ignore_status_msg:
-            msg = status_msg if presence != PresenceState.OFFLINE else None
-            new_fields["status_msg"] = msg
+            new_fields["status_msg"] = status_msg
 
         if presence == PresenceState.ONLINE or (
             presence == PresenceState.BUSY and self._busy_presence_enabled
         ):
             new_fields["last_active_ts"] = self.clock.time_msec()
 
-        await self._update_states([prev_state.copy_and_replace(**new_fields)])
+        await self._update_states(
+            [prev_state.copy_and_replace(**new_fields)], force_notify=force_notify
+        )
 
-    async def is_visible(self, observed_user, observer_user):
+    async def is_visible(self, observed_user: UserID, observer_user: UserID) -> bool:
         """Returns whether a user can see another user's presence."""
         observer_room_ids = await self.store.get_rooms_for_user(
             observer_user.to_string()
@@ -812,7 +1250,7 @@ async def get_all_presence_updates(
         )
         return rows
 
-    def notify_new_event(self):
+    def notify_new_event(self) -> None:
         """Called when new events have happened. Handles users and servers
         joining rooms and require being sent presence.
         """
@@ -831,7 +1269,7 @@ async def _process_presence():
 
         run_as_background_process("presence.notify_new_event", _process_presence)
 
-    async def _unsafe_process(self):
+    async def _unsafe_process(self) -> None:
         # Loop round handling deltas until we're up to date
         while True:
             with Measure(self.clock, "presence_delta"):
@@ -847,7 +1285,16 @@ async def _unsafe_process(self):
                 max_pos, deltas = await self.store.get_current_state_deltas(
                     self._event_pos, room_max_stream_ordering
                 )
-                await self._handle_state_delta(deltas)
+
+                # We may get multiple deltas for different rooms, but we want to
+                # handle them on a room by room basis, so we batch them up by
+                # room.
+                deltas_by_room: Dict[str, List[JsonDict]] = {}
+                for delta in deltas:
+                    deltas_by_room.setdefault(delta["room_id"], []).append(delta)
+
+                for room_id, deltas_for_room in deltas_by_room.items():
+                    await self._handle_state_delta(room_id, deltas_for_room)
 
                 self._event_pos = max_pos
 
@@ -856,17 +1303,21 @@ async def _unsafe_process(self):
                     max_pos
                 )
 
-    async def _handle_state_delta(self, deltas):
-        """Process current state deltas to find new joins that need to be
-        handled.
+    async def _handle_state_delta(self, room_id: str, deltas: List[JsonDict]) -> None:
+        """Process current state deltas for the room to find new joins that need
+        to be handled.
         """
-        # A map of destination to a set of user state that they should receive
-        presence_destinations = {}  # type: Dict[str, Set[UserPresenceState]]
+
+        # Sets of newly joined users. Note that if the local server is
+        # joining a remote room for the first time we'll see both the joining
+        # user and all remote users as newly joined.
+        newly_joined_users = set()
 
         for delta in deltas:
+            assert room_id == delta["room_id"]
+
             typ = delta["type"]
             state_key = delta["state_key"]
-            room_id = delta["room_id"]
             event_id = delta["event_id"]
             prev_event_id = delta["prev_event_id"]
 
@@ -895,72 +1346,55 @@ async def _handle_state_delta(self, deltas):
                     # Ignore changes to join events.
                     continue
 
-            # Retrieve any user presence state updates that need to be sent as a result,
-            # and the destinations that need to receive it
-            destinations, user_presence_states = await self._on_user_joined_room(
-                room_id, state_key
-            )
-
-            # Insert the destinations and respective updates into our destinations dict
-            for destination in destinations:
-                presence_destinations.setdefault(destination, set()).update(
-                    user_presence_states
-                )
-
-        # Send out user presence updates for each destination
-        for destination, user_state_set in presence_destinations.items():
-            self.federation.send_presence_to_destinations(
-                destinations=[destination], states=user_state_set
-            )
-
-    async def _on_user_joined_room(
-        self, room_id: str, user_id: str
-    ) -> Tuple[List[str], List[UserPresenceState]]:
-        """Called when we detect a user joining the room via the current state
-        delta stream. Returns the destinations that need to be updated and the
-        presence updates to send to them.
-
-        Args:
-            room_id: The ID of the room that the user has joined.
-            user_id: The ID of the user that has joined the room.
-
-        Returns:
-            A tuple of destinations and presence updates to send to them.
-        """
-        if self.is_mine_id(user_id):
-            # If this is a local user then we need to send their presence
-            # out to hosts in the room (who don't already have it)
-
-            # TODO: We should be able to filter the hosts down to those that
-            # haven't previously seen the user
-
-            remote_hosts = await self.state.get_current_hosts_in_room(room_id)
-
-            # Filter out ourselves.
-            filtered_remote_hosts = [
-                host for host in remote_hosts if host != self.server_name
-            ]
-
-            state = await self.current_state_for_user(user_id)
-            return filtered_remote_hosts, [state]
-        else:
-            # A remote user has joined the room, so we need to:
-            #   1. Check if this is a new server in the room
-            #   2. If so send any presence they don't already have for
-            #      local users in the room.
-
-            # TODO: We should be able to filter the users down to those that
-            # the server hasn't previously seen
+            newly_joined_users.add(state_key)
 
-            # TODO: Check that this is actually a new server joining the
-            # room.
-
-            remote_host = get_domain_from_id(user_id)
+        if not newly_joined_users:
+            # If nobody has joined then there's nothing to do.
+            return
 
-            users = await self.state.get_current_users_in_room(room_id)
-            user_ids = list(filter(self.is_mine_id, users))
+        # We want to send:
+        #   1. presence states of all local users in the room to newly joined
+        #      remote servers
+        #   2. presence states of newly joined users to all remote servers in
+        #      the room.
+        #
+        # TODO: Only send presence states to remote hosts that don't already
+        # have them (because they already share rooms).
+
+        # Get all the users who were already in the room, by fetching the
+        # current users in the room and removing the newly joined users.
+        users = await self.store.get_users_in_room(room_id)
+        prev_users = set(users) - newly_joined_users
+
+        # Construct sets for all the local users and remote hosts that were
+        # already in the room
+        prev_local_users = []
+        prev_remote_hosts = set()
+        for user_id in prev_users:
+            if self.is_mine_id(user_id):
+                prev_local_users.append(user_id)
+            else:
+                prev_remote_hosts.add(get_domain_from_id(user_id))
+
+        # Similarly, construct sets for all the local users and remote hosts
+        # that were *not* already in the room. Care needs to be taken with the
+        # calculating the remote hosts, as a host may have already been in the
+        # room even if there is a newly joined user from that host.
+        newly_joined_local_users = []
+        newly_joined_remote_hosts = set()
+        for user_id in newly_joined_users:
+            if self.is_mine_id(user_id):
+                newly_joined_local_users.append(user_id)
+            else:
+                host = get_domain_from_id(user_id)
+                if host not in prev_remote_hosts:
+                    newly_joined_remote_hosts.add(host)
 
-            states_d = await self.current_state_for_users(user_ids)
+        # Send presence states of all local users in the room to newly joined
+        # remote servers. (We actually only send states for local users already
+        # in the room, as we'll send states for newly joined local users below.)
+        if prev_local_users and newly_joined_remote_hosts:
+            local_states = await self.current_state_for_users(prev_local_users)
 
             # Filter out old presence, i.e. offline presence states where
             # the user hasn't been active for a week. We can change this
@@ -970,16 +1404,30 @@ async def _on_user_joined_room(
             now = self.clock.time_msec()
             states = [
                 state
-                for state in states_d.values()
+                for state in local_states.values()
                 if state.state != PresenceState.OFFLINE
                 or now - state.last_active_ts < 7 * 24 * 60 * 60 * 1000
                 or state.status_msg is not None
             ]
 
-            return [remote_host], states
+            self._federation_queue.send_presence_to_destinations(
+                destinations=newly_joined_remote_hosts,
+                states=states,
+            )
+
+        # Send presence states of newly joined users to all remote servers in
+        # the room
+        if newly_joined_local_users and (
+            prev_remote_hosts or newly_joined_remote_hosts
+        ):
+            local_states = await self.current_state_for_users(newly_joined_local_users)
+            self._federation_queue.send_presence_to_destinations(
+                destinations=prev_remote_hosts | newly_joined_remote_hosts,
+                states=list(local_states.values()),
+            )
 
 
-def should_notify(old_state, new_state):
+def should_notify(old_state: UserPresenceState, new_state: UserPresenceState) -> bool:
     """Decides if a presence state change should be sent to interested parties."""
     if old_state == new_state:
         return False
@@ -1015,7 +1463,9 @@ def should_notify(old_state, new_state):
     return False
 
 
-def format_user_presence_state(state, now, include_user_id=True):
+def format_user_presence_state(
+    state: UserPresenceState, now: int, include_user_id: bool = True
+) -> JsonDict:
     """Convert UserPresenceState to a format that can be sent down to clients
     and to other servers.
 
@@ -1027,7 +1477,7 @@ def format_user_presence_state(state, now, include_user_id=True):
         content["user_id"] = state.user_id
     if state.last_active_ts:
         content["last_active_ago"] = now - state.last_active_ts
-    if state.status_msg and state.state != PresenceState.OFFLINE:
+    if state.status_msg:
         content["status_msg"] = state.status_msg
     if state.state == PresenceState.ONLINE:
         content["currently_active"] = state.currently_active
@@ -1041,21 +1491,24 @@ def __init__(self, hs: "HomeServer"):
         #
         #   Presence -> Notifier -> PresenceEventSource -> Presence
         #
+        # Same with get_presence_router:
+        #
+        #   AuthHandler -> Notifier -> PresenceEventSource -> ModuleApi -> AuthHandler
         self.get_presence_handler = hs.get_presence_handler
+        self.get_presence_router = hs.get_presence_router
         self.clock = hs.get_clock()
         self.store = hs.get_datastore()
-        self.state = hs.get_state_handler()
 
     @log_function
     async def get_new_events(
         self,
-        user,
-        from_key,
-        room_ids=None,
-        include_offline=True,
-        explicit_room_id=None,
-        **kwargs
-    ):
+        user: UserID,
+        from_key: Optional[int],
+        room_ids: Optional[List[str]] = None,
+        include_offline: bool = True,
+        explicit_room_id: Optional[str] = None,
+        **kwargs,
+    ) -> Tuple[List[UserPresenceState], int]:
         # The process for getting presence events are:
         #  1. Get the rooms the user is in.
         #  2. Get the list of user in the rooms.
@@ -1068,10 +1521,25 @@ async def get_new_events(
         # We don't try and limit the presence updates by the current token, as
         # sending down the rare duplicate is not a concern.
 
+        user_id = user.to_string()
+        stream_change_cache = self.store.presence_stream_cache
+
         with Measure(self.clock, "presence.get_new_events"):
             if from_key is not None:
                 from_key = int(from_key)
 
+                # Check if this user should receive all current, online user presence. We only
+                # bother to do this if from_key is set, as otherwise the user will receive all
+                # user presence anyways.
+                if await self.store.should_user_receive_full_presence_with_token(
+                    user_id, from_key
+                ):
+                    # This user has been specified by a module to receive all current, online
+                    # user presence. Removing from_key and setting include_offline to false
+                    # will do effectively this.
+                    from_key = None
+                    include_offline = False
+
             max_token = self.store.get_current_presence_token()
             if from_key == max_token:
                 # This is necessary as due to the way stream ID generators work
@@ -1091,59 +1559,197 @@ async def get_new_events(
                 # doesn't return. C.f. #5503.
                 return [], max_token
 
-            presence = self.get_presence_handler()
-            stream_change_cache = self.store.presence_stream_cache
-
+            # Figure out which other users this user should receive updates for
             users_interested_in = await self._get_interested_in(user, explicit_room_id)
 
-            user_ids_changed = set()  # type: Collection[str]
-            changed = None
-            if from_key:
-                changed = stream_change_cache.get_all_entities_changed(from_key)
+            # We have a set of users that we're interested in the presence of. We want to
+            # cross-reference that with the users that have actually changed their presence.
 
-            if changed is not None and len(changed) < 500:
-                assert isinstance(user_ids_changed, set)
+            # Check whether this user should see all user updates
 
-                # For small deltas, its quicker to get all changes and then
-                # work out if we share a room or they're in our presence list
-                get_updates_counter.labels("stream").inc()
-                for other_user_id in changed:
-                    if other_user_id in users_interested_in:
-                        user_ids_changed.add(other_user_id)
-            else:
-                # Too many possible updates. Find all users we can see and check
-                # if any of them have changed.
-                get_updates_counter.labels("full").inc()
+            if users_interested_in == PresenceRouter.ALL_USERS:
+                # Provide presence state for all users
+                presence_updates = await self._filter_all_presence_updates_for_user(
+                    user_id, include_offline, from_key
+                )
 
-                if from_key:
-                    user_ids_changed = stream_change_cache.get_entities_changed(
-                        users_interested_in, from_key
-                    )
+                return presence_updates, max_token
+
+            # Make mypy happy. users_interested_in should now be a set
+            assert not isinstance(users_interested_in, str)
+
+            # The set of users that we're interested in and that have had a presence update.
+            # We'll actually pull the presence updates for these users at the end.
+            interested_and_updated_users: Union[Set[str], FrozenSet[str]] = set()
+
+            if from_key:
+                # First get all users that have had a presence update
+                updated_users = stream_change_cache.get_all_entities_changed(from_key)
+
+                # Cross-reference users we're interested in with those that have had updates.
+                # Use a slightly-optimised method for processing smaller sets of updates.
+                if updated_users is not None and len(updated_users) < 500:
+                    # For small deltas, it's quicker to get all changes and then
+                    # cross-reference with the users we're interested in
+                    get_updates_counter.labels("stream").inc()
+                    for other_user_id in updated_users:
+                        if other_user_id in users_interested_in:
+                            # mypy thinks this variable could be a FrozenSet as it's possibly set
+                            # to one in the `get_entities_changed` call below, and `add()` is not
+                            # method on a FrozenSet. That doesn't affect us here though, as
+                            # `interested_and_updated_users` is clearly a set() above.
+                            interested_and_updated_users.add(other_user_id)  # type: ignore
                 else:
-                    user_ids_changed = users_interested_in
+                    # Too many possible updates. Find all users we can see and check
+                    # if any of them have changed.
+                    get_updates_counter.labels("full").inc()
+
+                    interested_and_updated_users = (
+                        stream_change_cache.get_entities_changed(
+                            users_interested_in, from_key
+                        )
+                    )
+            else:
+                # No from_key has been specified. Return the presence for all users
+                # this user is interested in
+                interested_and_updated_users = users_interested_in
+
+            # Retrieve the current presence state for each user
+            users_to_state = await self.get_presence_handler().current_state_for_users(
+                interested_and_updated_users
+            )
+            presence_updates = list(users_to_state.values())
+
+        if not include_offline:
+            # Filter out offline presence states
+            presence_updates = self._filter_offline_presence_state(presence_updates)
+
+        return presence_updates, max_token
+
+    async def _filter_all_presence_updates_for_user(
+        self,
+        user_id: str,
+        include_offline: bool,
+        from_key: Optional[int] = None,
+    ) -> List[UserPresenceState]:
+        """
+        Computes the presence updates a user should receive.
+
+        First pulls presence updates from the database. Then consults PresenceRouter
+        for whether any updates should be excluded by user ID.
+
+        Args:
+            user_id: The User ID of the user to compute presence updates for.
+            include_offline: Whether to include offline presence states from the results.
+            from_key: The minimum stream ID of updates to pull from the database
+                before filtering.
 
-            updates = await presence.current_state_for_users(user_ids_changed)
+        Returns:
+            A list of presence states for the given user to receive.
+        """
+        if from_key:
+            # Only return updates since the last sync
+            updated_users = self.store.presence_stream_cache.get_all_entities_changed(
+                from_key
+            )
+            if not updated_users:
+                updated_users = []
+
+            # Get the actual presence update for each change
+            users_to_state = await self.get_presence_handler().current_state_for_users(
+                updated_users
+            )
+            presence_updates = list(users_to_state.values())
 
-        if include_offline:
-            return (list(updates.values()), max_token)
+            if not include_offline:
+                # Filter out offline states
+                presence_updates = self._filter_offline_presence_state(presence_updates)
         else:
-            return (
-                [s for s in updates.values() if s.state != PresenceState.OFFLINE],
-                max_token,
+            users_to_state = await self.store.get_presence_for_all_users(
+                include_offline=include_offline
             )
 
-    def get_current_key(self):
+            presence_updates = list(users_to_state.values())
+
+        # TODO: This feels wildly inefficient, and it's unfortunate we need to ask the
+        # module for information on a number of users when we then only take the info
+        # for a single user
+
+        # Filter through the presence router
+        users_to_state_set = await self.get_presence_router().get_users_for_states(
+            presence_updates
+        )
+
+        # We only want the mapping for the syncing user
+        presence_updates = list(users_to_state_set[user_id])
+
+        # Return presence information for all users
+        return presence_updates
+
+    def _filter_offline_presence_state(
+        self, presence_updates: Iterable[UserPresenceState]
+    ) -> List[UserPresenceState]:
+        """Given an iterable containing user presence updates, return a list with any offline
+        presence states removed.
+
+        Args:
+            presence_updates: Presence states to filter
+
+        Returns:
+            A new list with any offline presence states removed.
+        """
+        return [
+            update
+            for update in presence_updates
+            if update.state != PresenceState.OFFLINE
+        ]
+
+    def get_current_key(self) -> int:
         return self.store.get_current_presence_token()
 
     @cached(num_args=2, cache_context=True)
-    async def _get_interested_in(self, user, explicit_room_id, cache_context):
+    async def _get_interested_in(
+        self,
+        user: UserID,
+        explicit_room_id: Optional[str] = None,
+        cache_context: Optional[_CacheContext] = None,
+    ) -> Union[Set[str], str]:
         """Returns the set of users that the given user should see presence
-        updates for
+        updates for.
+
+        Args:
+            user: The user to retrieve presence updates for.
+            explicit_room_id: The users that are in the room will be returned.
+
+        Returns:
+            A set of user IDs to return presence updates for, or "ALL" to return all
+            known updates.
         """
         user_id = user.to_string()
         users_interested_in = set()
         users_interested_in.add(user_id)  # So that we receive our own presence
 
+        # cache_context isn't likely to ever be None due to the @cached decorator,
+        # but we can't have a non-optional argument after the optional argument
+        # explicit_room_id either. Assert cache_context is not None so we can use it
+        # without mypy complaining.
+        assert cache_context
+
+        # Check with the presence router whether we should poll additional users for
+        # their presence information
+        additional_users = await self.get_presence_router().get_interested_users(
+            user.to_string()
+        )
+        if additional_users == PresenceRouter.ALL_USERS:
+            # If the module requested that this user see the presence updates of *all*
+            # users, then simply return that instead of calculating what rooms this
+            # user shares
+            return PresenceRouter.ALL_USERS
+
+        # Add the additional users from the router
+        users_interested_in.update(additional_users)
+
+        # Find the users who share a room with this user
         users_who_share_room = await self.store.get_users_who_share_room_with_user(
             user_id, on_invalidate=cache_context.invalidate
         )
@@ -1158,15 +1764,20 @@ async def _get_interested_in(self, user, explicit_room_id, cache_context):
         return users_interested_in
 
 
-def handle_timeouts(user_states, is_mine_fn, syncing_user_ids, now):
+def handle_timeouts(
+    user_states: List[UserPresenceState],
+    is_mine_fn: Callable[[str], bool],
+    syncing_user_ids: Set[str],
+    now: int,
+) -> List[UserPresenceState]:
     """Checks the presence of users that have timed out and updates as
     appropriate.
 
     Args:
-        user_states(list): List of UserPresenceState's to check.
-        is_mine_fn (fn): Function that returns if a user_id is ours
-        syncing_user_ids (set): Set of user_ids with active syncs.
-        now (int): Current time in ms.
+        user_states: List of UserPresenceState's to check.
+        is_mine_fn: Function that returns if a user_id is ours
+        syncing_user_ids: Set of user_ids with active syncs.
+        now: Current time in ms.
 
     Returns:
         List of UserPresenceState updates
@@ -1183,14 +1794,16 @@ def handle_timeouts(user_states, is_mine_fn, syncing_user_ids, now):
     return list(changes.values())
 
 
-def handle_timeout(state, is_mine, syncing_user_ids, now):
+def handle_timeout(
+    state: UserPresenceState, is_mine: bool, syncing_user_ids: Set[str], now: int
+) -> Optional[UserPresenceState]:
     """Checks the presence of the user to see if any of the timers have elapsed
 
     Args:
-        state (UserPresenceState)
-        is_mine (bool): Whether the user is ours
-        syncing_user_ids (set): Set of user_ids with active syncs.
-        now (int): Current time in ms.
+        state
+        is_mine: Whether the user is ours
+        syncing_user_ids: Set of user_ids with active syncs.
+        now: Current time in ms.
 
     Returns:
         A UserPresenceState update or None if no update.
@@ -1226,9 +1839,7 @@ def handle_timeout(state, is_mine, syncing_user_ids, now):
             # don't set them as offline.
             sync_or_active = max(state.last_user_sync_ts, state.last_active_ts)
             if now - sync_or_active > SYNC_ONLINE_TIMEOUT:
-                state = state.copy_and_replace(
-                    state=PresenceState.OFFLINE, status_msg=None
-                )
+                state = state.copy_and_replace(state=PresenceState.OFFLINE)
                 changed = True
     else:
         # We expect to be poked occasionally by the other side.
@@ -1236,29 +1847,35 @@ def handle_timeout(state, is_mine, syncing_user_ids, now):
         # no one gets stuck online forever.
         if now - state.last_federation_update_ts > FEDERATION_TIMEOUT:
             # The other side seems to have disappeared.
-            state = state.copy_and_replace(state=PresenceState.OFFLINE, status_msg=None)
+            state = state.copy_and_replace(state=PresenceState.OFFLINE)
             changed = True
 
     return state if changed else None
 
 
-def handle_update(prev_state, new_state, is_mine, wheel_timer, now):
+def handle_update(
+    prev_state: UserPresenceState,
+    new_state: UserPresenceState,
+    is_mine: bool,
+    wheel_timer: WheelTimer,
+    now: int,
+) -> Tuple[UserPresenceState, bool, bool]:
     """Given a presence update:
         1. Add any appropriate timers.
         2. Check if we should notify anyone.
 
     Args:
-        prev_state (UserPresenceState)
-        new_state (UserPresenceState)
-        is_mine (bool): Whether the user is ours
-        wheel_timer (WheelTimer)
-        now (int): Time now in ms
+        prev_state
+        new_state
+        is_mine: Whether the user is ours
+        wheel_timer
+        now: Time now in ms
 
     Returns:
         3-tuple: `(new_state, persist_and_notify, federation_ping)` where:
             - new_state: is the state to actually persist
-            - persist_and_notify (bool): whether to persist and notify people
-            - federation_ping (bool): whether we should send a ping over federation
+            - persist_and_notify: whether to persist and notify people
+            - federation_ping: whether we should send a ping over federation
     """
     user_id = new_state.user_id
 
@@ -1314,21 +1931,22 @@ def handle_update(prev_state, new_state, is_mine, wheel_timer, now):
 
 
 async def get_interested_parties(
-    store: DataStore, states: List[UserPresenceState]
+    store: DataStore, presence_router: PresenceRouter, states: List[UserPresenceState]
 ) -> Tuple[Dict[str, List[UserPresenceState]], Dict[str, List[UserPresenceState]]]:
     """Given a list of states return which entities (rooms, users)
     are interested in the given states.
 
     Args:
-        store
-        states
+        store: The homeserver's data store.
+        presence_router: A module for augmenting the destinations for presence updates.
+        states: A list of incoming user presence updates.
 
     Returns:
         A 2-tuple of `(room_ids_to_states, users_to_states)`,
         with each item being a dict of `entity_name` -> `[UserPresenceState]`
     """
-    room_ids_to_states = {}  # type: Dict[str, List[UserPresenceState]]
-    users_to_states = {}  # type: Dict[str, List[UserPresenceState]]
+    room_ids_to_states: Dict[str, List[UserPresenceState]] = {}
+    users_to_states: Dict[str, List[UserPresenceState]] = {}
     for state in states:
         room_ids = await store.get_rooms_for_user(state.user_id)
         for room_id in room_ids:
@@ -1337,40 +1955,269 @@ async def get_interested_parties(
         # Always notify self
         users_to_states.setdefault(state.user_id, []).append(state)
 
+    # Ask a presence routing module for any additional parties if one
+    # is loaded.
+    router_users_to_states = await presence_router.get_users_for_states(states)
+
+    # Update the dictionaries with additional destinations and state to send
+    for user_id, user_states in router_users_to_states.items():
+        users_to_states.setdefault(user_id, []).extend(user_states)
+
     return room_ids_to_states, users_to_states
 
 
 async def get_interested_remotes(
-    store: DataStore, states: List[UserPresenceState], state_handler: StateHandler
-) -> List[Tuple[Collection[str], List[UserPresenceState]]]:
+    store: DataStore,
+    presence_router: PresenceRouter,
+    states: List[UserPresenceState],
+) -> Dict[str, Set[UserPresenceState]]:
     """Given a list of presence states figure out which remote servers
     should be sent which.
 
     All the presence states should be for local users only.
 
     Args:
-        store
-        states
-        state_handler
+        store: The homeserver's data store.
+        presence_router: A module for augmenting the destinations for presence updates.
+        states: A list of incoming user presence updates.
 
     Returns:
-        A list of 2-tuples of destinations and states, where for
-        each tuple the list of UserPresenceState should be sent to each
-        destination
+        A map from destinations to presence states to send to that destination.
     """
-    hosts_and_states = []  # type: List[Tuple[Collection[str], List[UserPresenceState]]]
+    hosts_and_states: Dict[str, Set[UserPresenceState]] = {}
 
     # First we look up the rooms each user is in (as well as any explicit
     # subscriptions), then for each distinct room we look up the remote
     # hosts in those rooms.
-    room_ids_to_states, users_to_states = await get_interested_parties(store, states)
+    room_ids_to_states, users_to_states = await get_interested_parties(
+        store, presence_router, states
+    )
 
     for room_id, states in room_ids_to_states.items():
-        hosts = await state_handler.get_current_hosts_in_room(room_id)
-        hosts_and_states.append((hosts, states))
+        user_ids = await store.get_users_in_room(room_id)
+        hosts = {get_domain_from_id(user_id) for user_id in user_ids}
+        for host in hosts:
+            hosts_and_states.setdefault(host, set()).update(states)
 
     for user_id, states in users_to_states.items():
         host = get_domain_from_id(user_id)
-        hosts_and_states.append(([host], states))
+        hosts_and_states.setdefault(host, set()).update(states)
 
     return hosts_and_states
+
+
+class PresenceFederationQueue:
+    """Handles sending ad hoc presence updates over federation, which are *not*
+    due to state updates (that get handled via the presence stream), e.g.
+    federation pings and sending existing present states to newly joined hosts.
+
+    Only the last N minutes will be queued, so if a federation sender instance
+    is down for longer then some updates will be dropped. This is OK as presence
+    is ephemeral, and so it will self correct eventually.
+
+    On workers the class tracks the last received position of the stream from
+    replication, and handles querying for missed updates over HTTP replication,
+    c.f. `get_current_token` and `get_replication_rows`.
+    """
+
+    # How long to keep entries in the queue for. Workers that are down for
+    # longer than this duration will miss out on older updates.
+    _KEEP_ITEMS_IN_QUEUE_FOR_MS = 5 * 60 * 1000
+
+    # How often to check if we can expire entries from the queue.
+    _CLEAR_ITEMS_EVERY_MS = 60 * 1000
+
+    def __init__(self, hs: "HomeServer", presence_handler: BasePresenceHandler):
+        self._clock = hs.get_clock()
+        self._notifier = hs.get_notifier()
+        self._instance_name = hs.get_instance_name()
+        self._presence_handler = presence_handler
+        self._repl_client = ReplicationGetStreamUpdates.make_client(hs)
+
+        # Should we keep a queue of recent presence updates? We only bother if
+        # another process may be handling federation sending.
+        self._queue_presence_updates = True
+
+        # Whether this instance is a presence writer.
+        self._presence_writer = self._instance_name in hs.config.worker.writers.presence
+
+        # The FederationSender instance, if this process sends federation traffic directly.
+        self._federation = None
+
+        if hs.should_send_federation():
+            self._federation = hs.get_federation_sender()
+
+            # We don't bother queuing up presence states if only this instance
+            # is sending federation.
+            if hs.config.worker.federation_shard_config.instances == [
+                self._instance_name
+            ]:
+                self._queue_presence_updates = False
+
+        # The queue of recently queued updates as tuples of: `(timestamp,
+        # stream_id, destinations, user_ids)`. We don't store the full states
+        # for efficiency, and remote workers will already have the full states
+        # cached.
+        self._queue: List[Tuple[int, int, Collection[str], Set[str]]] = []
+
+        self._next_id = 1
+
+        # Map from instance name to current token
+        self._current_tokens: Dict[str, int] = {}
+
+        if self._queue_presence_updates:
+            self._clock.looping_call(self._clear_queue, self._CLEAR_ITEMS_EVERY_MS)
+
+    def _clear_queue(self):
+        """Clear out older entries from the queue."""
+        clear_before = self._clock.time_msec() - self._KEEP_ITEMS_IN_QUEUE_FOR_MS
+
+        # The queue is sorted by timestamp, so we can bisect to find the right
+        # place to purge before. Note that we are searching using a 1-tuple with
+        # the time, which does The Right Thing since the queue is a tuple where
+        # the first item is a timestamp.
+        index = bisect(self._queue, (clear_before,))
+        self._queue = self._queue[index:]
+
+    def send_presence_to_destinations(
+        self, states: Collection[UserPresenceState], destinations: Collection[str]
+    ) -> None:
+        """Send the presence states to the given destinations.
+
+        Will forward to the local federation sender (if there is one) and queue
+        to send over replication (if there are other federation sender instances.).
+
+        Must only be called on the presence writer process.
+        """
+
+        # This should only be called on a presence writer.
+        assert self._presence_writer
+
+        if self._federation:
+            self._federation.send_presence_to_destinations(
+                states=states,
+                destinations=destinations,
+            )
+
+        if not self._queue_presence_updates:
+            return
+
+        now = self._clock.time_msec()
+
+        stream_id = self._next_id
+        self._next_id += 1
+
+        self._queue.append((now, stream_id, destinations, {s.user_id for s in states}))
+
+        self._notifier.notify_replication()
+
+    def get_current_token(self, instance_name: str) -> int:
+        """Get the current position of the stream.
+
+        On workers this returns the last stream ID received from replication.
+        """
+        if instance_name == self._instance_name:
+            return self._next_id - 1
+        else:
+            return self._current_tokens.get(instance_name, 0)
+
+    async def get_replication_rows(
+        self,
+        instance_name: str,
+        from_token: int,
+        upto_token: int,
+        target_row_count: int,
+    ) -> Tuple[List[Tuple[int, Tuple[str, str]]], int, bool]:
+        """Get all the updates between the two tokens.
+
+        We return rows in the form of `(destination, user_id)` to keep the size
+        of each row bounded (rather than returning the sets in a row).
+
+        On workers this will query the presence writer process via HTTP replication.
+        """
+        if instance_name != self._instance_name:
+            # If not local we query over http replication from the presence
+            # writer
+            result = await self._repl_client(
+                instance_name=instance_name,
+                stream_name=PresenceFederationStream.NAME,
+                from_token=from_token,
+                upto_token=upto_token,
+            )
+            return result["updates"], result["upto_token"], result["limited"]
+
+        # If the from_token is the current token then there's nothing to return
+        # and we can trivially no-op.
+        if from_token == self._next_id - 1:
+            return [], upto_token, False
+
+        # We can find the correct position in the queue by noting that there is
+        # exactly one entry per stream ID, and that the last entry has an ID of
+        # `self._next_id - 1`, so we can count backwards from the end.
+        #
+        # Since we are returning all states in the range `from_token < stream_id
+        # <= upto_token` we look for the index with a `stream_id` of `from_token
+        # + 1`.
+        #
+        # Since the start of the queue is periodically truncated we need to
+        # handle the case where `from_token` stream ID has already been dropped.
+        start_idx = max(from_token + 1 - self._next_id, -len(self._queue))
+
+        to_send: List[Tuple[int, Tuple[str, str]]] = []
+        limited = False
+        new_id = upto_token
+        for _, stream_id, destinations, user_ids in self._queue[start_idx:]:
+            if stream_id <= from_token:
+                # Paranoia check that we are actually only sending states that
+                # are have stream_id strictly greater than from_token. We should
+                # never hit this.
+                logger.warning(
+                    "Tried returning presence federation stream ID: %d less than from_token: %d (next_id: %d, len: %d)",
+                    stream_id,
+                    from_token,
+                    self._next_id,
+                    len(self._queue),
+                )
+                continue
+
+            if stream_id > upto_token:
+                break
+
+            new_id = stream_id
+
+            to_send.extend(
+                (stream_id, (destination, user_id))
+                for destination in destinations
+                for user_id in user_ids
+            )
+
+            if len(to_send) > target_row_count:
+                limited = True
+                break
+
+        return to_send, new_id, limited
+
+    async def process_replication_rows(
+        self, stream_name: str, instance_name: str, token: int, rows: list
+    ):
+        if stream_name != PresenceFederationStream.NAME:
+            return
+
+        # We keep track of the current tokens (so that we can catch up with anything we missed after a disconnect)
+        self._current_tokens[instance_name] = token
+
+        # If we're a federation sender we pull out the presence states to send
+        # and forward them on.
+        if not self._federation:
+            return
+
+        hosts_to_users: Dict[str, Set[str]] = {}
+        for row in rows:
+            hosts_to_users.setdefault(row.destination, set()).add(row.user_id)
+
+        for host, user_ids in hosts_to_users.items():
+            states = await self._presence_handler.current_state_for_users(user_ids)
+            self._federation.send_presence_to_destinations(
+                states=states.values(),
+                destinations=[host],
+            )
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index ee8f868791..6e3f0bffee 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -296,7 +295,7 @@ async def set_displayname(
                 400, "Displayname is too long (max %i)" % (MAX_DISPLAYNAME_LEN,)
             )
 
-        displayname_to_set = new_displayname  # type: Optional[str]
+        displayname_to_set: Optional[str] = new_displayname
         if new_displayname == "":
             displayname_to_set = None
 
@@ -327,7 +326,10 @@ async def set_displayname(
                 target_user.to_string(), profile
             )
 
-        await self._update_join_states(requester, target_user)
+        # Don't ratelimit when the admin makes the change.
+        # FIXME: this is because we call this function on registration to update DINUM's
+        #  custom userdir.
+        await self._update_join_states(requester, target_user, ratelimit=not by_admin)
 
         # start a profile replication push
         run_in_background(self._replicate_profiles)
@@ -436,7 +438,7 @@ async def set_avatar_url(
                 400, "Avatar URL is too long (max %i)" % (MAX_AVATAR_URL_LEN,)
             )
 
-        avatar_url_to_set = new_avatar_url  # type: Optional[str]
+        avatar_url_to_set: Optional[str] = new_avatar_url
         if new_avatar_url == "":
             avatar_url_to_set = None
 
@@ -552,12 +554,16 @@ async def on_profile_query(self, args: JsonDict) -> JsonDict:
         return response
 
     async def _update_join_states(
-        self, requester: Requester, target_user: UserID
+        self,
+        requester: Requester,
+        target_user: UserID,
+        ratelimit: bool = True,
     ) -> None:
         if not self.hs.is_mine(target_user):
             return
 
-        await self.ratelimit(requester)
+        if ratelimit:
+            await self.ratelimit(requester)
 
         # Do not actually update the room state for shadow-banned users.
         if requester.shadow_banned:
diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py
index a54fe1968e..c679a8303e 100644
--- a/synapse/handlers/read_marker.py
+++ b/synapse/handlers/read_marker.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py
index dbfe9bfaca..fb495229a7 100644
--- a/synapse/handlers/receipts.py
+++ b/synapse/handlers/receipts.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,9 +14,10 @@
 import logging
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
+from synapse.api.constants import ReadReceiptEventFields
 from synapse.appservice import ApplicationService
 from synapse.handlers._base import BaseHandler
-from synapse.types import JsonDict, ReadReceipt, get_domain_from_id
+from synapse.types import JsonDict, ReadReceipt, UserID, get_domain_from_id
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -31,6 +31,8 @@ def __init__(self, hs: "HomeServer"):
 
         self.server_name = hs.config.server_name
         self.store = hs.get_datastore()
+        self.event_auth_handler = hs.get_event_auth_handler()
+
         self.hs = hs
 
         # We only need to poke the federation sender explicitly if its on the
@@ -60,6 +62,20 @@ async def _received_remote_receipt(self, origin: str, content: JsonDict) -> None
         """Called when we receive an EDU of type m.receipt from a remote HS."""
         receipts = []
         for room_id, room_values in content.items():
+            # If we're not in the room just ditch the event entirely. This is
+            # probably an old server that has come back and thinks we're still in
+            # the room (or we've been rejoined to the room by a state reset).
+            is_in_room = await self.event_auth_handler.check_host_in_room(
+                room_id, self.server_name
+            )
+            if not is_in_room:
+                logger.info(
+                    "Ignoring receipt for room %r from server %s as we're not in the room",
+                    room_id,
+                    origin,
+                )
+                continue
+
             for receipt_type, users in room_values.items():
                 for user_id, user_values in users.items():
                     if get_domain_from_id(user_id) != origin:
@@ -84,8 +100,8 @@ async def _received_remote_receipt(self, origin: str, content: JsonDict) -> None
 
     async def _handle_new_receipts(self, receipts: List[ReadReceipt]) -> bool:
         """Takes a list of receipts, stores them and informs the notifier."""
-        min_batch_id = None  # type: Optional[int]
-        max_batch_id = None  # type: Optional[int]
+        min_batch_id: Optional[int] = None
+        max_batch_id: Optional[int] = None
 
         for receipt in receipts:
             res = await self.store.insert_receipt(
@@ -123,7 +139,7 @@ async def _handle_new_receipts(self, receipts: List[ReadReceipt]) -> bool:
         return True
 
     async def received_client_receipt(
-        self, room_id: str, receipt_type: str, user_id: str, event_id: str
+        self, room_id: str, receipt_type: str, user_id: str, event_id: str, hidden: bool
     ) -> None:
         """Called when a client tells us a local user has read up to the given
         event_id in the room.
@@ -133,23 +149,74 @@ async def received_client_receipt(
             receipt_type=receipt_type,
             user_id=user_id,
             event_ids=[event_id],
-            data={"ts": int(self.clock.time_msec())},
+            data={"ts": int(self.clock.time_msec()), "hidden": hidden},
         )
 
         is_new = await self._handle_new_receipts([receipt])
         if not is_new:
             return
 
-        if self.federation_sender:
+        if self.federation_sender and not (
+            self.hs.config.experimental.msc2285_enabled and hidden
+        ):
             await self.federation_sender.send_read_receipt(receipt)
 
 
 class ReceiptEventSource:
     def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastore()
+        self.config = hs.config
+
+    @staticmethod
+    def filter_out_hidden(events: List[JsonDict], user_id: str) -> List[JsonDict]:
+        visible_events = []
+
+        # filter out hidden receipts the user shouldn't see
+        for event in events:
+            content = event.get("content", {})
+            new_event = event.copy()
+            new_event["content"] = {}
+
+            for event_id in content.keys():
+                event_content = content.get(event_id, {})
+                m_read = event_content.get("m.read", {})
+
+                # If m_read is missing copy over the original event_content as there is nothing to process here
+                if not m_read:
+                    new_event["content"][event_id] = event_content.copy()
+                    continue
+
+                new_users = {}
+                for rr_user_id, user_rr in m_read.items():
+                    try:
+                        hidden = user_rr.get("hidden")
+                    except AttributeError:
+                        # Due to https://github.com/matrix-org/synapse/issues/10376
+                        # there are cases where user_rr is a string, in those cases
+                        # we just ignore the read receipt
+                        continue
+
+                    if hidden is not True or rr_user_id == user_id:
+                        new_users[rr_user_id] = user_rr.copy()
+                        # If hidden has a value replace hidden with the correct prefixed key
+                        if hidden is not None:
+                            new_users[rr_user_id].pop("hidden")
+                            new_users[rr_user_id][
+                                ReadReceiptEventFields.MSC2285_HIDDEN
+                            ] = hidden
+
+                # Set new users unless empty
+                if len(new_users.keys()) > 0:
+                    new_event["content"][event_id] = {"m.read": new_users}
+
+            # Append new_event to visible_events unless empty
+            if len(new_event["content"].keys()) > 0:
+                visible_events.append(new_event)
+
+        return visible_events
 
     async def get_new_events(
-        self, from_key: int, room_ids: List[str], **kwargs
+        self, from_key: int, room_ids: List[str], user: UserID, **kwargs
     ) -> Tuple[List[JsonDict], int]:
         from_key = int(from_key)
         to_key = self.get_current_key()
@@ -161,6 +228,9 @@ async def get_new_events(
             room_ids, from_key=from_key, to_key=to_key
         )
 
+        if self.config.experimental.msc2285_enabled:
+            events = ReceiptEventSource.filter_out_hidden(events, user.to_string())
+
         return (events, to_key)
 
     async def get_new_events_as(
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 1b2a515ee9..2e33c2b1bf 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,9 +15,10 @@
 """Contains functions for registering clients."""
 
 import logging
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple
+from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple
 
 from prometheus_client import Counter
+from typing_extensions import TypedDict
 
 from synapse import types
 from synapse.api.constants import MAX_USERID_LENGTH, EventTypes, JoinRules, LoginType
@@ -56,6 +56,13 @@
 )
 
 
+class LoginDict(TypedDict):
+    device_id: str
+    access_token: str
+    valid_until_ms: Optional[int]
+    refresh_token: Optional[str]
+
+
 class RegistrationHandler(BaseHandler):
     def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
@@ -68,6 +75,7 @@ def __init__(self, hs: "HomeServer"):
         self.identity_handler = self.hs.get_identity_handler()
         self.ratelimiter = hs.get_registration_ratelimiter()
         self.macaroon_gen = hs.get_macaroon_generator()
+        self._account_validity_handler = hs.get_account_validity_handler()
         self._server_notices_mxid = hs.config.server_notices_mxid
         self._server_name = hs.hostname
 
@@ -89,6 +97,7 @@ def __init__(self, hs: "HomeServer"):
             self.pusher_pool = hs.get_pusherpool()
 
         self.session_lifetime = hs.config.session_lifetime
+        self.access_token_lifetime = hs.config.access_token_lifetime
 
     async def check_username(
         self,
@@ -184,7 +193,7 @@ async def register_user(
         user_type: Optional[str] = None,
         default_display_name: Optional[str] = None,
         address: Optional[str] = None,
-        bind_emails: Iterable[str] = [],
+        bind_emails: Optional[Iterable[str]] = None,
         by_admin: bool = False,
         user_agent_ips: Optional[List[Tuple[str, str]]] = None,
         auth_provider_id: Optional[str] = None,
@@ -211,7 +220,7 @@ async def register_user(
             bind_emails: list of emails to bind to this account.
             by_admin: True if this registration is being made via the
               admin api, otherwise False.
-            user_agent_ips: Tuples of IP addresses and user-agents used
+            user_agent_ips: Tuples of user-agents and IP addresses used
                 during the registration process.
             auth_provider_id: The SSO IdP the user used, if any.
         Returns:
@@ -219,7 +228,9 @@ async def register_user(
         Raises:
             SynapseError if there was a problem registering.
         """
-        self.check_registration_ratelimit(address)
+        bind_emails = bind_emails or []
+
+        await self.check_registration_ratelimit(address)
 
         result = await self.spam_checker.check_registration_for_spam(
             threepid,
@@ -277,6 +288,10 @@ async def register_user(
 
             if default_display_name:
                 requester = create_requester(user)
+                # FIXME: this function call is DINUM-specific code to update DINUM's
+                #  custom Sydent-powered userdir, and needed some custom changes to
+                #  ignore the ratelimiter. On mainline, we don't need to call this
+                #  function.
                 await self.profile_handler.set_displayname(
                     user, requester, default_display_name, by_admin=True
                 )
@@ -417,11 +432,32 @@ async def _create_and_join_rooms(self, user_id: str) -> None:
                 room_alias = RoomAlias.from_string(r)
 
                 if self.hs.hostname != room_alias.domain:
-                    logger.warning(
-                        "Cannot create room alias %s, "
-                        "it does not match server domain",
+                    # If the alias is remote, try to join the room. This might fail
+                    # because the room might be invite only, but we don't have any local
+                    # user in the room to invite this one with, so at this point that's
+                    # the best we can do.
+                    logger.info(
+                        "Cannot automatically create room with alias %s as it isn't"
+                        " local, trying to join the room instead",
                         r,
                     )
+
+                    (
+                        room,
+                        remote_room_hosts,
+                    ) = await room_member_handler.lookup_room_alias(room_alias)
+                    room_id = room.to_string()
+
+                    await room_member_handler.update_membership(
+                        requester=create_requester(
+                            user_id, authenticated_entity=self._server_name
+                        ),
+                        target=UserID.from_string(user_id),
+                        room_id=room_id,
+                        remote_room_hosts=remote_room_hosts,
+                        action="join",
+                        ratelimit=False,
+                    )
                 else:
                     # A shallow copy is OK here since the only key that is
                     # modified is room_alias_name.
@@ -479,22 +515,32 @@ async def _join_rooms(self, user_id: str) -> None:
                     )
 
                 # Calculate whether the room requires an invite or can be
-                # joined directly. Note that unless a join rule of public exists,
-                # it is treated as requiring an invite.
-                requires_invite = True
-
-                state = await self.store.get_filtered_current_state_ids(
-                    room_id, StateFilter.from_types([(EventTypes.JoinRules, "")])
+                # joined directly. By default, we consider the room as requiring an
+                # invite if the homeserver is in the room (unless told otherwise by the
+                # join rules). Otherwise we consider it as being joinable, at the risk of
+                # failing to join, but in this case there's little more we can do since
+                # we don't have a local user in the room to craft up an invite with.
+                requires_invite = await self.store.is_host_joined(
+                    room_id,
+                    self.server_name,
                 )
 
-                event_id = state.get((EventTypes.JoinRules, ""))
-                if event_id:
-                    join_rules_event = await self.store.get_event(
-                        event_id, allow_none=True
+                if requires_invite:
+                    # If the server is in the room, check if the room is public.
+                    state = await self.store.get_filtered_current_state_ids(
+                        room_id, StateFilter.from_types([(EventTypes.JoinRules, "")])
                     )
-                    if join_rules_event:
-                        join_rule = join_rules_event.content.get("join_rule", None)
-                        requires_invite = join_rule and join_rule != JoinRules.PUBLIC
+
+                    event_id = state.get((EventTypes.JoinRules, ""))
+                    if event_id:
+                        join_rules_event = await self.store.get_event(
+                            event_id, allow_none=True
+                        )
+                        if join_rules_event:
+                            join_rule = join_rules_event.content.get("join_rule", None)
+                            requires_invite = (
+                                join_rule and join_rule != JoinRules.PUBLIC
+                            )
 
                 # Send the invite, if necessary.
                 if requires_invite:
@@ -663,7 +709,7 @@ async def shadow_register(self, localpart, display_name, auth_result, params):
             },
         )
 
-    def check_registration_ratelimit(self, address: Optional[str]) -> None:
+    async def check_registration_ratelimit(self, address: Optional[str]) -> None:
         """A simple helper method to check whether the registration rate limit has been hit
         for a given IP address
 
@@ -677,7 +723,7 @@ def check_registration_ratelimit(self, address: Optional[str]) -> None:
         if not address:
             return
 
-        self.ratelimiter.ratelimit(address)
+        await self.ratelimiter.ratelimit(None, address)
 
     async def register_with_store(
         self,
@@ -736,6 +782,10 @@ async def register_with_store(
                 shadow_banned=shadow_banned,
             )
 
+            # Only call the account validity module(s) on the main process, to avoid
+            # repeating e.g. database writes on all of the workers.
+            await self._account_validity_handler.on_user_registration(user_id)
+
     async def register_device(
         self,
         user_id: str,
@@ -744,7 +794,8 @@ async def register_device(
         is_guest: bool = False,
         is_appservice_ghost: bool = False,
         auth_provider_id: Optional[str] = None,
-    ) -> Tuple[str, str]:
+        should_issue_refresh_token: bool = False,
+    ) -> Tuple[str, str, Optional[int], Optional[str]]:
         """Register a device for a user and generate an access token.
 
         The access token will be limited by the homeserver's session_lifetime config.
@@ -756,8 +807,9 @@ async def register_device(
             is_guest: Whether this is a guest account
             auth_provider_id: The SSO IdP the user used, if any (just used for the
                 prometheus metrics).
+            should_issue_refresh_token: Whether it should also issue a refresh token
         Returns:
-            Tuple of device ID and access token
+            Tuple of device ID, access token, access token expiration time and refresh token
         """
         res = await self._register_device_client(
             user_id=user_id,
@@ -765,6 +817,7 @@ async def register_device(
             initial_display_name=initial_display_name,
             is_guest=is_guest,
             is_appservice_ghost=is_appservice_ghost,
+            should_issue_refresh_token=should_issue_refresh_token,
         )
 
         login_counter.labels(
@@ -772,7 +825,12 @@ async def register_device(
             auth_provider=(auth_provider_id or ""),
         ).inc()
 
-        return res["device_id"], res["access_token"]
+        return (
+            res["device_id"],
+            res["access_token"],
+            res["valid_until_ms"],
+            res["refresh_token"],
+        )
 
     async def register_device_inner(
         self,
@@ -781,7 +839,8 @@ async def register_device_inner(
         initial_display_name: Optional[str],
         is_guest: bool = False,
         is_appservice_ghost: bool = False,
-    ) -> Dict[str, str]:
+        should_issue_refresh_token: bool = False,
+    ) -> LoginDict:
         """Helper for register_device
 
         Does the bits that need doing on the main process. Not for use outside this
@@ -796,23 +855,40 @@ class and RegisterDeviceReplicationServlet.
                 )
             valid_until_ms = self.clock.time_msec() + self.session_lifetime
 
+        refresh_token = None
+        refresh_token_id = None
+
         registered_device_id = await self.device_handler.check_device_registered(
             user_id, device_id, initial_display_name
         )
         if is_guest:
             assert valid_until_ms is None
-            access_token = self.macaroon_gen.generate_access_token(
-                user_id, ["guest = true"]
-            )
+            access_token = self.macaroon_gen.generate_guest_access_token(user_id)
         else:
+            if should_issue_refresh_token:
+                (
+                    refresh_token,
+                    refresh_token_id,
+                ) = await self._auth_handler.get_refresh_token_for_user_id(
+                    user_id,
+                    device_id=registered_device_id,
+                )
+                valid_until_ms = self.clock.time_msec() + self.access_token_lifetime
+
             access_token = await self._auth_handler.get_access_token_for_user_id(
                 user_id,
                 device_id=registered_device_id,
                 valid_until_ms=valid_until_ms,
                 is_appservice_ghost=is_appservice_ghost,
+                refresh_token_id=refresh_token_id,
             )
 
-        return {"device_id": registered_device_id, "access_token": access_token}
+        return {
+            "device_id": registered_device_id,
+            "access_token": access_token,
+            "valid_until_ms": valid_until_ms,
+            "refresh_token": refresh_token,
+        }
 
     async def post_registration_actions(
         self, user_id: str, auth_result: dict, access_token: Optional[str]
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index f3da38a71e..8ee88b6b80 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 # Copyright 2018-2019 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -33,7 +32,14 @@
     RoomCreationPreset,
     RoomEncryptionAlgorithms,
 )
-from synapse.api.errors import AuthError, Codes, NotFoundError, StoreError, SynapseError
+from synapse.api.errors import (
+    AuthError,
+    Codes,
+    LimitExceededError,
+    NotFoundError,
+    StoreError,
+    SynapseError,
+)
 from synapse.api.filtering import Filter
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion
 from synapse.events import EventBase
@@ -77,10 +83,11 @@ def __init__(self, hs: "HomeServer"):
         self.spam_checker = hs.get_spam_checker()
         self.event_creation_handler = hs.get_event_creation_handler()
         self.room_member_handler = hs.get_room_member_handler()
+        self._event_auth_handler = hs.get_event_auth_handler()
         self.config = hs.config
 
         # Room state based off defined presets
-        self._presets_dict = {
+        self._presets_dict: Dict[str, Dict[str, Any]] = {
             RoomCreationPreset.PRIVATE_CHAT: {
                 "join_rules": JoinRules.INVITE,
                 "history_visibility": HistoryVisibility.SHARED,
@@ -102,7 +109,7 @@ def __init__(self, hs: "HomeServer"):
                 "guest_can_join": False,
                 "power_level_content_override": {},
             },
-        }  # type: Dict[str, Dict[str, Any]]
+        }
 
         # Modify presets to selectively enable encryption by default per homeserver config
         for preset_name, preset_config in self._presets_dict.items():
@@ -120,17 +127,13 @@ def __init__(self, hs: "HomeServer"):
         # If a user tries to update the same room multiple times in quick
         # succession, only process the first attempt and return its result to
         # subsequent requests
-        self._upgrade_response_cache = ResponseCache(
+        self._upgrade_response_cache: ResponseCache[Tuple[str, str]] = ResponseCache(
             hs.get_clock(), "room_upgrade", timeout_ms=FIVE_MINUTES_IN_MS
-        )  # type: ResponseCache[Tuple[str, str]]
+        )
         self._server_notices_mxid = hs.config.server_notices_mxid
 
         self.third_party_event_rules = hs.get_third_party_event_rules()
 
-        self._invite_burst_count = (
-            hs.config.ratelimiting.rc_invites_per_room.burst_count
-        )
-
     async def upgrade_room(
         self, requester: Requester, old_room_id: str, new_version: RoomVersion
     ) -> str:
@@ -224,7 +227,7 @@ async def _upgrade_room(
             },
         )
         old_room_version = await self.store.get_room_version_id(old_room_id)
-        await self.auth.check_from_context(
+        await self._event_auth_handler.check_from_context(
             old_room_version, tombstone_event, tombstone_context
         )
 
@@ -386,10 +389,10 @@ async def clone_existing_room(
         ):
             raise SynapseError(403, "You are not permitted to create rooms")
 
-        creation_content = {
+        creation_content: JsonDict = {
             "room_version": new_room_version.identifier,
             "predecessor": {"room_id": old_room_id, "event_id": tombstone_event_id},
-        }  # type: JsonDict
+        }
 
         # Check if old room was non-federatable
 
@@ -627,15 +630,11 @@ async def create_room(
         else:
             is_requester_admin = await self.auth.is_server_admin(requester.user)
 
-        # Check whether the third party rules allows/changes the room create
-        # request.
-        event_allowed = await self.third_party_event_rules.on_create_room(
+        # Let the third party rules modify the room creation config if needed, or abort
+        # the room creation entirely with an exception.
+        await self.third_party_event_rules.on_create_room(
             requester, config, is_requester_admin=is_requester_admin
         )
-        if not event_allowed:
-            raise SynapseError(
-                403, "You are not permitted to create rooms", Codes.FORBIDDEN
-            )
 
         invite_list = config.get("invite", [])
         invite_3pid_list = config.get("invite_3pid", [])
@@ -695,8 +694,18 @@ async def create_room(
             invite_3pid_list = []
             invite_list = []
 
-        if len(invite_list) + len(invite_3pid_list) > self._invite_burst_count:
-            raise SynapseError(400, "Cannot invite so many users at once")
+        if invite_list or invite_3pid_list:
+            try:
+                # If there are invites in the request, see if the ratelimiting settings
+                # allow that number of invites to be sent from the current user.
+                await self.room_member_handler.ratelimit_multiple_invites(
+                    requester,
+                    room_id=None,
+                    n_invites=len(invite_list) + len(invite_3pid_list),
+                    update=False,
+                )
+            except LimitExceededError:
+                raise SynapseError(400, "Cannot invite so many users at once")
 
         await self.event_creation_handler.assert_accepted_privacy_policy(requester)
 
@@ -944,7 +953,7 @@ async def send(etype: str, content: JsonDict, **kwargs) -> int:
                 etype=EventTypes.PowerLevels, content=pl_content
             )
         else:
-            power_level_content = {
+            power_level_content: JsonDict = {
                 "users": {creator_id: 100},
                 "users_default": 0,
                 "events": {
@@ -963,7 +972,8 @@ async def send(etype: str, content: JsonDict, **kwargs) -> int:
                 "kick": 50,
                 "redact": 50,
                 "invite": 50,
-            }  # type: JsonDict
+                "historical": 100,
+            }
 
             if config["original_invitees_have_ops"]:
                 for invitee in invite_list:
@@ -1349,7 +1359,7 @@ async def shutdown_room(
             new_room_id = None
             logger.info("Shutting down room %r", room_id)
 
-        users = await self.state.get_current_users_in_room(room_id)
+        users = await self.store.get_users_in_room(room_id)
         kicked_users = []
         failed_to_kick_users = []
         for user_id in users:
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 01fade718e..6d433fad41 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,7 +20,12 @@
 from unpaddedbase64 import decode_base64, encode_base64
 
 from synapse.api.constants import EventTypes, HistoryVisibility, JoinRules
-from synapse.api.errors import Codes, HttpResponseException
+from synapse.api.errors import (
+    Codes,
+    HttpResponseException,
+    RequestSendFailed,
+    SynapseError,
+)
 from synapse.types import JsonDict, ThirdPartyInstanceID
 from synapse.util.caches.descriptors import cached
 from synapse.util.caches.response_cache import ResponseCache
@@ -43,19 +47,19 @@ class RoomListHandler(BaseHandler):
     def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
         self.enable_room_list_search = hs.config.enable_room_list_search
-        self.response_cache = ResponseCache(
-            hs.get_clock(), "room_list"
-        )  # type: ResponseCache[Tuple[Optional[int], Optional[str], ThirdPartyInstanceID]]
-        self.remote_response_cache = ResponseCache(
-            hs.get_clock(), "remote_room_list", timeout_ms=30 * 1000
-        )  # type: ResponseCache[Tuple[str, Optional[int], Optional[str], bool, Optional[str]]]
+        self.response_cache: ResponseCache[
+            Tuple[Optional[int], Optional[str], Optional[ThirdPartyInstanceID]]
+        ] = ResponseCache(hs.get_clock(), "room_list")
+        self.remote_response_cache: ResponseCache[
+            Tuple[str, Optional[int], Optional[str], bool, Optional[str]]
+        ] = ResponseCache(hs.get_clock(), "remote_room_list", timeout_ms=30 * 1000)
 
     async def get_local_public_room_list(
         self,
         limit: Optional[int] = None,
         since_token: Optional[str] = None,
         search_filter: Optional[dict] = None,
-        network_tuple: ThirdPartyInstanceID = EMPTY_THIRD_PARTY_ID,
+        network_tuple: Optional[ThirdPartyInstanceID] = EMPTY_THIRD_PARTY_ID,
         from_federation: bool = False,
     ) -> JsonDict:
         """Generate a local public room list.
@@ -112,7 +116,7 @@ async def _get_public_room_list(
         limit: Optional[int] = None,
         since_token: Optional[str] = None,
         search_filter: Optional[dict] = None,
-        network_tuple: ThirdPartyInstanceID = EMPTY_THIRD_PARTY_ID,
+        network_tuple: Optional[ThirdPartyInstanceID] = EMPTY_THIRD_PARTY_ID,
         from_federation: bool = False,
     ) -> JsonDict:
         """Generate a public room list.
@@ -135,10 +139,10 @@ async def _get_public_room_list(
         if since_token:
             batch_token = RoomListNextBatch.from_token(since_token)
 
-            bounds = (
+            bounds: Optional[Tuple[int, str]] = (
                 batch_token.last_joined_members,
                 batch_token.last_room_id,
-            )  # type: Optional[Tuple[int, str]]
+            )
             forwards = batch_token.direction_is_forward
             has_batch_token = True
         else:
@@ -178,7 +182,7 @@ def build_room_entry(room):
 
         results = [build_room_entry(r) for r in results]
 
-        response = {}  # type: JsonDict
+        response: JsonDict = {}
         num_results = len(results)
         if limit is not None:
             more_to_come = num_results == probing_limit
@@ -352,6 +356,12 @@ async def get_remote_public_room_list(
         include_all_networks: bool = False,
         third_party_instance_id: Optional[str] = None,
     ) -> JsonDict:
+        """Get the public room list from remote server
+
+        Raises:
+            SynapseError
+        """
+
         if not self.enable_room_list_search:
             return {"chunk": [], "total_room_count_estimate": 0}
 
@@ -379,7 +389,11 @@ async def get_remote_public_room_list(
                 ):
                     logger.debug("Falling back to locally-filtered /publicRooms")
                 else:
-                    raise  # Not an error that should trigger a fallback.
+                    # Not an error that should trigger a fallback.
+                    raise SynapseError(502, "Failed to fetch room list")
+            except RequestSendFailed:
+                # Not an error that should trigger a fallback.
+                raise SynapseError(502, "Failed to fetch room list")
 
             # if we reach this point, then we fall back to the situation where
             # we currently don't support searching across federation, so we have
@@ -387,13 +401,16 @@ async def get_remote_public_room_list(
             limit = None
             since_token = None
 
-        res = await self._get_remote_list_cached(
-            server_name,
-            limit=limit,
-            since_token=since_token,
-            include_all_networks=include_all_networks,
-            third_party_instance_id=third_party_instance_id,
-        )
+        try:
+            res = await self._get_remote_list_cached(
+                server_name,
+                limit=limit,
+                since_token=since_token,
+                include_all_networks=include_all_networks,
+                third_party_instance_id=third_party_instance_id,
+            )
+        except (RequestSendFailed, HttpResponseException):
+            raise SynapseError(502, "Failed to fetch room list")
 
         if search_filter:
             res = {
@@ -415,6 +432,10 @@ async def _get_remote_list_cached(
         include_all_networks: bool = False,
         third_party_instance_id: Optional[str] = None,
     ) -> JsonDict:
+        """Wrapper around FederationClient.get_public_rooms that caches the
+        result.
+        """
+
         repl_layer = self.hs.get_federation_client()
         if search_filter:
             # We can't cache when asking for search
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 10af3782f4..362cd0cdde 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016-2020 The Matrix.org Foundation C.I.C.
 # Copyright 2020 Sorunome
 #
@@ -17,10 +16,15 @@
 import logging
 import random
 from http import HTTPStatus
-from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple
+from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple
 
 from synapse import types
-from synapse.api.constants import AccountDataTypes, EventTypes, Membership
+from synapse.api.constants import (
+    AccountDataTypes,
+    EventContentFields,
+    EventTypes,
+    Membership,
+)
 from synapse.api.errors import (
     AuthError,
     Codes,
@@ -29,6 +33,7 @@
     SynapseError,
 )
 from synapse.api.ratelimiting import Ratelimiter
+from synapse.event_auth import get_named_level, get_power_level_event
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
 from synapse.types import (
@@ -72,6 +77,7 @@ def __init__(self, hs: "HomeServer"):
         self.profile_handler = hs.get_profile_handler()
         self.event_creation_handler = hs.get_event_creation_handler()
         self.account_data_handler = hs.get_account_data_handler()
+        self.event_auth_handler = hs.get_event_auth_handler()
 
         self.member_linearizer = Linearizer(name="member")
 
@@ -83,22 +89,26 @@ def __init__(self, hs: "HomeServer"):
         self.allow_per_room_profiles = self.config.allow_per_room_profiles
 
         self._join_rate_limiter_local = Ratelimiter(
+            store=self.store,
             clock=self.clock,
             rate_hz=hs.config.ratelimiting.rc_joins_local.per_second,
             burst_count=hs.config.ratelimiting.rc_joins_local.burst_count,
         )
         self._join_rate_limiter_remote = Ratelimiter(
+            store=self.store,
             clock=self.clock,
             rate_hz=hs.config.ratelimiting.rc_joins_remote.per_second,
             burst_count=hs.config.ratelimiting.rc_joins_remote.burst_count,
         )
 
         self._invites_per_room_limiter = Ratelimiter(
+            store=self.store,
             clock=self.clock,
             rate_hz=hs.config.ratelimiting.rc_invites_per_room.per_second,
             burst_count=hs.config.ratelimiting.rc_invites_per_room.burst_count,
         )
         self._invites_per_user_limiter = Ratelimiter(
+            store=self.store,
             clock=self.clock,
             rate_hz=hs.config.ratelimiting.rc_invites_per_user.per_second,
             burst_count=hs.config.ratelimiting.rc_invites_per_user.burst_count,
@@ -206,15 +216,45 @@ async def _user_left_room(self, target: UserID, room_id: str) -> None:
     async def forget(self, user: UserID, room_id: str) -> None:
         raise NotImplementedError()
 
-    def ratelimit_invite(self, room_id: Optional[str], invitee_user_id: str):
+    async def ratelimit_multiple_invites(
+        self,
+        requester: Optional[Requester],
+        room_id: Optional[str],
+        n_invites: int,
+        update: bool = True,
+    ):
+        """Ratelimit more than one invite sent by the given requester in the given room.
+
+        Args:
+            requester: The requester sending the invites.
+            room_id: The room the invites are being sent in.
+            n_invites: The amount of invites to ratelimit for.
+            update: Whether to update the ratelimiter's cache.
+
+        Raises:
+            LimitExceededError: The requester can't send that many invites in the room.
+        """
+        await self._invites_per_room_limiter.ratelimit(
+            requester,
+            room_id,
+            update=update,
+            n_actions=n_invites,
+        )
+
+    async def ratelimit_invite(
+        self,
+        requester: Optional[Requester],
+        room_id: Optional[str],
+        invitee_user_id: str,
+    ):
         """Ratelimit invites by room and by target user.
 
         If room ID is missing then we just rate limit by target user.
         """
         if room_id:
-            self._invites_per_room_limiter.ratelimit(room_id)
+            await self._invites_per_room_limiter.ratelimit(requester, room_id)
 
-        self._invites_per_user_limiter.ratelimit(invitee_user_id)
+        await self._invites_per_user_limiter.ratelimit(requester, invitee_user_id)
 
     async def _local_membership_update(
         self,
@@ -223,11 +263,42 @@ async def _local_membership_update(
         room_id: str,
         membership: str,
         prev_event_ids: List[str],
+        auth_event_ids: Optional[List[str]] = None,
         txn_id: Optional[str] = None,
         ratelimit: bool = True,
         content: Optional[dict] = None,
         require_consent: bool = True,
+        outlier: bool = False,
     ) -> Tuple[str, int]:
+        """
+        Internal membership update function to get an existing event or create
+        and persist a new event for the new membership change.
+
+        Args:
+            requester:
+            target:
+            room_id:
+            membership:
+            prev_event_ids: The event IDs to use as the prev events
+
+            auth_event_ids:
+                The event ids to use as the auth_events for the new event.
+                Should normally be left as None, which will cause them to be calculated
+                based on the room state at the prev_events.
+
+            txn_id:
+            ratelimit:
+            content:
+            require_consent:
+
+            outlier: Indicates whether the event is an `outlier`, i.e. if
+                it's from an arbitrary point and floating in the DAG as
+                opposed to being inline with the current DAG.
+
+        Returns:
+            Tuple of event ID and stream ordering position
+        """
+
         user_id = target.to_string()
 
         if content is None:
@@ -264,7 +335,9 @@ async def _local_membership_update(
             },
             txn_id=txn_id,
             prev_event_ids=prev_event_ids,
+            auth_event_ids=auth_event_ids,
             require_consent=require_consent,
+            outlier=outlier,
         )
 
         prev_state_ids = await context.get_prev_state_ids()
@@ -284,7 +357,7 @@ async def _local_membership_update(
                 (
                     allowed,
                     time_allowed,
-                ) = self._join_rate_limiter_local.can_requester_do_action(requester)
+                ) = await self._join_rate_limiter_local.can_do_action(requester)
 
                 if not allowed:
                     raise LimitExceededError(
@@ -360,6 +433,9 @@ async def update_membership(
         content: Optional[dict] = None,
         new_room: bool = False,
         require_consent: bool = True,
+        outlier: bool = False,
+        prev_event_ids: Optional[List[str]] = None,
+        auth_event_ids: Optional[List[str]] = None,
     ) -> Tuple[str, int]:
         """Update a user's membership in a room.
 
@@ -374,6 +450,14 @@ async def update_membership(
             ratelimit: Whether to rate limit the request.
             content: The content of the created event.
             require_consent: Whether consent is required.
+            outlier: Indicates whether the event is an `outlier`, i.e. if
+                it's from an arbitrary point and floating in the DAG as
+                opposed to being inline with the current DAG.
+            prev_event_ids: The event IDs to use as the prev events
+            auth_event_ids:
+                The event ids to use as the auth_events for the new event.
+                Should normally be left as None, which will cause them to be calculated
+                based on the room state at the prev_events.
 
         Returns:
             A tuple of the new event ID and stream ID.
@@ -401,6 +485,9 @@ async def update_membership(
                 content=content,
                 new_room=new_room,
                 require_consent=require_consent,
+                outlier=outlier,
+                prev_event_ids=prev_event_ids,
+                auth_event_ids=auth_event_ids,
             )
 
         return result
@@ -418,10 +505,36 @@ async def update_membership_locked(
         content: Optional[dict] = None,
         new_room: bool = False,
         require_consent: bool = True,
+        outlier: bool = False,
+        prev_event_ids: Optional[List[str]] = None,
+        auth_event_ids: Optional[List[str]] = None,
     ) -> Tuple[str, int]:
         """Helper for update_membership.
 
         Assumes that the membership linearizer is already held for the room.
+
+        Args:
+            requester:
+            target:
+            room_id:
+            action:
+            txn_id:
+            remote_room_hosts:
+            third_party_signed:
+            ratelimit:
+            content:
+            require_consent:
+            outlier: Indicates whether the event is an `outlier`, i.e. if
+                it's from an arbitrary point and floating in the DAG as
+                opposed to being inline with the current DAG.
+            prev_event_ids: The event IDs to use as the prev events
+            auth_event_ids:
+                The event ids to use as the auth_events for the new event.
+                Should normally be left as None, which will cause them to be calculated
+                based on the room state at the prev_events.
+
+        Returns:
+            A tuple of the new event ID and stream ID.
         """
         content_specified = bool(content)
         if content is None:
@@ -471,9 +584,7 @@ async def update_membership_locked(
         if effective_membership_state == Membership.INVITE:
             target_id = target.to_string()
             if ratelimit:
-                # Don't ratelimit application services.
-                if not requester.app_service or requester.app_service.is_rate_limited():
-                    self.ratelimit_invite(room_id, target_id)
+                await self.ratelimit_invite(requester, room_id, target_id)
 
             # block any attempts to invite the server notices mxid
             if target_id == self._server_notices_mxid:
@@ -515,6 +626,21 @@ async def update_membership_locked(
             if block_invite:
                 raise SynapseError(403, "Invites have been disabled on this server")
 
+        if prev_event_ids:
+            return await self._local_membership_update(
+                requester=requester,
+                target=target,
+                room_id=room_id,
+                membership=effective_membership_state,
+                txn_id=txn_id,
+                ratelimit=ratelimit,
+                prev_event_ids=prev_event_ids,
+                auth_event_ids=auth_event_ids,
+                content=content,
+                require_consent=require_consent,
+                outlier=outlier,
+            )
+
         latest_event_ids = await self.store.get_prev_events_for_room(room_id)
 
         current_state_ids = await self.state_handler.get_current_state_ids(
@@ -599,18 +725,22 @@ async def update_membership_locked(
             if not is_requester_admin:
                 # We assume that if the spam checker allowed the user to create
                 # a room then they're allowed to join it.
-                if not new_room and not self.spam_checker.user_may_join_room(
+                if not new_room and not await self.spam_checker.user_may_join_room(
                     target.to_string(), room_id, is_invited=inviter is not None
                 ):
                     raise SynapseError(403, "Not allowed to join this room")
 
-            if not is_host_in_room:
+            # Check if a remote join should be performed.
+            remote_join, remote_room_hosts = await self._should_perform_remote_join(
+                target.to_string(), room_id, remote_room_hosts, content, is_host_in_room
+            )
+            if remote_join:
                 if ratelimit:
                     time_now_s = self.clock.time()
                     (
                         allowed,
                         time_allowed,
-                    ) = self._join_rate_limiter_remote.can_requester_do_action(
+                    ) = await self._join_rate_limiter_remote.can_do_action(
                         requester,
                     )
 
@@ -723,10 +853,112 @@ async def update_membership_locked(
             txn_id=txn_id,
             ratelimit=ratelimit,
             prev_event_ids=latest_event_ids,
+            auth_event_ids=auth_event_ids,
             content=content,
             require_consent=require_consent,
+            outlier=outlier,
+        )
+
+    async def _should_perform_remote_join(
+        self,
+        user_id: str,
+        room_id: str,
+        remote_room_hosts: List[str],
+        content: JsonDict,
+        is_host_in_room: bool,
+    ) -> Tuple[bool, List[str]]:
+        """
+        Check whether the server should do a remote join (as opposed to a local
+        join) for a user.
+
+        Generally a remote join is used if:
+
+        * The server is not yet in the room.
+        * The server is in the room, the room has restricted join rules, the user
+          is not joined or invited to the room, and the server does not have
+          another user who is capable of issuing invites.
+
+        Args:
+            user_id: The user joining the room.
+            room_id: The room being joined.
+            remote_room_hosts: A list of remote room hosts.
+            content: The content to use as the event body of the join. This may
+                be modified.
+            is_host_in_room: True if the host is in the room.
+
+        Returns:
+            A tuple of:
+                True if a remote join should be performed. False if the join can be
+                done locally.
+
+                A list of remote room hosts to use. This is an empty list if a
+                local join is to be done.
+        """
+        # If the host isn't in the room, pass through the prospective hosts.
+        if not is_host_in_room:
+            return True, remote_room_hosts
+
+        # If the host is in the room, but not one of the authorised hosts
+        # for restricted join rules, a remote join must be used.
+        room_version = await self.store.get_room_version(room_id)
+        current_state_ids = await self.store.get_current_state_ids(room_id)
+
+        # If restricted join rules are not being used, a local join can always
+        # be used.
+        if not await self.event_auth_handler.has_restricted_join_rules(
+            current_state_ids, room_version
+        ):
+            return False, []
+
+        # If the user is invited to the room or already joined, the join
+        # event can always be issued locally.
+        prev_member_event_id = current_state_ids.get((EventTypes.Member, user_id), None)
+        prev_member_event = None
+        if prev_member_event_id:
+            prev_member_event = await self.store.get_event(prev_member_event_id)
+            if prev_member_event.membership in (
+                Membership.JOIN,
+                Membership.INVITE,
+            ):
+                return False, []
+
+        # If the local host has a user who can issue invites, then a local
+        # join can be done.
+        #
+        # If not, generate a new list of remote hosts based on which
+        # can issue invites.
+        event_map = await self.store.get_events(current_state_ids.values())
+        current_state = {
+            state_key: event_map[event_id]
+            for state_key, event_id in current_state_ids.items()
+        }
+        allowed_servers = get_servers_from_users(
+            get_users_which_can_issue_invite(current_state)
         )
 
+        # If the local server is not one of allowed servers, then a remote
+        # join must be done. Return the list of prospective servers based on
+        # which can issue invites.
+        if self.hs.hostname not in allowed_servers:
+            return True, list(allowed_servers)
+
+        # Ensure the member should be allowed access via membership in a room.
+        await self.event_auth_handler.check_restricted_join_rules(
+            current_state_ids, room_version, user_id, prev_member_event
+        )
+
+        # If this is going to be a local join, additional information must
+        # be included in the event content in order to efficiently validate
+        # the event.
+        content[
+            "join_authorised_via_users_server"
+        ] = await self.event_auth_handler.get_user_which_could_invite(
+            room_id,
+            current_state_ids,
+        )
+
+        return False, []
+
     async def transfer_room_state_on_room_upgrade(
         self, old_room_id: str, room_id: str
     ) -> None:
@@ -1063,6 +1295,11 @@ async def _make_and_store_3pid_invite(
         if room_name_event:
             room_name = room_name_event.content.get("name", "")
 
+        room_type = None
+        room_create_event = room_state.get((EventTypes.Create, ""))
+        if room_create_event:
+            room_type = room_create_event.content.get(EventContentFields.ROOM_TYPE)
+
         room_join_rules = ""
         join_rules_event = room_state.get((EventTypes.JoinRules, ""))
         if join_rules_event:
@@ -1089,6 +1326,7 @@ async def _make_and_store_3pid_invite(
             room_avatar_url=room_avatar_url,
             room_join_rules=room_join_rules,
             room_name=room_name,
+            room_type=room_type,
             inviter_display_name=inviter_display_name,
             inviter_avatar_url=inviter_avatar_url,
             id_access_token=id_access_token,
@@ -1147,7 +1385,7 @@ async def _is_server_notice_room(self, room_id: str) -> bool:
 
 
 class RoomMemberMasterHandler(RoomMemberHandler):
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.distributor = hs.get_distributor()
@@ -1439,3 +1677,63 @@ async def forget(self, user: UserID, room_id: str) -> None:
 
         if membership:
             await self.store.forget(user_id, room_id)
+
+
+def get_users_which_can_issue_invite(auth_events: StateMap[EventBase]) -> List[str]:
+    """
+    Return the list of users which can issue invites.
+
+    This is done by exploring the joined users and comparing their power levels
+    to the necessyar power level to issue an invite.
+
+    Args:
+        auth_events: state in force at this point in the room
+
+    Returns:
+        The users which can issue invites.
+    """
+    invite_level = get_named_level(auth_events, "invite", 0)
+    users_default_level = get_named_level(auth_events, "users_default", 0)
+    power_level_event = get_power_level_event(auth_events)
+
+    # Custom power-levels for users.
+    if power_level_event:
+        users = power_level_event.content.get("users", {})
+    else:
+        users = {}
+
+    result = []
+
+    # Check which members are able to invite by ensuring they're joined and have
+    # the necessary power level.
+    for (event_type, state_key), event in auth_events.items():
+        if event_type != EventTypes.Member:
+            continue
+
+        if event.membership != Membership.JOIN:
+            continue
+
+        # Check if the user has a custom power level.
+        if users.get(state_key, users_default_level) >= invite_level:
+            result.append(state_key)
+
+    return result
+
+
+def get_servers_from_users(users: List[str]) -> Set[str]:
+    """
+    Resolve a list of users into their servers.
+
+    Args:
+        users: A list of users.
+
+    Returns:
+        A set of servers.
+    """
+    servers = set()
+    for user in users:
+        try:
+            servers.add(get_domain_from_id(user))
+        except SynapseError:
+            pass
+    return servers
diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py
index 9f2bde25dd..221552a2a6 100644
--- a/synapse/handlers/room_member_worker.py
+++ b/synapse/handlers/room_member_worker.py
@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
-# Copyright 2020 The Matrix.org Foundation C.I.C.
+# Copyright 2018-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py
new file mode 100644
index 0000000000..ac6cfc0da9
--- /dev/null
+++ b/synapse/handlers/room_summary.py
@@ -0,0 +1,1171 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+import logging
+import re
+from collections import deque
+from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Set, Tuple
+
+import attr
+
+from synapse.api.constants import (
+    EventContentFields,
+    EventTypes,
+    HistoryVisibility,
+    JoinRules,
+    Membership,
+    RoomTypes,
+)
+from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
+from synapse.events import EventBase
+from synapse.events.utils import format_event_for_client_v2
+from synapse.types import JsonDict
+from synapse.util.caches.response_cache import ResponseCache
+from synapse.util.stringutils import random_string
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+# number of rooms to return. We'll stop once we hit this limit.
+MAX_ROOMS = 50
+
+# max number of events to return per room.
+MAX_ROOMS_PER_SPACE = 50
+
+# max number of federation servers to hit per room
+MAX_SERVERS_PER_SPACE = 3
+
+
+@attr.s(slots=True, frozen=True, auto_attribs=True)
+class _PaginationKey:
+    """The key used to find unique pagination session."""
+
+    # The first three entries match the request parameters (and cannot change
+    # during a pagination session).
+    room_id: str
+    suggested_only: bool
+    max_depth: Optional[int]
+    # The randomly generated token.
+    token: str
+
+
+@attr.s(slots=True, frozen=True, auto_attribs=True)
+class _PaginationSession:
+    """The information that is stored for pagination."""
+
+    # The time the pagination session was created, in milliseconds.
+    creation_time_ms: int
+    # The queue of rooms which are still to process.
+    room_queue: List["_RoomQueueEntry"]
+    # A set of rooms which have been processed.
+    processed_rooms: Set[str]
+
+
+class RoomSummaryHandler:
+    # The time a pagination session remains valid for.
+    _PAGINATION_SESSION_VALIDITY_PERIOD_MS = 5 * 60 * 1000
+
+    def __init__(self, hs: "HomeServer"):
+        self._clock = hs.get_clock()
+        self._event_auth_handler = hs.get_event_auth_handler()
+        self._store = hs.get_datastore()
+        self._event_serializer = hs.get_event_client_serializer()
+        self._server_name = hs.hostname
+        self._federation_client = hs.get_federation_client()
+
+        # A map of query information to the current pagination state.
+        #
+        # TODO Allow for multiple workers to share this data.
+        # TODO Expire pagination tokens.
+        self._pagination_sessions: Dict[_PaginationKey, _PaginationSession] = {}
+
+        # If a user tries to fetch the same page multiple times in quick succession,
+        # only process the first attempt and return its result to subsequent requests.
+        self._pagination_response_cache: ResponseCache[
+            Tuple[str, bool, Optional[int], Optional[int], Optional[str]]
+        ] = ResponseCache(
+            hs.get_clock(),
+            "get_room_hierarchy",
+        )
+
+    def _expire_pagination_sessions(self):
+        """Expire pagination session which are old."""
+        expire_before = (
+            self._clock.time_msec() - self._PAGINATION_SESSION_VALIDITY_PERIOD_MS
+        )
+        to_expire = []
+
+        for key, value in self._pagination_sessions.items():
+            if value.creation_time_ms < expire_before:
+                to_expire.append(key)
+
+        for key in to_expire:
+            logger.debug("Expiring pagination session id %s", key)
+            del self._pagination_sessions[key]
+
+    async def get_space_summary(
+        self,
+        requester: str,
+        room_id: str,
+        suggested_only: bool = False,
+        max_rooms_per_space: Optional[int] = None,
+    ) -> JsonDict:
+        """
+        Implementation of the space summary C-S API
+
+        Args:
+            requester:  user id of the user making this request
+
+            room_id: room id to start the summary at
+
+            suggested_only: whether we should only return children with the "suggested"
+                flag set.
+
+            max_rooms_per_space: an optional limit on the number of child rooms we will
+                return. This does not apply to the root room (ie, room_id), and
+                is overridden by MAX_ROOMS_PER_SPACE.
+
+        Returns:
+            summary dict to return
+        """
+        # First of all, check that the room is accessible.
+        if not await self._is_local_room_accessible(room_id, requester):
+            raise AuthError(
+                403,
+                "User %s not in room %s, and room previews are disabled"
+                % (requester, room_id),
+            )
+
+        # the queue of rooms to process
+        room_queue = deque((_RoomQueueEntry(room_id, ()),))
+
+        # rooms we have already processed
+        processed_rooms: Set[str] = set()
+
+        # events we have already processed. We don't necessarily have their event ids,
+        # so instead we key on (room id, state key)
+        processed_events: Set[Tuple[str, str]] = set()
+
+        rooms_result: List[JsonDict] = []
+        events_result: List[JsonDict] = []
+
+        while room_queue and len(rooms_result) < MAX_ROOMS:
+            queue_entry = room_queue.popleft()
+            room_id = queue_entry.room_id
+            if room_id in processed_rooms:
+                # already done this room
+                continue
+
+            logger.debug("Processing room %s", room_id)
+
+            is_in_room = await self._store.is_host_joined(room_id, self._server_name)
+
+            # The client-specified max_rooms_per_space limit doesn't apply to the
+            # room_id specified in the request, so we ignore it if this is the
+            # first room we are processing.
+            max_children = max_rooms_per_space if processed_rooms else None
+
+            if is_in_room:
+                room_entry = await self._summarize_local_room(
+                    requester, None, room_id, suggested_only, max_children
+                )
+
+                events: Sequence[JsonDict] = []
+                if room_entry:
+                    rooms_result.append(room_entry.room)
+                    events = room_entry.children_state_events
+
+                logger.debug(
+                    "Query of local room %s returned events %s",
+                    room_id,
+                    ["%s->%s" % (ev["room_id"], ev["state_key"]) for ev in events],
+                )
+            else:
+                fed_rooms = await self._summarize_remote_room(
+                    queue_entry,
+                    suggested_only,
+                    max_children,
+                    exclude_rooms=processed_rooms,
+                )
+
+                # The results over federation might include rooms that the we,
+                # as the requesting server, are allowed to see, but the requesting
+                # user is not permitted see.
+                #
+                # Filter the returned results to only what is accessible to the user.
+                events = []
+                for room_entry in fed_rooms:
+                    room = room_entry.room
+                    fed_room_id = room_entry.room_id
+
+                    # The user can see the room, include it!
+                    if await self._is_remote_room_accessible(
+                        requester, fed_room_id, room
+                    ):
+                        # Before returning to the client, remove the allowed_room_ids
+                        # and allowed_spaces keys.
+                        room.pop("allowed_room_ids", None)
+                        room.pop("allowed_spaces", None)
+
+                        rooms_result.append(room)
+                        events.extend(room_entry.children_state_events)
+
+                    # All rooms returned don't need visiting again (even if the user
+                    # didn't have access to them).
+                    processed_rooms.add(fed_room_id)
+
+                logger.debug(
+                    "Query of %s returned rooms %s, events %s",
+                    room_id,
+                    [room_entry.room.get("room_id") for room_entry in fed_rooms],
+                    ["%s->%s" % (ev["room_id"], ev["state_key"]) for ev in events],
+                )
+
+            # the room we queried may or may not have been returned, but don't process
+            # it again, anyway.
+            processed_rooms.add(room_id)
+
+            # XXX: is it ok that we blindly iterate through any events returned by
+            #   a remote server, whether or not they actually link to any rooms in our
+            #   tree?
+            for ev in events:
+                # remote servers might return events we have already processed
+                # (eg, Dendrite returns inward pointers as well as outward ones), so
+                # we need to filter them out, to avoid returning duplicate links to the
+                # client.
+                ev_key = (ev["room_id"], ev["state_key"])
+                if ev_key in processed_events:
+                    continue
+                events_result.append(ev)
+
+                # add the child to the queue. we have already validated
+                # that the vias are a list of server names.
+                room_queue.append(
+                    _RoomQueueEntry(ev["state_key"], ev["content"]["via"])
+                )
+                processed_events.add(ev_key)
+
+        return {"rooms": rooms_result, "events": events_result}
+
+    async def get_room_hierarchy(
+        self,
+        requester: str,
+        requested_room_id: str,
+        suggested_only: bool = False,
+        max_depth: Optional[int] = None,
+        limit: Optional[int] = None,
+        from_token: Optional[str] = None,
+    ) -> JsonDict:
+        """
+        Implementation of the room hierarchy C-S API.
+
+        Args:
+            requester: The user ID of the user making this request.
+            requested_room_id: The room ID to start the hierarchy at (the "root" room).
+            suggested_only: Whether we should only return children with the "suggested"
+                flag set.
+            max_depth: The maximum depth in the tree to explore, must be a
+                non-negative integer.
+
+                0 would correspond to just the root room, 1 would include just
+                the root room's children, etc.
+            limit: An optional limit on the number of rooms to return per
+                page. Must be a positive integer.
+            from_token: An optional pagination token.
+
+        Returns:
+            The JSON hierarchy dictionary.
+        """
+        # If a user tries to fetch the same page multiple times in quick succession,
+        # only process the first attempt and return its result to subsequent requests.
+        #
+        # This is due to the pagination process mutating internal state, attempting
+        # to process multiple requests for the same page will result in errors.
+        return await self._pagination_response_cache.wrap(
+            (requested_room_id, suggested_only, max_depth, limit, from_token),
+            self._get_room_hierarchy,
+            requester,
+            requested_room_id,
+            suggested_only,
+            max_depth,
+            limit,
+            from_token,
+        )
+
+    async def _get_room_hierarchy(
+        self,
+        requester: str,
+        requested_room_id: str,
+        suggested_only: bool = False,
+        max_depth: Optional[int] = None,
+        limit: Optional[int] = None,
+        from_token: Optional[str] = None,
+    ) -> JsonDict:
+        """See docstring for SpaceSummaryHandler.get_room_hierarchy."""
+
+        # First of all, check that the room is accessible.
+        if not await self._is_local_room_accessible(requested_room_id, requester):
+            raise AuthError(
+                403,
+                "User %s not in room %s, and room previews are disabled"
+                % (requester, requested_room_id),
+            )
+
+        # If this is continuing a previous session, pull the persisted data.
+        if from_token:
+            self._expire_pagination_sessions()
+
+            pagination_key = _PaginationKey(
+                requested_room_id, suggested_only, max_depth, from_token
+            )
+            if pagination_key not in self._pagination_sessions:
+                raise SynapseError(400, "Unknown pagination token", Codes.INVALID_PARAM)
+
+            # Load the previous state.
+            pagination_session = self._pagination_sessions[pagination_key]
+            room_queue = pagination_session.room_queue
+            processed_rooms = pagination_session.processed_rooms
+        else:
+            # The queue of rooms to process, the next room is last on the stack.
+            room_queue = [_RoomQueueEntry(requested_room_id, ())]
+
+            # Rooms we have already processed.
+            processed_rooms = set()
+
+        rooms_result: List[JsonDict] = []
+
+        # Cap the limit to a server-side maximum.
+        if limit is None:
+            limit = MAX_ROOMS
+        else:
+            limit = min(limit, MAX_ROOMS)
+
+        # Iterate through the queue until we reach the limit or run out of
+        # rooms to include.
+        while room_queue and len(rooms_result) < limit:
+            queue_entry = room_queue.pop()
+            room_id = queue_entry.room_id
+            current_depth = queue_entry.depth
+            if room_id in processed_rooms:
+                # already done this room
+                continue
+
+            logger.debug("Processing room %s", room_id)
+
+            # A map of summaries for children rooms that might be returned over
+            # federation. The rationale for caching these and *maybe* using them
+            # is to prefer any information local to the homeserver before trusting
+            # data received over federation.
+            children_room_entries: Dict[str, JsonDict] = {}
+            # A set of room IDs which are children that did not have information
+            # returned over federation and are known to be inaccessible to the
+            # current server. We should not reach out over federation to try to
+            # summarise these rooms.
+            inaccessible_children: Set[str] = set()
+
+            # If the room is known locally, summarise it!
+            is_in_room = await self._store.is_host_joined(room_id, self._server_name)
+            if is_in_room:
+                room_entry = await self._summarize_local_room(
+                    requester,
+                    None,
+                    room_id,
+                    suggested_only,
+                    # TODO Handle max children.
+                    max_children=None,
+                )
+
+            # Otherwise, attempt to use information for federation.
+            else:
+                # A previous call might have included information for this room.
+                # It can be used if either:
+                #
+                # 1. The room is not a space.
+                # 2. The maximum depth has been achieved (since no children
+                #    information is needed).
+                if queue_entry.remote_room and (
+                    queue_entry.remote_room.get("room_type") != RoomTypes.SPACE
+                    or (max_depth is not None and current_depth >= max_depth)
+                ):
+                    room_entry = _RoomEntry(
+                        queue_entry.room_id, queue_entry.remote_room
+                    )
+
+                # If the above isn't true, attempt to fetch the room
+                # information over federation.
+                else:
+                    (
+                        room_entry,
+                        children_room_entries,
+                        inaccessible_children,
+                    ) = await self._summarize_remote_room_hierarchy(
+                        queue_entry,
+                        suggested_only,
+                    )
+
+                # Ensure this room is accessible to the requester (and not just
+                # the homeserver).
+                if room_entry and not await self._is_remote_room_accessible(
+                    requester, queue_entry.room_id, room_entry.room
+                ):
+                    room_entry = None
+
+            # This room has been processed and should be ignored if it appears
+            # elsewhere in the hierarchy.
+            processed_rooms.add(room_id)
+
+            # There may or may not be a room entry based on whether it is
+            # inaccessible to the requesting user.
+            if room_entry:
+                # Add the room (including the stripped m.space.child events).
+                rooms_result.append(room_entry.as_json())
+
+                # If this room is not at the max-depth, check if there are any
+                # children to process.
+                if max_depth is None or current_depth < max_depth:
+                    # The children get added in reverse order so that the next
+                    # room to process, according to the ordering, is the last
+                    # item in the list.
+                    room_queue.extend(
+                        _RoomQueueEntry(
+                            ev["state_key"],
+                            ev["content"]["via"],
+                            current_depth + 1,
+                            children_room_entries.get(ev["state_key"]),
+                        )
+                        for ev in reversed(room_entry.children_state_events)
+                        if ev["type"] == EventTypes.SpaceChild
+                        and ev["state_key"] not in inaccessible_children
+                    )
+
+        result: JsonDict = {"rooms": rooms_result}
+
+        # If there's additional data, generate a pagination token (and persist state).
+        if room_queue:
+            next_batch = random_string(24)
+            result["next_batch"] = next_batch
+            pagination_key = _PaginationKey(
+                requested_room_id, suggested_only, max_depth, next_batch
+            )
+            self._pagination_sessions[pagination_key] = _PaginationSession(
+                self._clock.time_msec(), room_queue, processed_rooms
+            )
+
+        return result
+
+    async def federation_space_summary(
+        self,
+        origin: str,
+        room_id: str,
+        suggested_only: bool,
+        max_rooms_per_space: Optional[int],
+        exclude_rooms: Iterable[str],
+    ) -> JsonDict:
+        """
+        Implementation of the space summary Federation API
+
+        Args:
+            origin: The server requesting the spaces summary.
+
+            room_id: room id to start the summary at
+
+            suggested_only: whether we should only return children with the "suggested"
+                flag set.
+
+            max_rooms_per_space: an optional limit on the number of child rooms we will
+                return. Unlike the C-S API, this applies to the root room (room_id).
+                It is clipped to MAX_ROOMS_PER_SPACE.
+
+            exclude_rooms: a list of rooms to skip over (presumably because the
+                calling server has already seen them).
+
+        Returns:
+            summary dict to return
+        """
+        # the queue of rooms to process
+        room_queue = deque((room_id,))
+
+        # the set of rooms that we should not walk further. Initialise it with the
+        # excluded-rooms list; we will add other rooms as we process them so that
+        # we do not loop.
+        processed_rooms: Set[str] = set(exclude_rooms)
+
+        rooms_result: List[JsonDict] = []
+        events_result: List[JsonDict] = []
+
+        while room_queue and len(rooms_result) < MAX_ROOMS:
+            room_id = room_queue.popleft()
+            if room_id in processed_rooms:
+                # already done this room
+                continue
+
+            room_entry = await self._summarize_local_room(
+                None, origin, room_id, suggested_only, max_rooms_per_space
+            )
+
+            processed_rooms.add(room_id)
+
+            if room_entry:
+                rooms_result.append(room_entry.room)
+                events_result.extend(room_entry.children_state_events)
+
+                # add any children to the queue
+                room_queue.extend(
+                    edge_event["state_key"]
+                    for edge_event in room_entry.children_state_events
+                )
+
+        return {"rooms": rooms_result, "events": events_result}
+
+    async def get_federation_hierarchy(
+        self,
+        origin: str,
+        requested_room_id: str,
+        suggested_only: bool,
+    ):
+        """
+        Implementation of the room hierarchy Federation API.
+
+        This is similar to get_room_hierarchy, but does not recurse into the space.
+        It also considers whether anyone on the server may be able to access the
+        room, as opposed to whether a specific user can.
+
+        Args:
+            origin: The server requesting the spaces summary.
+            requested_room_id: The room ID to start the hierarchy at (the "root" room).
+            suggested_only: whether we should only return children with the "suggested"
+                flag set.
+
+        Returns:
+            The JSON hierarchy dictionary.
+        """
+        root_room_entry = await self._summarize_local_room(
+            None, origin, requested_room_id, suggested_only, max_children=None
+        )
+        if root_room_entry is None:
+            # Room is inaccessible to the requesting server.
+            raise SynapseError(404, "Unknown room: %s" % (requested_room_id,))
+
+        children_rooms_result: List[JsonDict] = []
+        inaccessible_children: List[str] = []
+
+        # Iterate through each child and potentially add it, but not its children,
+        # to the response.
+        for child_room in root_room_entry.children_state_events:
+            room_id = child_room.get("state_key")
+            assert isinstance(room_id, str)
+            # If the room is unknown, skip it.
+            if not await self._store.is_host_joined(room_id, self._server_name):
+                continue
+
+            room_entry = await self._summarize_local_room(
+                None, origin, room_id, suggested_only, max_children=0
+            )
+            # If the room is accessible, include it in the results.
+            #
+            # Note that only the room summary (without information on children)
+            # is included in the summary.
+            if room_entry:
+                children_rooms_result.append(room_entry.room)
+            #  Otherwise, note that the requesting server shouldn't bother
+            #  trying to summarize this room - they do not have access to it.
+            else:
+                inaccessible_children.append(room_id)
+
+        return {
+            # Include the requested room (including the stripped children events).
+            "room": root_room_entry.as_json(),
+            "children": children_rooms_result,
+            "inaccessible_children": inaccessible_children,
+        }
+
+    async def _summarize_local_room(
+        self,
+        requester: Optional[str],
+        origin: Optional[str],
+        room_id: str,
+        suggested_only: bool,
+        max_children: Optional[int],
+    ) -> Optional["_RoomEntry"]:
+        """
+        Generate a room entry and a list of event entries for a given room.
+
+        Args:
+            requester:
+                The user requesting the summary, if it is a local request. None
+                if this is a federation request.
+            origin:
+                The server requesting the summary, if it is a federation request.
+                None if this is a local request.
+            room_id: The room ID to summarize.
+            suggested_only: True if only suggested children should be returned.
+                Otherwise, all children are returned.
+            max_children:
+                The maximum number of children rooms to include. This is capped
+                to a server-set limit.
+
+        Returns:
+            A room entry if the room should be returned. None, otherwise.
+        """
+        if not await self._is_local_room_accessible(room_id, requester, origin):
+            return None
+
+        room_entry = await self._build_room_entry(room_id, for_federation=bool(origin))
+
+        # If the room is not a space or the children don't matter, return just
+        # the room information.
+        if room_entry.get("room_type") != RoomTypes.SPACE or max_children == 0:
+            return _RoomEntry(room_id, room_entry)
+
+        # Otherwise, look for child rooms/spaces.
+        child_events = await self._get_child_events(room_id)
+
+        if suggested_only:
+            # we only care about suggested children
+            child_events = filter(_is_suggested_child_event, child_events)
+
+        if max_children is None or max_children > MAX_ROOMS_PER_SPACE:
+            max_children = MAX_ROOMS_PER_SPACE
+
+        now = self._clock.time_msec()
+        events_result: List[JsonDict] = []
+        for edge_event in itertools.islice(child_events, max_children):
+            events_result.append(
+                await self._event_serializer.serialize_event(
+                    edge_event,
+                    time_now=now,
+                    event_format=format_event_for_client_v2,
+                )
+            )
+
+        return _RoomEntry(room_id, room_entry, events_result)
+
+    async def _summarize_remote_room(
+        self,
+        room: "_RoomQueueEntry",
+        suggested_only: bool,
+        max_children: Optional[int],
+        exclude_rooms: Iterable[str],
+    ) -> Iterable["_RoomEntry"]:
+        """
+        Request room entries and a list of event entries for a given room by querying a remote server.
+
+        Args:
+            room: The room to summarize.
+            suggested_only: True if only suggested children should be returned.
+                Otherwise, all children are returned.
+            max_children:
+                The maximum number of children rooms to include. This is capped
+                to a server-set limit.
+            exclude_rooms:
+                Rooms IDs which do not need to be summarized.
+
+        Returns:
+            An iterable of room entries.
+        """
+        room_id = room.room_id
+        logger.info("Requesting summary for %s via %s", room_id, room.via)
+
+        # we need to make the exclusion list json-serialisable
+        exclude_rooms = list(exclude_rooms)
+
+        via = itertools.islice(room.via, MAX_SERVERS_PER_SPACE)
+        try:
+            res = await self._federation_client.get_space_summary(
+                via,
+                room_id,
+                suggested_only=suggested_only,
+                max_rooms_per_space=max_children,
+                exclude_rooms=exclude_rooms,
+            )
+        except Exception as e:
+            logger.warning(
+                "Unable to get summary of %s via federation: %s",
+                room_id,
+                e,
+                exc_info=logger.isEnabledFor(logging.DEBUG),
+            )
+            return ()
+
+        # Group the events by their room.
+        children_by_room: Dict[str, List[JsonDict]] = {}
+        for ev in res.events:
+            if ev.event_type == EventTypes.SpaceChild:
+                children_by_room.setdefault(ev.room_id, []).append(ev.data)
+
+        # Generate the final results.
+        results = []
+        for fed_room in res.rooms:
+            fed_room_id = fed_room.get("room_id")
+            if not fed_room_id or not isinstance(fed_room_id, str):
+                continue
+
+            results.append(
+                _RoomEntry(
+                    fed_room_id,
+                    fed_room,
+                    children_by_room.get(fed_room_id, []),
+                )
+            )
+
+        return results
+
+    async def _summarize_remote_room_hierarchy(
+        self, room: "_RoomQueueEntry", suggested_only: bool
+    ) -> Tuple[Optional["_RoomEntry"], Dict[str, JsonDict], Set[str]]:
+        """
+        Request room entries and a list of event entries for a given room by querying a remote server.
+
+        Args:
+            room: The room to summarize.
+            suggested_only: True if only suggested children should be returned.
+                Otherwise, all children are returned.
+
+        Returns:
+            A tuple of:
+                The room entry.
+                Partial room data return over federation.
+                A set of inaccessible children room IDs.
+        """
+        room_id = room.room_id
+        logger.info("Requesting summary for %s via %s", room_id, room.via)
+
+        via = itertools.islice(room.via, MAX_SERVERS_PER_SPACE)
+        try:
+            (
+                room_response,
+                children,
+                inaccessible_children,
+            ) = await self._federation_client.get_room_hierarchy(
+                via,
+                room_id,
+                suggested_only=suggested_only,
+            )
+        except Exception as e:
+            logger.warning(
+                "Unable to get hierarchy of %s via federation: %s",
+                room_id,
+                e,
+                exc_info=logger.isEnabledFor(logging.DEBUG),
+            )
+            return None, {}, set()
+
+        # Map the children to their room ID.
+        children_by_room_id = {
+            c["room_id"]: c
+            for c in children
+            if "room_id" in c and isinstance(c["room_id"], str)
+        }
+
+        return (
+            _RoomEntry(room_id, room_response, room_response.pop("children_state", ())),
+            children_by_room_id,
+            set(inaccessible_children),
+        )
+
+    async def _is_local_room_accessible(
+        self, room_id: str, requester: Optional[str], origin: Optional[str] = None
+    ) -> bool:
+        """
+        Calculate whether the room should be shown to the requester.
+
+        It should return true if:
+
+        * The requester is joined or can join the room (per MSC3173).
+        * The origin server has any user that is joined or can join the room.
+        * The history visibility is set to world readable.
+
+        Args:
+            room_id: The room ID to check accessibility of.
+            requester:
+                The user making the request, if it is a local request.
+                None if this is a federation request.
+            origin:
+                The server making the request, if it is a federation request.
+                None if this is a local request.
+
+        Returns:
+             True if the room is accessible to the requesting user or server.
+        """
+        state_ids = await self._store.get_current_state_ids(room_id)
+
+        # If there's no state for the room, it isn't known.
+        if not state_ids:
+            # The user might have a pending invite for the room.
+            if requester and await self._store.get_invite_for_local_user_in_room(
+                requester, room_id
+            ):
+                return True
+
+            logger.info("room %s is unknown, omitting from summary", room_id)
+            return False
+
+        room_version = await self._store.get_room_version(room_id)
+
+        # Include the room if it has join rules of public or knock.
+        join_rules_event_id = state_ids.get((EventTypes.JoinRules, ""))
+        if join_rules_event_id:
+            join_rules_event = await self._store.get_event(join_rules_event_id)
+            join_rule = join_rules_event.content.get("join_rule")
+            if join_rule == JoinRules.PUBLIC or (
+                room_version.msc2403_knocking and join_rule == JoinRules.KNOCK
+            ):
+                return True
+
+        # Include the room if it is peekable.
+        hist_vis_event_id = state_ids.get((EventTypes.RoomHistoryVisibility, ""))
+        if hist_vis_event_id:
+            hist_vis_ev = await self._store.get_event(hist_vis_event_id)
+            hist_vis = hist_vis_ev.content.get("history_visibility")
+            if hist_vis == HistoryVisibility.WORLD_READABLE:
+                return True
+
+        # Otherwise we need to check information specific to the user or server.
+
+        # If we have an authenticated requesting user, check if they are a member
+        # of the room (or can join the room).
+        if requester:
+            member_event_id = state_ids.get((EventTypes.Member, requester), None)
+
+            # If they're in the room they can see info on it.
+            if member_event_id:
+                member_event = await self._store.get_event(member_event_id)
+                if member_event.membership in (Membership.JOIN, Membership.INVITE):
+                    return True
+
+            # Otherwise, check if they should be allowed access via membership in a space.
+            if await self._event_auth_handler.has_restricted_join_rules(
+                state_ids, room_version
+            ):
+                allowed_rooms = (
+                    await self._event_auth_handler.get_rooms_that_allow_join(state_ids)
+                )
+                if await self._event_auth_handler.is_user_in_rooms(
+                    allowed_rooms, requester
+                ):
+                    return True
+
+        # If this is a request over federation, check if the host is in the room or
+        # has a user who could join the room.
+        elif origin:
+            if await self._event_auth_handler.check_host_in_room(
+                room_id, origin
+            ) or await self._store.is_host_invited(room_id, origin):
+                return True
+
+            # Alternately, if the host has a user in any of the spaces specified
+            # for access, then the host can see this room (and should do filtering
+            # if the requester cannot see it).
+            if await self._event_auth_handler.has_restricted_join_rules(
+                state_ids, room_version
+            ):
+                allowed_rooms = (
+                    await self._event_auth_handler.get_rooms_that_allow_join(state_ids)
+                )
+                for space_id in allowed_rooms:
+                    if await self._event_auth_handler.check_host_in_room(
+                        space_id, origin
+                    ):
+                        return True
+
+        logger.info(
+            "room %s is unpeekable and requester %s is not a member / not allowed to join, omitting from summary",
+            room_id,
+            requester or origin,
+        )
+        return False
+
+    async def _is_remote_room_accessible(
+        self, requester: str, room_id: str, room: JsonDict
+    ) -> bool:
+        """
+        Calculate whether the room received over federation should be shown to the requester.
+
+        It should return true if:
+
+        * The requester is joined or can join the room (per MSC3173).
+        * The history visibility is set to world readable.
+
+        Note that the local server is not in the requested room (which is why the
+        remote call was made in the first place), but the user could have access
+        due to an invite, etc.
+
+        Args:
+            requester: The user requesting the summary.
+            room_id: The room ID returned over federation.
+            room: The summary of the room returned over federation.
+
+        Returns:
+            True if the room is accessible to the requesting user.
+        """
+        # The API doesn't return the room version so assume that a
+        # join rule of knock is valid.
+        if (
+            room.get("join_rules") in (JoinRules.PUBLIC, JoinRules.KNOCK)
+            or room.get("world_readable") is True
+        ):
+            return True
+
+        # Check if the user is a member of any of the allowed spaces
+        # from the response.
+        allowed_rooms = room.get("allowed_room_ids") or room.get("allowed_spaces")
+        if allowed_rooms and isinstance(allowed_rooms, list):
+            if await self._event_auth_handler.is_user_in_rooms(
+                allowed_rooms, requester
+            ):
+                return True
+
+        # Finally, check locally if we can access the room. The user might
+        # already be in the room (if it was a child room), or there might be a
+        # pending invite, etc.
+        return await self._is_local_room_accessible(room_id, requester)
+
+    async def _build_room_entry(self, room_id: str, for_federation: bool) -> JsonDict:
+        """
+        Generate en entry summarising a single room.
+
+        Args:
+            room_id: The room ID to summarize.
+            for_federation: True if this is a summary requested over federation
+                (which includes additional fields).
+
+        Returns:
+            The JSON dictionary for the room.
+        """
+        stats = await self._store.get_room_with_stats(room_id)
+
+        # currently this should be impossible because we call
+        # _is_local_room_accessible on the room before we get here, so
+        # there should always be an entry
+        assert stats is not None, "unable to retrieve stats for %s" % (room_id,)
+
+        current_state_ids = await self._store.get_current_state_ids(room_id)
+        create_event = await self._store.get_event(
+            current_state_ids[(EventTypes.Create, "")]
+        )
+
+        entry = {
+            "room_id": stats["room_id"],
+            "name": stats["name"],
+            "topic": stats["topic"],
+            "canonical_alias": stats["canonical_alias"],
+            "num_joined_members": stats["joined_members"],
+            "avatar_url": stats["avatar"],
+            "join_rules": stats["join_rules"],
+            "world_readable": (
+                stats["history_visibility"] == HistoryVisibility.WORLD_READABLE
+            ),
+            "guest_can_join": stats["guest_access"] == "can_join",
+            "creation_ts": create_event.origin_server_ts,
+            "room_type": create_event.content.get(EventContentFields.ROOM_TYPE),
+        }
+
+        # Federation requests need to provide additional information so the
+        # requested server is able to filter the response appropriately.
+        if for_federation:
+            room_version = await self._store.get_room_version(room_id)
+            if await self._event_auth_handler.has_restricted_join_rules(
+                current_state_ids, room_version
+            ):
+                allowed_rooms = (
+                    await self._event_auth_handler.get_rooms_that_allow_join(
+                        current_state_ids
+                    )
+                )
+                if allowed_rooms:
+                    entry["allowed_room_ids"] = allowed_rooms
+                    # TODO Remove this key once the API is stable.
+                    entry["allowed_spaces"] = allowed_rooms
+
+        # Filter out Nones – rather omit the field altogether
+        room_entry = {k: v for k, v in entry.items() if v is not None}
+
+        return room_entry
+
+    async def _get_child_events(self, room_id: str) -> Iterable[EventBase]:
+        """
+        Get the child events for a given room.
+
+        The returned results are sorted for stability.
+
+        Args:
+            room_id: The room id to get the children of.
+
+        Returns:
+            An iterable of sorted child events.
+        """
+
+        # look for child rooms/spaces.
+        current_state_ids = await self._store.get_current_state_ids(room_id)
+
+        events = await self._store.get_events_as_list(
+            [
+                event_id
+                for key, event_id in current_state_ids.items()
+                if key[0] == EventTypes.SpaceChild
+            ]
+        )
+
+        # filter out any events without a "via" (which implies it has been redacted),
+        # and order to ensure we return stable results.
+        return sorted(filter(_has_valid_via, events), key=_child_events_comparison_key)
+
+    async def get_room_summary(
+        self,
+        requester: Optional[str],
+        room_id: str,
+        remote_room_hosts: Optional[List[str]] = None,
+    ) -> JsonDict:
+        """
+        Implementation of the room summary C-S API from MSC3266
+
+        Args:
+            requester:  user id of the user making this request, will be None
+                for unauthenticated requests
+
+            room_id: room id to summarise.
+
+            remote_room_hosts: a list of homeservers to try fetching data through
+                if we don't know it ourselves
+
+        Returns:
+            summary dict to return
+        """
+        is_in_room = await self._store.is_host_joined(room_id, self._server_name)
+
+        if is_in_room:
+            room_entry = await self._summarize_local_room(
+                requester,
+                None,
+                room_id,
+                # Suggested-only doesn't matter since no children are requested.
+                suggested_only=False,
+                max_children=0,
+            )
+
+            if not room_entry:
+                raise NotFoundError("Room not found or is not accessible")
+
+            room_summary = room_entry.room
+
+            # If there was a requester, add their membership.
+            if requester:
+                (
+                    membership,
+                    _,
+                ) = await self._store.get_local_current_membership_for_user_in_room(
+                    requester, room_id
+                )
+
+                room_summary["membership"] = membership or "leave"
+        else:
+            # TODO federation API, descoped from initial unstable implementation
+            #      as MSC needs more maturing on that side.
+            raise SynapseError(400, "Federation is not currently supported.")
+
+        return room_summary
+
+
+@attr.s(frozen=True, slots=True, auto_attribs=True)
+class _RoomQueueEntry:
+    # The room ID of this entry.
+    room_id: str
+    # The server to query if the room is not known locally.
+    via: Sequence[str]
+    # The minimum number of hops necessary to get to this room (compared to the
+    # originally requested room).
+    depth: int = 0
+    # The room summary for this room returned via federation. This will only be
+    # used if the room is not known locally (and is not a space).
+    remote_room: Optional[JsonDict] = None
+
+
+@attr.s(frozen=True, slots=True, auto_attribs=True)
+class _RoomEntry:
+    room_id: str
+    # The room summary for this room.
+    room: JsonDict
+    # An iterable of the sorted, stripped children events for children of this room.
+    #
+    # This may not include all children.
+    children_state_events: Sequence[JsonDict] = ()
+
+    def as_json(self) -> JsonDict:
+        """
+        Returns a JSON dictionary suitable for the room hierarchy endpoint.
+
+        It returns the room summary including the stripped m.space.child events
+        as a sub-key.
+        """
+        result = dict(self.room)
+        result["children_state"] = self.children_state_events
+        return result
+
+
+def _has_valid_via(e: EventBase) -> bool:
+    via = e.content.get("via")
+    if not via or not isinstance(via, Sequence):
+        return False
+    for v in via:
+        if not isinstance(v, str):
+            logger.debug("Ignoring edge event %s with invalid via entry", e.event_id)
+            return False
+    return True
+
+
+def _is_suggested_child_event(edge_event: EventBase) -> bool:
+    suggested = edge_event.content.get("suggested")
+    if isinstance(suggested, bool) and suggested:
+        return True
+    logger.debug("Ignorning not-suggested child %s", edge_event.state_key)
+    return False
+
+
+# Order may only contain characters in the range of \x20 (space) to \x7E (~) inclusive.
+_INVALID_ORDER_CHARS_RE = re.compile(r"[^\x20-\x7E]")
+
+
+def _child_events_comparison_key(child: EventBase) -> Tuple[bool, Optional[str], str]:
+    """
+    Generate a value for comparing two child events for ordering.
+
+    The rules for ordering are supposed to be:
+
+    1. The 'order' key, if it is valid.
+    2. The 'origin_server_ts' of the 'm.room.create' event.
+    3. The 'room_id'.
+
+    But we skip step 2 since we may not have any state from the room.
+
+    Args:
+        child: The event for generating a comparison key.
+
+    Returns:
+        The comparison key as a tuple of:
+            False if the ordering is valid.
+            The ordering field.
+            The room ID.
+    """
+    order = child.content.get("order")
+    # If order is not a string or doesn't meet the requirements, ignore it.
+    if not isinstance(order, str):
+        order = None
+    elif len(order) > 50 or _INVALID_ORDER_CHARS_RE.search(order):
+        order = None
+
+    # Items without an order come last.
+    return (order is None, order, child.room_id)
diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml.py
similarity index 98%
rename from synapse/handlers/saml_handler.py
rename to synapse/handlers/saml.py
index ec2ba11c75..e6e71e9729 100644
--- a/synapse/handlers/saml_handler.py
+++ b/synapse/handlers/saml.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -84,7 +83,7 @@ def __init__(self, hs: "HomeServer"):
         self.unstable_idp_brand = None
 
         # a map from saml session id to Saml2SessionData object
-        self._outstanding_requests_dict = {}  # type: Dict[str, Saml2SessionData]
+        self._outstanding_requests_dict: Dict[str, Saml2SessionData] = {}
 
         self._sso_handler = hs.get_sso_handler()
         self._sso_handler.register_identity_provider(self)
@@ -373,7 +372,7 @@ def expire_sessions(self):
 
 
 DOT_REPLACE_PATTERN = re.compile(
-    ("[^%s]" % (re.escape("".join(mxid_localpart_allowed_characters)),))
+    "[^%s]" % (re.escape("".join(mxid_localpart_allowed_characters)),)
 )
 
 
@@ -387,10 +386,10 @@ def dot_replace_for_mxid(username: str) -> str:
     return username
 
 
-MXID_MAPPER_MAP = {
+MXID_MAPPER_MAP: Dict[str, Callable[[str], str]] = {
     "hexencode": map_username_to_mxid_localpart,
     "dotreplace": dot_replace_for_mxid,
-}  # type: Dict[str, Callable[[str], str]]
+}
 
 
 @attr.s
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index d742dfbd53..8226d6f5a1 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -193,7 +192,7 @@ async def search(
         # If doing a subset of all rooms seearch, check if any of the rooms
         # are from an upgraded room, and search their contents as well
         if search_filter.rooms:
-            historical_room_ids = []  # type: List[str]
+            historical_room_ids: List[str] = []
             for room_id in search_filter.rooms:
                 # Add any previous rooms to the search if they exist
                 ids = await self.get_old_rooms_from_upgraded_room(room_id)
@@ -217,9 +216,9 @@ async def search(
         rank_map = {}  # event_id -> rank of event
         allowed_events = []
         # Holds result of grouping by room, if applicable
-        room_groups = {}  # type: Dict[str, JsonDict]
+        room_groups: Dict[str, JsonDict] = {}
         # Holds result of grouping by sender, if applicable
-        sender_group = {}  # type: Dict[str, JsonDict]
+        sender_group: Dict[str, JsonDict] = {}
 
         # Holds the next_batch for the entire result set if one of those exists
         global_next_batch = None
@@ -263,7 +262,7 @@ async def search(
                 s["results"].append(e.event_id)
 
         elif order_by == "recent":
-            room_events = []  # type: List[EventBase]
+            room_events: List[EventBase] = []
             i = 0
 
             pagination_token = batch_token
diff --git a/synapse/handlers/send_email.py b/synapse/handlers/send_email.py
new file mode 100644
index 0000000000..a31fe3e3c7
--- /dev/null
+++ b/synapse/handlers/send_email.py
@@ -0,0 +1,187 @@
+# Copyright 2021 The Matrix.org C.I.C. Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import email.utils
+import logging
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from io import BytesIO
+from typing import TYPE_CHECKING, Optional
+
+from pkg_resources import parse_version
+
+import twisted
+from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import IOpenSSLContextFactory, IReactorTCP
+from twisted.mail.smtp import ESMTPSender, ESMTPSenderFactory
+
+from synapse.logging.context import make_deferred_yieldable
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+_is_old_twisted = parse_version(twisted.__version__) < parse_version("21")
+
+
+class _NoTLSESMTPSender(ESMTPSender):
+    """Extend ESMTPSender to disable TLS
+
+    Unfortunately, before Twisted 21.2, ESMTPSender doesn't give an easy way to disable
+    TLS, so we override its internal method which it uses to generate a context factory.
+    """
+
+    def _getContextFactory(self) -> Optional[IOpenSSLContextFactory]:
+        return None
+
+
+async def _sendmail(
+    reactor: IReactorTCP,
+    smtphost: str,
+    smtpport: int,
+    from_addr: str,
+    to_addr: str,
+    msg_bytes: bytes,
+    username: Optional[bytes] = None,
+    password: Optional[bytes] = None,
+    require_auth: bool = False,
+    require_tls: bool = False,
+    enable_tls: bool = True,
+) -> None:
+    """A simple wrapper around ESMTPSenderFactory, to allow substitution in tests
+
+    Params:
+        reactor: reactor to use to make the outbound connection
+        smtphost: hostname to connect to
+        smtpport: port to connect to
+        from_addr: "From" address for email
+        to_addr: "To" address for email
+        msg_bytes: Message content
+        username: username to authenticate with, if auth is enabled
+        password: password to give when authenticating
+        require_auth: if auth is not offered, fail the request
+        require_tls: if TLS is not offered, fail the reqest
+        enable_tls: True to enable TLS. If this is False and require_tls is True,
+           the request will fail.
+    """
+    msg = BytesIO(msg_bytes)
+    d: "Deferred[object]" = Deferred()
+
+    def build_sender_factory(**kwargs) -> ESMTPSenderFactory:
+        return ESMTPSenderFactory(
+            username,
+            password,
+            from_addr,
+            to_addr,
+            msg,
+            d,
+            heloFallback=True,
+            requireAuthentication=require_auth,
+            requireTransportSecurity=require_tls,
+            **kwargs,
+        )
+
+    if _is_old_twisted:
+        # before twisted 21.2, we have to override the ESMTPSender protocol to disable
+        # TLS
+        factory = build_sender_factory()
+
+        if not enable_tls:
+            factory.protocol = _NoTLSESMTPSender
+    else:
+        # for twisted 21.2 and later, there is a 'hostname' parameter which we should
+        # set to enable TLS.
+        factory = build_sender_factory(hostname=smtphost if enable_tls else None)
+
+    # the IReactorTCP interface claims host has to be a bytes, which seems to be wrong
+    reactor.connectTCP(smtphost, smtpport, factory, timeout=30, bindAddress=None)  # type: ignore[arg-type]
+
+    await make_deferred_yieldable(d)
+
+
+class SendEmailHandler:
+    def __init__(self, hs: "HomeServer"):
+        self.hs = hs
+
+        self._reactor = hs.get_reactor()
+
+        self._from = hs.config.email.email_notif_from
+        self._smtp_host = hs.config.email.email_smtp_host
+        self._smtp_port = hs.config.email.email_smtp_port
+
+        user = hs.config.email.email_smtp_user
+        self._smtp_user = user.encode("utf-8") if user is not None else None
+        passwd = hs.config.email.email_smtp_pass
+        self._smtp_pass = passwd.encode("utf-8") if passwd is not None else None
+        self._require_transport_security = hs.config.email.require_transport_security
+        self._enable_tls = hs.config.email.enable_smtp_tls
+
+        self._sendmail = _sendmail
+
+    async def send_email(
+        self,
+        email_address: str,
+        subject: str,
+        app_name: str,
+        html: str,
+        text: str,
+    ) -> None:
+        """Send a multipart email with the given information.
+
+        Args:
+            email_address: The address to send the email to.
+            subject: The email's subject.
+            app_name: The app name to include in the From header.
+            html: The HTML content to include in the email.
+            text: The plain text content to include in the email.
+        """
+        try:
+            from_string = self._from % {"app": app_name}
+        except (KeyError, TypeError):
+            from_string = self._from
+
+        raw_from = email.utils.parseaddr(from_string)[1]
+        raw_to = email.utils.parseaddr(email_address)[1]
+
+        if raw_to == "":
+            raise RuntimeError("Invalid 'to' address")
+
+        html_part = MIMEText(html, "html", "utf8")
+        text_part = MIMEText(text, "plain", "utf8")
+
+        multipart_msg = MIMEMultipart("alternative")
+        multipart_msg["Subject"] = subject
+        multipart_msg["From"] = from_string
+        multipart_msg["To"] = email_address
+        multipart_msg["Date"] = email.utils.formatdate()
+        multipart_msg["Message-ID"] = email.utils.make_msgid()
+        multipart_msg.attach(text_part)
+        multipart_msg.attach(html_part)
+
+        logger.info("Sending email to %s" % email_address)
+
+        await self._sendmail(
+            self._reactor,
+            self._smtp_host,
+            self._smtp_port,
+            raw_from,
+            raw_to,
+            multipart_msg.as_string().encode("utf8"),
+            username=self._smtp_user,
+            password=self._smtp_pass,
+            require_auth=self._smtp_user is not None,
+            require_tls=self._require_transport_security,
+            enable_tls=self._enable_tls,
+        )
diff --git a/synapse/handlers/set_password.py b/synapse/handlers/set_password.py
index 6243fab091..56207c7059 100644
--- a/synapse/handlers/set_password.py
+++ b/synapse/handlers/set_password.py
@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2017-2018 New Vector Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py
deleted file mode 100644
index 5d9418969d..0000000000
--- a/synapse/handlers/space_summary.py
+++ /dev/null
@@ -1,395 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2021 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import itertools
-import logging
-from collections import deque
-from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence, Set, Tuple, cast
-
-import attr
-
-from synapse.api.constants import EventContentFields, EventTypes, HistoryVisibility
-from synapse.api.errors import AuthError
-from synapse.events import EventBase
-from synapse.events.utils import format_event_for_client_v2
-from synapse.types import JsonDict
-
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
-logger = logging.getLogger(__name__)
-
-# number of rooms to return. We'll stop once we hit this limit.
-# TODO: allow clients to reduce this with a request param.
-MAX_ROOMS = 50
-
-# max number of events to return per room.
-MAX_ROOMS_PER_SPACE = 50
-
-# max number of federation servers to hit per room
-MAX_SERVERS_PER_SPACE = 3
-
-
-class SpaceSummaryHandler:
-    def __init__(self, hs: "HomeServer"):
-        self._clock = hs.get_clock()
-        self._auth = hs.get_auth()
-        self._room_list_handler = hs.get_room_list_handler()
-        self._state_handler = hs.get_state_handler()
-        self._store = hs.get_datastore()
-        self._event_serializer = hs.get_event_client_serializer()
-        self._server_name = hs.hostname
-        self._federation_client = hs.get_federation_client()
-
-    async def get_space_summary(
-        self,
-        requester: str,
-        room_id: str,
-        suggested_only: bool = False,
-        max_rooms_per_space: Optional[int] = None,
-    ) -> JsonDict:
-        """
-        Implementation of the space summary C-S API
-
-        Args:
-            requester:  user id of the user making this request
-
-            room_id: room id to start the summary at
-
-            suggested_only: whether we should only return children with the "suggested"
-                flag set.
-
-            max_rooms_per_space: an optional limit on the number of child rooms we will
-                return. This does not apply to the root room (ie, room_id), and
-                is overridden by MAX_ROOMS_PER_SPACE.
-
-        Returns:
-            summary dict to return
-        """
-        # first of all, check that the user is in the room in question (or it's
-        # world-readable)
-        await self._auth.check_user_in_room_or_world_readable(room_id, requester)
-
-        # the queue of rooms to process
-        room_queue = deque((_RoomQueueEntry(room_id, ()),))
-
-        # rooms we have already processed
-        processed_rooms = set()  # type: Set[str]
-
-        # events we have already processed. We don't necessarily have their event ids,
-        # so instead we key on (room id, state key)
-        processed_events = set()  # type: Set[Tuple[str, str]]
-
-        rooms_result = []  # type: List[JsonDict]
-        events_result = []  # type: List[JsonDict]
-
-        while room_queue and len(rooms_result) < MAX_ROOMS:
-            queue_entry = room_queue.popleft()
-            room_id = queue_entry.room_id
-            if room_id in processed_rooms:
-                # already done this room
-                continue
-
-            logger.debug("Processing room %s", room_id)
-
-            is_in_room = await self._store.is_host_joined(room_id, self._server_name)
-
-            # The client-specified max_rooms_per_space limit doesn't apply to the
-            # room_id specified in the request, so we ignore it if this is the
-            # first room we are processing.
-            max_children = max_rooms_per_space if processed_rooms else None
-
-            if is_in_room:
-                rooms, events = await self._summarize_local_room(
-                    requester, room_id, suggested_only, max_children
-                )
-            else:
-                rooms, events = await self._summarize_remote_room(
-                    queue_entry,
-                    suggested_only,
-                    max_children,
-                    exclude_rooms=processed_rooms,
-                )
-
-            logger.debug(
-                "Query of %s returned rooms %s, events %s",
-                queue_entry.room_id,
-                [room.get("room_id") for room in rooms],
-                ["%s->%s" % (ev["room_id"], ev["state_key"]) for ev in events],
-            )
-
-            rooms_result.extend(rooms)
-
-            # any rooms returned don't need visiting again
-            processed_rooms.update(cast(str, room.get("room_id")) for room in rooms)
-
-            # the room we queried may or may not have been returned, but don't process
-            # it again, anyway.
-            processed_rooms.add(room_id)
-
-            # XXX: is it ok that we blindly iterate through any events returned by
-            #   a remote server, whether or not they actually link to any rooms in our
-            #   tree?
-            for ev in events:
-                # remote servers might return events we have already processed
-                # (eg, Dendrite returns inward pointers as well as outward ones), so
-                # we need to filter them out, to avoid returning duplicate links to the
-                # client.
-                ev_key = (ev["room_id"], ev["state_key"])
-                if ev_key in processed_events:
-                    continue
-                events_result.append(ev)
-
-                # add the child to the queue. we have already validated
-                # that the vias are a list of server names.
-                room_queue.append(
-                    _RoomQueueEntry(ev["state_key"], ev["content"]["via"])
-                )
-                processed_events.add(ev_key)
-
-        return {"rooms": rooms_result, "events": events_result}
-
-    async def federation_space_summary(
-        self,
-        room_id: str,
-        suggested_only: bool,
-        max_rooms_per_space: Optional[int],
-        exclude_rooms: Iterable[str],
-    ) -> JsonDict:
-        """
-        Implementation of the space summary Federation API
-
-        Args:
-            room_id: room id to start the summary at
-
-            suggested_only: whether we should only return children with the "suggested"
-                flag set.
-
-            max_rooms_per_space: an optional limit on the number of child rooms we will
-                return. Unlike the C-S API, this applies to the root room (room_id).
-                It is clipped to MAX_ROOMS_PER_SPACE.
-
-            exclude_rooms: a list of rooms to skip over (presumably because the
-                calling server has already seen them).
-
-        Returns:
-            summary dict to return
-        """
-        # the queue of rooms to process
-        room_queue = deque((room_id,))
-
-        # the set of rooms that we should not walk further. Initialise it with the
-        # excluded-rooms list; we will add other rooms as we process them so that
-        # we do not loop.
-        processed_rooms = set(exclude_rooms)  # type: Set[str]
-
-        rooms_result = []  # type: List[JsonDict]
-        events_result = []  # type: List[JsonDict]
-
-        while room_queue and len(rooms_result) < MAX_ROOMS:
-            room_id = room_queue.popleft()
-            if room_id in processed_rooms:
-                # already done this room
-                continue
-
-            logger.debug("Processing room %s", room_id)
-
-            rooms, events = await self._summarize_local_room(
-                None, room_id, suggested_only, max_rooms_per_space
-            )
-
-            processed_rooms.add(room_id)
-
-            rooms_result.extend(rooms)
-            events_result.extend(events)
-
-            # add any children to the queue
-            room_queue.extend(edge_event["state_key"] for edge_event in events)
-
-        return {"rooms": rooms_result, "events": events_result}
-
-    async def _summarize_local_room(
-        self,
-        requester: Optional[str],
-        room_id: str,
-        suggested_only: bool,
-        max_children: Optional[int],
-    ) -> Tuple[Sequence[JsonDict], Sequence[JsonDict]]:
-        if not await self._is_room_accessible(room_id, requester):
-            return (), ()
-
-        room_entry = await self._build_room_entry(room_id)
-
-        # look for child rooms/spaces.
-        child_events = await self._get_child_events(room_id)
-
-        if suggested_only:
-            # we only care about suggested children
-            child_events = filter(_is_suggested_child_event, child_events)
-
-        if max_children is None or max_children > MAX_ROOMS_PER_SPACE:
-            max_children = MAX_ROOMS_PER_SPACE
-
-        now = self._clock.time_msec()
-        events_result = []  # type: List[JsonDict]
-        for edge_event in itertools.islice(child_events, max_children):
-            events_result.append(
-                await self._event_serializer.serialize_event(
-                    edge_event,
-                    time_now=now,
-                    event_format=format_event_for_client_v2,
-                )
-            )
-        return (room_entry,), events_result
-
-    async def _summarize_remote_room(
-        self,
-        room: "_RoomQueueEntry",
-        suggested_only: bool,
-        max_children: Optional[int],
-        exclude_rooms: Iterable[str],
-    ) -> Tuple[Sequence[JsonDict], Sequence[JsonDict]]:
-        room_id = room.room_id
-        logger.info("Requesting summary for %s via %s", room_id, room.via)
-
-        # we need to make the exclusion list json-serialisable
-        exclude_rooms = list(exclude_rooms)
-
-        via = itertools.islice(room.via, MAX_SERVERS_PER_SPACE)
-        try:
-            res = await self._federation_client.get_space_summary(
-                via,
-                room_id,
-                suggested_only=suggested_only,
-                max_rooms_per_space=max_children,
-                exclude_rooms=exclude_rooms,
-            )
-        except Exception as e:
-            logger.warning(
-                "Unable to get summary of %s via federation: %s",
-                room_id,
-                e,
-                exc_info=logger.isEnabledFor(logging.DEBUG),
-            )
-            return (), ()
-
-        return res.rooms, tuple(
-            ev.data
-            for ev in res.events
-            if ev.event_type == EventTypes.MSC1772_SPACE_CHILD
-        )
-
-    async def _is_room_accessible(self, room_id: str, requester: Optional[str]) -> bool:
-        # if we have an authenticated requesting user, first check if they are in the
-        # room
-        if requester:
-            try:
-                await self._auth.check_user_in_room(room_id, requester)
-                return True
-            except AuthError:
-                pass
-
-        # otherwise, check if the room is peekable
-        hist_vis_ev = await self._state_handler.get_current_state(
-            room_id, EventTypes.RoomHistoryVisibility, ""
-        )
-        if hist_vis_ev:
-            hist_vis = hist_vis_ev.content.get("history_visibility")
-            if hist_vis == HistoryVisibility.WORLD_READABLE:
-                return True
-
-        logger.info(
-            "room %s is unpeekable and user %s is not a member, omitting from summary",
-            room_id,
-            requester,
-        )
-        return False
-
-    async def _build_room_entry(self, room_id: str) -> JsonDict:
-        """Generate en entry suitable for the 'rooms' list in the summary response"""
-        stats = await self._store.get_room_with_stats(room_id)
-
-        # currently this should be impossible because we call
-        # check_user_in_room_or_world_readable on the room before we get here, so
-        # there should always be an entry
-        assert stats is not None, "unable to retrieve stats for %s" % (room_id,)
-
-        current_state_ids = await self._store.get_current_state_ids(room_id)
-        create_event = await self._store.get_event(
-            current_state_ids[(EventTypes.Create, "")]
-        )
-
-        # TODO: update once MSC1772 lands
-        room_type = create_event.content.get(EventContentFields.MSC1772_ROOM_TYPE)
-
-        entry = {
-            "room_id": stats["room_id"],
-            "name": stats["name"],
-            "topic": stats["topic"],
-            "canonical_alias": stats["canonical_alias"],
-            "num_joined_members": stats["joined_members"],
-            "avatar_url": stats["avatar"],
-            "world_readable": (
-                stats["history_visibility"] == HistoryVisibility.WORLD_READABLE
-            ),
-            "guest_can_join": stats["guest_access"] == "can_join",
-            "room_type": room_type,
-        }
-
-        # Filter out Nones – rather omit the field altogether
-        room_entry = {k: v for k, v in entry.items() if v is not None}
-
-        return room_entry
-
-    async def _get_child_events(self, room_id: str) -> Iterable[EventBase]:
-        # look for child rooms/spaces.
-        current_state_ids = await self._store.get_current_state_ids(room_id)
-
-        events = await self._store.get_events_as_list(
-            [
-                event_id
-                for key, event_id in current_state_ids.items()
-                # TODO: update once MSC1772 lands
-                if key[0] == EventTypes.MSC1772_SPACE_CHILD
-            ]
-        )
-
-        # filter out any events without a "via" (which implies it has been redacted)
-        return (e for e in events if _has_valid_via(e))
-
-
-@attr.s(frozen=True, slots=True)
-class _RoomQueueEntry:
-    room_id = attr.ib(type=str)
-    via = attr.ib(type=Sequence[str])
-
-
-def _has_valid_via(e: EventBase) -> bool:
-    via = e.content.get("via")
-    if not via or not isinstance(via, Sequence):
-        return False
-    for v in via:
-        if not isinstance(v, str):
-            logger.debug("Ignoring edge event %s with invalid via entry", e.event_id)
-            return False
-    return True
-
-
-def _is_suggested_child_event(edge_event: EventBase) -> bool:
-    suggested = edge_event.content.get("suggested")
-    if isinstance(suggested, bool) and suggested:
-        return True
-    logger.debug("Ignorning not-suggested child %s", edge_event.state_key)
-    return False
diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py
index 415b1c2d17..1b855a685c 100644
--- a/synapse/handlers/sso.py
+++ b/synapse/handlers/sso.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,6 +18,7 @@
     Any,
     Awaitable,
     Callable,
+    Collection,
     Dict,
     Iterable,
     List,
@@ -41,7 +41,12 @@
 from synapse.http import get_request_user_agent
 from synapse.http.server import respond_with_html, respond_with_redirect
 from synapse.http.site import SynapseRequest
-from synapse.types import Collection, JsonDict, UserID, contains_invalid_mxid_characters
+from synapse.types import (
+    JsonDict,
+    UserID,
+    contains_invalid_mxid_characters,
+    create_requester,
+)
 from synapse.util.async_helpers import Linearizer
 from synapse.util.stringutils import random_string
 
@@ -185,19 +190,22 @@ def __init__(self, hs: "HomeServer"):
         self._auth_handler = hs.get_auth_handler()
         self._error_template = hs.config.sso_error_template
         self._bad_user_template = hs.config.sso_auth_bad_user_template
+        self._profile_handler = hs.get_profile_handler()
 
         # The following template is shown after a successful user interactive
         # authentication session. It tells the user they can close the window.
         self._sso_auth_success_template = hs.config.sso_auth_success_template
 
+        self._sso_update_profile_information = hs.config.sso_update_profile_information
+
         # a lock on the mappings
         self._mapping_lock = Linearizer(name="sso_user_mapping", clock=hs.get_clock())
 
         # a map from session id to session data
-        self._username_mapping_sessions = {}  # type: Dict[str, UsernameMappingSession]
+        self._username_mapping_sessions: Dict[str, UsernameMappingSession] = {}
 
         # map from idp_id to SsoIdentityProvider
-        self._identity_providers = {}  # type: Dict[str, SsoIdentityProvider]
+        self._identity_providers: Dict[str, SsoIdentityProvider] = {}
 
         self._consent_at_registration = hs.config.consent.user_consent_at_registration
 
@@ -288,7 +296,7 @@ async def handle_redirect_request(
             )
 
         # if the client chose an IdP, use that
-        idp = None  # type: Optional[SsoIdentityProvider]
+        idp: Optional[SsoIdentityProvider] = None
         if idp_id:
             idp = self._identity_providers.get(idp_id)
             if not idp:
@@ -458,6 +466,21 @@ async def complete_sso_login_request(
                     request.getClientIP(),
                 )
                 new_user = True
+            elif self._sso_update_profile_information:
+                attributes = await self._call_attribute_mapper(sso_to_matrix_id_mapper)
+                if attributes.display_name:
+                    user_id_obj = UserID.from_string(user_id)
+                    profile_display_name = await self._profile_handler.get_displayname(
+                        user_id_obj
+                    )
+                    if profile_display_name != attributes.display_name:
+                        requester = create_requester(
+                            user_id,
+                            authenticated_entity=user_id,
+                        )
+                        await self._profile_handler.set_displayname(
+                            user_id_obj, requester, attributes.display_name, True
+                        )
 
         await self._auth_handler.complete_sso_login(
             user_id,
@@ -646,9 +669,9 @@ async def complete_sso_ui_auth_request(
             remote_user_id,
         )
 
-        user_id_to_verify = await self._auth_handler.get_session_data(
+        user_id_to_verify: str = await self._auth_handler.get_session_data(
             ui_auth_session_id, UIAuthSessionDataConstants.REQUEST_USER_ID
-        )  # type: str
+        )
 
         if not user_id:
             logger.warning(
@@ -770,7 +793,7 @@ async def handle_submit_username_request(
         session.use_display_name = use_display_name
 
         emails_from_idp = set(session.emails)
-        filtered_emails = set()  # type: Set[str]
+        filtered_emails: Set[str] = set()
 
         # we iterate through the list rather than just building a set conjunction, so
         # that we can log attempts to use unknown addresses
diff --git a/synapse/handlers/state_deltas.py b/synapse/handlers/state_deltas.py
index ee8f87e59a..077c7c0649 100644
--- a/synapse/handlers/state_deltas.py
+++ b/synapse/handlers/state_deltas.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py
index 4227f8c1a2..3fd89af2a4 100644
--- a/synapse/handlers/stats.py
+++ b/synapse/handlers/stats.py
@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
+# Copyright 2018-2021 The Matrix.org Foundation C.I.C.
 # Copyright 2020 Sorunome
-# Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -47,12 +45,11 @@ def __init__(self, hs: "HomeServer"):
         self.clock = hs.get_clock()
         self.notifier = hs.get_notifier()
         self.is_mine_id = hs.is_mine_id
-        self.stats_bucket_size = hs.config.stats_bucket_size
 
         self.stats_enabled = hs.config.stats_enabled
 
         # The current position in the current_state_delta stream
-        self.pos = None  # type: Optional[int]
+        self.pos: Optional[int] = None
 
         # Guard to ensure we only process deltas one at a time
         self._is_processing = False
@@ -108,20 +105,6 @@ async def _unsafe_process(self) -> None:
                 room_deltas = {}
                 user_deltas = {}
 
-            # Then count deltas for total_events and total_event_bytes.
-            (
-                room_count,
-                user_count,
-            ) = await self.store.get_changes_room_total_events_and_bytes(
-                self.pos, max_pos
-            )
-
-            for room_id, fields in room_count.items():
-                room_deltas.setdefault(room_id, Counter()).update(fields)
-
-            for user_id, fields in user_count.items():
-                user_deltas.setdefault(user_id, Counter()).update(fields)
-
             logger.debug("room_deltas: %s", room_deltas)
             logger.debug("user_deltas: %s", user_deltas)
 
@@ -148,10 +131,10 @@ async def _handle_deltas(
             mapping from room/user ID to changes in the various fields.
         """
 
-        room_to_stats_deltas = {}  # type: Dict[str, CounterType[str]]
-        user_to_stats_deltas = {}  # type: Dict[str, CounterType[str]]
+        room_to_stats_deltas: Dict[str, CounterType[str]] = {}
+        user_to_stats_deltas: Dict[str, CounterType[str]] = {}
 
-        room_to_state_updates = {}  # type: Dict[str, Dict[str, Any]]
+        room_to_state_updates: Dict[str, Dict[str, Any]] = {}
 
         for delta in deltas:
             typ = delta["type"]
@@ -181,14 +164,12 @@ async def _handle_deltas(
                 )
                 continue
 
-            event_content = {}  # type: JsonDict
+            event_content: JsonDict = {}
 
-            sender = None
             if event_id is not None:
                 event = await self.store.get_event(event_id, allow_none=True)
                 if event:
                     event_content = event.content or {}
-                    sender = event.sender
 
             # All the values in this dict are deltas (RELATIVE changes)
             room_stats_delta = room_to_stats_deltas.setdefault(room_id, Counter())
@@ -246,12 +227,6 @@ async def _handle_deltas(
                     room_stats_delta["joined_members"] += 1
                 elif membership == Membership.INVITE:
                     room_stats_delta["invited_members"] += 1
-
-                    if sender and self.is_mine_id(sender):
-                        user_to_stats_deltas.setdefault(sender, Counter())[
-                            "invites_sent"
-                        ] += 1
-
                 elif membership == Membership.LEAVE:
                     room_stats_delta["left_members"] += 1
                 elif membership == Membership.BAN:
@@ -281,10 +256,6 @@ async def _handle_deltas(
                 room_state["is_federatable"] = (
                     event_content.get("m.federate", True) is True
                 )
-                if sender and self.is_mine_id(sender):
-                    user_to_stats_deltas.setdefault(sender, Counter())[
-                        "rooms_created"
-                    ] += 1
             elif typ == EventTypes.JoinRules:
                 room_state["join_rules"] = event_content.get("join_rule")
             elif typ == EventTypes.RoomHistoryVisibility:
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 17277619ad..590642f510 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018, 2019 New Vector Ltd
 #
@@ -15,7 +14,17 @@
 # limitations under the License.
 import itertools
 import logging
-from typing import TYPE_CHECKING, Any, Dict, FrozenSet, List, Optional, Set, Tuple
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Collection,
+    Dict,
+    FrozenSet,
+    List,
+    Optional,
+    Set,
+    Tuple,
+)
 
 import attr
 from prometheus_client import Counter
@@ -24,11 +33,11 @@
 from synapse.api.filtering import FilterCollection
 from synapse.events import EventBase
 from synapse.logging.context import current_context
+from synapse.logging.opentracing import SynapseTags, log_kv, set_tag, start_active_span
 from synapse.push.clientformat import format_push_rules_for_user
 from synapse.storage.roommember import MemberSummary
 from synapse.storage.state import StateFilter
 from synapse.types import (
-    Collection,
     JsonDict,
     MutableStateMap,
     Requester,
@@ -40,7 +49,7 @@
 from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.caches.lrucache import LruCache
-from synapse.util.caches.response_cache import ResponseCache
+from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext
 from synapse.util.metrics import Measure, measure_func
 from synapse.visibility import filter_events_for_client
 
@@ -74,12 +83,15 @@
 LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE = 100
 
 
+SyncRequestKey = Tuple[Any, ...]
+
+
 @attr.s(slots=True, frozen=True)
 class SyncConfig:
     user = attr.ib(type=UserID)
     filter_collection = attr.ib(type=FilterCollection)
     is_guest = attr.ib(type=bool)
-    request_key = attr.ib(type=Tuple[Any, ...])
+    request_key = attr.ib(type=SyncRequestKey)
     device_id = attr.ib(type=Optional[str])
 
 
@@ -257,16 +269,26 @@ def __init__(self, hs: "HomeServer"):
         self.presence_handler = hs.get_presence_handler()
         self.event_sources = hs.get_event_sources()
         self.clock = hs.get_clock()
-        self.response_cache = ResponseCache(
-            hs.get_clock(), "sync"
-        )  # type: ResponseCache[Tuple[Any, ...]]
         self.state = hs.get_state_handler()
         self.auth = hs.get_auth()
         self.storage = hs.get_storage()
         self.state_store = self.storage.state
 
-        # ExpiringCache((User, Device)) -> LruCache(state_key => event_id)
-        self.lazy_loaded_members_cache = ExpiringCache(
+        # TODO: flush cache entries on subsequent sync request.
+        #    Once we get the next /sync request (ie, one with the same access token
+        #    that sets 'since' to 'next_batch'), we know that device won't need a
+        #    cached result any more, and we could flush the entry from the cache to save
+        #    memory.
+        self.response_cache: ResponseCache[SyncRequestKey] = ResponseCache(
+            hs.get_clock(),
+            "sync",
+            timeout_ms=hs.config.caches.sync_response_cache_duration,
+        )
+
+        # ExpiringCache((User, Device)) -> LruCache(user_id => event_id)
+        self.lazy_loaded_members_cache: ExpiringCache[
+            Tuple[str, Optional[str]], LruCache[str, str]
+        ] = ExpiringCache(
             "lazy_loaded_members_cache",
             self.clock,
             max_len=0,
@@ -298,6 +320,7 @@ async def wait_for_sync_for_user(
             since_token,
             timeout,
             full_state,
+            cache_context=True,
         )
         logger.debug("Returning sync response for %s", user_id)
         return res
@@ -305,9 +328,10 @@ async def wait_for_sync_for_user(
     async def _wait_for_sync_for_user(
         self,
         sync_config: SyncConfig,
-        since_token: Optional[StreamToken] = None,
-        timeout: int = 0,
-        full_state: bool = False,
+        since_token: Optional[StreamToken],
+        timeout: int,
+        full_state: bool,
+        cache_context: ResponseCacheContext[SyncRequestKey],
     ) -> SyncResult:
         if since_token is None:
             sync_type = "initial_sync"
@@ -320,16 +344,27 @@ async def _wait_for_sync_for_user(
         if context:
             context.tag = sync_type
 
+        # if we have a since token, delete any to-device messages before that token
+        # (since we now know that the device has received them)
+        if since_token is not None:
+            since_stream_id = since_token.to_device_key
+            deleted = await self.store.delete_messages_for_device(
+                sync_config.user.to_string(), sync_config.device_id, since_stream_id
+            )
+            logger.debug(
+                "Deleted %d to-device messages up to %d", deleted, since_stream_id
+            )
+
         if timeout == 0 or since_token is None or full_state:
             # we are going to return immediately, so don't bother calling
             # notifier.wait_for_events.
-            result = await self.current_sync_for_user(
+            result: SyncResult = await self.current_sync_for_user(
                 sync_config, since_token, full_state=full_state
             )
         else:
 
-            def current_sync_callback(before_token, after_token):
-                return self.current_sync_for_user(sync_config, since_token)
+            async def current_sync_callback(before_token, after_token) -> SyncResult:
+                return await self.current_sync_for_user(sync_config, since_token)
 
             result = await self.notifier.wait_for_events(
                 sync_config.user.to_string(),
@@ -338,6 +373,17 @@ def current_sync_callback(before_token, after_token):
                 from_token=since_token,
             )
 
+            # if nothing has happened in any of the users' rooms since /sync was called,
+            # the resultant next_batch will be the same as since_token (since the result
+            # is generated when wait_for_events is first called, and not regenerated
+            # when wait_for_events times out).
+            #
+            # If that happens, we mustn't cache it, so that when the client comes back
+            # with the same cache token, we don't immediately return the same empty
+            # result, causing a tightloop. (#8518)
+            if result.next_batch == since_token:
+                cache_context.should_cache = False
+
         if result:
             if sync_config.filter_collection.lazy_load_members():
                 lazy_loaded = "true"
@@ -354,7 +400,14 @@ async def current_sync_for_user(
         full_state: bool = False,
     ) -> SyncResult:
         """Get the sync for client needed to match what the server has now."""
-        return await self.generate_sync_result(sync_config, since_token, full_state)
+        with start_active_span("current_sync_for_user"):
+            log_kv({"since_token": since_token})
+            sync_result = await self.generate_sync_result(
+                sync_config, since_token, full_state
+            )
+
+            set_tag(SynapseTags.SYNC_RESULT, bool(sync_result))
+            return sync_result
 
     async def push_rules_for_user(self, user: UserID) -> JsonDict:
         user_id = user.to_string()
@@ -397,7 +450,7 @@ async def ephemeral_by_room(
             )
             now_token = now_token.copy_and_replace("typing_key", typing_key)
 
-            ephemeral_by_room = {}  # type: JsonDict
+            ephemeral_by_room: JsonDict = {}
 
             for event in typing:
                 # we want to exclude the room_id from the event, but modifying the
@@ -459,9 +512,9 @@ async def _load_filtered_recents(
                 # We check if there are any state events, if there are then we pass
                 # all current state events to the filter_events function. This is to
                 # ensure that we always include current state in the timeline
-                current_state_ids = frozenset()  # type: FrozenSet[str]
+                current_state_ids: FrozenSet[str] = frozenset()
                 if any(e.is_state() for e in recents):
-                    current_state_ids_map = await self.state.get_current_state_ids(
+                    current_state_ids_map = await self.store.get_current_state_ids(
                         room_id
                     )
                     current_state_ids = frozenset(current_state_ids_map.values())
@@ -521,7 +574,7 @@ async def _load_filtered_recents(
                 # ensure that we always include current state in the timeline
                 current_state_ids = frozenset()
                 if any(e.is_state() for e in loaded_recents):
-                    current_state_ids_map = await self.state.get_current_state_ids(
+                    current_state_ids_map = await self.store.get_current_state_ids(
                         room_id
                     )
                     current_state_ids = frozenset(current_state_ids_map.values())
@@ -554,7 +607,7 @@ async def _load_filtered_recents(
         )
 
     async def get_state_after_event(
-        self, event: EventBase, state_filter: StateFilter = StateFilter.all()
+        self, event: EventBase, state_filter: Optional[StateFilter] = None
     ) -> StateMap[str]:
         """
         Get the room state after the given event
@@ -564,7 +617,7 @@ async def get_state_after_event(
             state_filter: The state filter used to fetch state from the database.
         """
         state_ids = await self.state_store.get_state_ids_for_event(
-            event.event_id, state_filter=state_filter
+            event.event_id, state_filter=state_filter or StateFilter.all()
         )
         if event.is_state():
             state_ids = dict(state_ids)
@@ -575,7 +628,7 @@ async def get_state_at(
         self,
         room_id: str,
         stream_position: StreamToken,
-        state_filter: StateFilter = StateFilter.all(),
+        state_filter: Optional[StateFilter] = None,
     ) -> StateMap[str]:
         """Get the room state at a particular stream position
 
@@ -595,7 +648,7 @@ async def get_state_at(
         if last_events:
             last_event = last_events[-1]
             state = await self.get_state_after_event(
-                last_event, state_filter=state_filter
+                last_event, state_filter=state_filter or StateFilter.all()
             )
 
         else:
@@ -739,8 +792,10 @@ async def compute_summary(
 
     def get_lazy_loaded_members_cache(
         self, cache_key: Tuple[str, Optional[str]]
-    ) -> LruCache:
-        cache = self.lazy_loaded_members_cache.get(cache_key)
+    ) -> LruCache[str, str]:
+        cache: Optional[LruCache[str, str]] = self.lazy_loaded_members_cache.get(
+            cache_key
+        )
         if cache is None:
             logger.debug("creating LruCache for %r", cache_key)
             cache = LruCache(LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE)
@@ -939,7 +994,7 @@ async def compute_state_delta(
                     if t[0] == EventTypes.Member:
                         cache.set(t[1], event_id)
 
-        state = {}  # type: Dict[str, EventBase]
+        state: Dict[str, EventBase] = {}
         if state_ids:
             state = await self.store.get_events(list(state_ids.values()))
 
@@ -978,6 +1033,7 @@ async def generate_sync_result(
         # to query up to a given point.
         # Always use the `now_token` in `SyncResultBuilder`
         now_token = self.event_sources.get_current_token()
+        log_kv({"now_token": now_token})
 
         logger.debug(
             "Calculating sync response for %r between %s and %s",
@@ -1042,9 +1098,13 @@ async def generate_sync_result(
 
         logger.debug("Fetching OTK data")
         device_id = sync_config.device_id
-        one_time_key_counts = {}  # type: JsonDict
-        unused_fallback_key_types = []  # type: List[str]
+        one_time_key_counts: JsonDict = {}
+        unused_fallback_key_types: List[str] = []
         if device_id:
+            # TODO: We should have a way to let clients differentiate between the states of:
+            #   * no change in OTK count since the provided since token
+            #   * the server has zero OTKs left for this device
+            #  Spec issue: https://github.com/matrix-org/matrix-doc/issues/3298
             one_time_key_counts = await self.store.count_e2e_one_time_keys(
                 user_id, device_id
             )
@@ -1191,7 +1251,7 @@ async def _generate_sync_entry_for_device_list(
 
             # Step 1b, check for newly joined rooms
             for room_id in newly_joined_rooms:
-                joined_users = await self.state.get_current_users_in_room(room_id)
+                joined_users = await self.store.get_users_in_room(room_id)
                 newly_joined_or_invited_or_knocked_users.update(joined_users)
 
             # TODO: Check that these users are actually new, i.e. either they
@@ -1207,7 +1267,7 @@ async def _generate_sync_entry_for_device_list(
 
             # Now find users that we no longer track
             for room_id in newly_left_rooms:
-                left_users = await self.state.get_current_users_in_room(room_id)
+                left_users = await self.store.get_users_in_room(room_id)
                 newly_left_users.update(left_users)
 
             # Remove any users that we still share a room with.
@@ -1231,20 +1291,17 @@ async def _generate_sync_entry_for_to_device(
             since_stream_id = int(sync_result_builder.since_token.to_device_key)
 
         if since_stream_id != int(now_token.to_device_key):
-            # We only delete messages when a new message comes in, but that's
-            # fine so long as we delete them at some point.
-
-            deleted = await self.store.delete_messages_for_device(
-                user_id, device_id, since_stream_id
-            )
-            logger.debug(
-                "Deleted %d to-device messages up to %d", deleted, since_stream_id
-            )
-
             messages, stream_id = await self.store.get_new_messages_for_device(
                 user_id, device_id, since_stream_id, now_token.to_device_key
             )
 
+            for message in messages:
+                # We pop here as we shouldn't be sending the message ID down
+                # `/sync`
+                message_id = message.pop("message_id", None)
+                if message_id:
+                    set_tag(SynapseTags.TO_DEVICE_MESSAGE_ID, message_id)
+
             logger.debug(
                 "Returning %d to-device messages between %d and %d (current token: %d)",
                 len(messages),
@@ -1355,7 +1412,7 @@ async def _generate_sync_entry_for_presence(
 
         extra_users_ids = set(newly_joined_or_invited_users)
         for room_id in newly_joined_rooms:
-            users = await self.state.get_current_users_in_room(room_id)
+            users = await self.store.get_users_in_room(room_id)
             extra_users_ids.update(users)
         extra_users_ids.discard(user.to_string())
 
@@ -1394,7 +1451,7 @@ async def _generate_sync_entry_for_rooms(
         )
 
         if block_all_room_ephemeral:
-            ephemeral_by_room = {}  # type: Dict[str, List[JsonDict]]
+            ephemeral_by_room: Dict[str, List[JsonDict]] = {}
         else:
             now_token, ephemeral_by_room = await self.ephemeral_by_room(
                 sync_result_builder,
@@ -1425,7 +1482,7 @@ async def _generate_sync_entry_for_rooms(
 
         # If there is ignored users account data and it matches the proper type,
         # then use it.
-        ignored_users = frozenset()  # type: FrozenSet[str]
+        ignored_users: FrozenSet[str] = frozenset()
         if ignored_account_data:
             ignored_users_data = ignored_account_data.get("ignored_users", {})
             if isinstance(ignored_users_data, dict):
@@ -1543,7 +1600,7 @@ async def _get_rooms_changed(
             user_id, since_token.room_key, now_token.room_key
         )
 
-        mem_change_events_by_room_id = {}  # type: Dict[str, List[EventBase]]
+        mem_change_events_by_room_id: Dict[str, List[EventBase]] = {}
         for event in rooms_changed:
             mem_change_events_by_room_id.setdefault(event.room_id, []).append(event)
 
@@ -1556,7 +1613,7 @@ async def _get_rooms_changed(
             logger.debug(
                 "Membership changes in %s: [%s]",
                 room_id,
-                ", ".join(("%s (%s)" % (e.event_id, e.membership) for e in events)),
+                ", ".join("%s (%s)" % (e.event_id, e.membership) for e in events),
             )
 
             non_joins = [e for e in events if e.membership != Membership.JOIN]
@@ -1679,7 +1736,7 @@ async def _get_rooms_changed(
                 # This is all screaming out for a refactor, as the logic here is
                 # subtle and the moving parts numerous.
                 if leave_event.internal_metadata.is_out_of_band_membership():
-                    batch_events = [leave_event]  # type: Optional[List[EventBase]]
+                    batch_events: Optional[List[EventBase]] = [leave_event]
                 else:
                     batch_events = None
 
@@ -1928,7 +1985,7 @@ async def _generate_room_entry(
             room_id, batch, sync_config, since_token, now_token, full_state=full_state
         )
 
-        summary = {}  # type: Optional[JsonDict]
+        summary: Optional[JsonDict] = {}
 
         # we include a summary in room responses when we're lazy loading
         # members (as the client otherwise doesn't have enough info to form
@@ -1952,7 +2009,7 @@ async def _generate_room_entry(
             )
 
         if room_builder.rtype == "joined":
-            unread_notifications = {}  # type: Dict[str, int]
+            unread_notifications: Dict[str, int] = {}
             room_sync = JoinedSyncResult(
                 room_id=room_id,
                 timeline=batch,
diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py
index 096d199f4c..a97c448595 100644
--- a/synapse/handlers/typing.py
+++ b/synapse/handlers/typing.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,7 +18,10 @@
 
 from synapse.api.errors import AuthError, ShadowBanError, SynapseError
 from synapse.appservice import ApplicationService
-from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.metrics.background_process_metrics import (
+    run_as_background_process,
+    wrap_as_background_process,
+)
 from synapse.replication.tcp.streams import TypingStream
 from synapse.types import JsonDict, Requester, UserID, get_domain_from_id
 from synapse.util.caches.stream_change_cache import StreamChangeCache
@@ -66,11 +68,11 @@ def __init__(self, hs: "HomeServer"):
             )
 
         # map room IDs to serial numbers
-        self._room_serials = {}  # type: Dict[str, int]
+        self._room_serials: Dict[str, int] = {}
         # map room IDs to sets of users currently typing
-        self._room_typing = {}  # type: Dict[str, Set[str]]
+        self._room_typing: Dict[str, Set[str]] = {}
 
-        self._member_last_federation_poke = {}  # type: Dict[RoomMember, int]
+        self._member_last_federation_poke: Dict[RoomMember, int] = {}
         self.wheel_timer = WheelTimer(bucket_size=5000)
         self._latest_room_serial = 0
 
@@ -86,6 +88,7 @@ def _reset(self) -> None:
         self._member_last_federation_poke = {}
         self.wheel_timer = WheelTimer(bucket_size=5000)
 
+    @wrap_as_background_process("typing._handle_timeouts")
     def _handle_timeouts(self) -> None:
         logger.debug("Checking for typing timeouts")
 
@@ -205,6 +208,7 @@ def __init__(self, hs: "HomeServer"):
 
         self.auth = hs.get_auth()
         self.notifier = hs.get_notifier()
+        self.event_auth_handler = hs.get_event_auth_handler()
 
         self.hs = hs
 
@@ -213,7 +217,7 @@ def __init__(self, hs: "HomeServer"):
         hs.get_distributor().observe("user_left_room", self.user_left_room)
 
         # clock time we expect to stop
-        self._member_typing_until = {}  # type: Dict[RoomMember, int]
+        self._member_typing_until: Dict[RoomMember, int] = {}
 
         # caches which room_ids changed at which serials
         self._typing_stream_change_cache = StreamChangeCache(
@@ -323,6 +327,20 @@ async def _recv_edu(self, origin: str, content: JsonDict) -> None:
         room_id = content["room_id"]
         user_id = content["user_id"]
 
+        # If we're not in the room just ditch the event entirely. This is
+        # probably an old server that has come back and thinks we're still in
+        # the room (or we've been rejoined to the room by a state reset).
+        is_in_room = await self.event_auth_handler.check_host_in_room(
+            room_id, self.server_name
+        )
+        if not is_in_room:
+            logger.info(
+                "Ignoring typing update for room %r from server %s as we're not in the room",
+                room_id,
+                origin,
+            )
+            return
+
         member = RoomMember(user_id=user_id, room_id=room_id)
 
         # Check that the string is a valid user id
@@ -388,9 +406,9 @@ async def get_all_typing_updates(
         if last_id == current_id:
             return [], current_id, False
 
-        changed_rooms = self._typing_stream_change_cache.get_all_entities_changed(
-            last_id
-        )  # type: Optional[Iterable[str]]
+        changed_rooms: Optional[
+            Iterable[str]
+        ] = self._typing_stream_change_cache.get_all_entities_changed(last_id)
 
         if changed_rooms is None:
             changed_rooms = self._room_serials
diff --git a/synapse/handlers/ui_auth/__init__.py b/synapse/handlers/ui_auth/__init__.py
index a68d5e790e..4c3b669fae 100644
--- a/synapse/handlers/ui_auth/__init__.py
+++ b/synapse/handlers/ui_auth/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/handlers/ui_auth/checkers.py b/synapse/handlers/ui_auth/checkers.py
index 3d66bf305e..5414ce77d8 100644
--- a/synapse/handlers/ui_auth/checkers.py
+++ b/synapse/handlers/ui_auth/checkers.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +13,7 @@
 # limitations under the License.
 
 import logging
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from twisted.web.client import PartialDownloadError
 
@@ -23,13 +22,16 @@
 from synapse.config.emailconfig import ThreepidBehaviour
 from synapse.util import json_decoder
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
 class UserInteractiveAuthChecker:
     """Abstract base class for an interactive auth checker"""
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         pass
 
     def is_enabled(self) -> bool:
@@ -58,10 +60,10 @@ async def check_auth(self, authdict: dict, clientip: str) -> Any:
 class DummyAuthChecker(UserInteractiveAuthChecker):
     AUTH_TYPE = LoginType.DUMMY
 
-    def is_enabled(self):
+    def is_enabled(self) -> bool:
         return True
 
-    async def check_auth(self, authdict, clientip):
+    async def check_auth(self, authdict: dict, clientip: str) -> Any:
         return True
 
 
@@ -71,24 +73,24 @@ class TermsAuthChecker(UserInteractiveAuthChecker):
     def is_enabled(self):
         return True
 
-    async def check_auth(self, authdict, clientip):
+    async def check_auth(self, authdict: dict, clientip: str) -> Any:
         return True
 
 
 class RecaptchaAuthChecker(UserInteractiveAuthChecker):
     AUTH_TYPE = LoginType.RECAPTCHA
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
         self._enabled = bool(hs.config.recaptcha_private_key)
         self._http_client = hs.get_proxied_http_client()
         self._url = hs.config.recaptcha_siteverify_api
         self._secret = hs.config.recaptcha_private_key
 
-    def is_enabled(self):
+    def is_enabled(self) -> bool:
         return self._enabled
 
-    async def check_auth(self, authdict, clientip):
+    async def check_auth(self, authdict: dict, clientip: str) -> Any:
         try:
             user_response = authdict["response"]
         except KeyError:
@@ -133,11 +135,11 @@ async def check_auth(self, authdict, clientip):
 
 
 class _BaseThreepidAuthChecker:
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.hs = hs
         self.store = hs.get_datastore()
 
-    async def _check_threepid(self, medium, authdict):
+    async def _check_threepid(self, medium: str, authdict: dict) -> dict:
         if "threepid_creds" not in authdict:
             raise LoginError(400, "Missing threepid_creds", Codes.MISSING_PARAM)
 
@@ -207,31 +209,31 @@ async def _check_threepid(self, medium, authdict):
 class EmailIdentityAuthChecker(UserInteractiveAuthChecker, _BaseThreepidAuthChecker):
     AUTH_TYPE = LoginType.EMAIL_IDENTITY
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         UserInteractiveAuthChecker.__init__(self, hs)
         _BaseThreepidAuthChecker.__init__(self, hs)
 
-    def is_enabled(self):
+    def is_enabled(self) -> bool:
         return self.hs.config.threepid_behaviour_email in (
             ThreepidBehaviour.REMOTE,
             ThreepidBehaviour.LOCAL,
         )
 
-    async def check_auth(self, authdict, clientip):
+    async def check_auth(self, authdict: dict, clientip: str) -> Any:
         return await self._check_threepid("email", authdict)
 
 
 class MsisdnAuthChecker(UserInteractiveAuthChecker, _BaseThreepidAuthChecker):
     AUTH_TYPE = LoginType.MSISDN
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         UserInteractiveAuthChecker.__init__(self, hs)
         _BaseThreepidAuthChecker.__init__(self, hs)
 
-    def is_enabled(self):
+    def is_enabled(self) -> bool:
         return bool(self.hs.config.account_threepid_delegate_msisdn)
 
-    async def check_auth(self, authdict, clientip):
+    async def check_auth(self, authdict: dict, clientip: str) -> Any:
         return await self._check_threepid("msisdn", authdict)
 
 
diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py
index b121286d95..6edb1da50a 100644
--- a/synapse/handlers/user_directory.py
+++ b/synapse/handlers/user_directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -45,7 +44,6 @@ def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.store = hs.get_datastore()
-        self.state = hs.get_state_handler()
         self.server_name = hs.hostname
         self.clock = hs.get_clock()
         self.notifier = hs.get_notifier()
@@ -54,7 +52,7 @@ def __init__(self, hs: "HomeServer"):
         self.search_all_users = hs.config.user_directory_search_all_users
         self.spam_checker = hs.get_spam_checker()
         # The current position in the current_state_delta stream
-        self.pos = None  # type: Optional[int]
+        self.pos: Optional[int] = None
 
         # Guard to ensure we only process deltas one at a time
         self._is_processing = False
@@ -303,10 +301,12 @@ async def _handle_room_publicity_change(
             # ignore the change
             return
 
-        users_with_profile = await self.state.get_current_users_in_room(room_id)
+        other_users_in_room_with_profiles = (
+            await self.store.get_users_in_room_with_profiles(room_id)
+        )
 
         # Remove every user from the sharing tables for that room.
-        for user_id in users_with_profile.keys():
+        for user_id in other_users_in_room_with_profiles.keys():
             await self.store.remove_user_who_share_room(user_id, room_id)
 
         # Then, re-add them to the tables.
@@ -315,7 +315,7 @@ async def _handle_room_publicity_change(
         # which when ran over an entire room, will result in the same values
         # being added multiple times. The batching upserts shouldn't make this
         # too bad, though.
-        for user_id, profile in users_with_profile.items():
+        for user_id, profile in other_users_in_room_with_profiles.items():
             await self._handle_new_user(room_id, user_id, profile)
 
     async def _handle_new_user(
@@ -337,7 +337,7 @@ async def _handle_new_user(
             room_id
         )
         # Now we update users who share rooms with users.
-        users_with_profile = await self.state.get_current_users_in_room(room_id)
+        other_users_in_room = await self.store.get_users_in_room(room_id)
 
         if is_public:
             await self.store.add_users_in_public_rooms(room_id, (user_id,))
@@ -353,14 +353,14 @@ async def _handle_new_user(
 
                 # We don't care about appservice users.
                 if not is_appservice:
-                    for other_user_id in users_with_profile:
+                    for other_user_id in other_users_in_room:
                         if user_id == other_user_id:
                             continue
 
                         to_insert.add((user_id, other_user_id))
 
             # Next we need to update for every local user in the room
-            for other_user_id in users_with_profile:
+            for other_user_id in other_users_in_room:
                 if user_id == other_user_id:
                     continue
 
diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py
index 142b007d01..578fc48ef4 100644
--- a/synapse/http/__init__.py
+++ b/synapse/http/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -70,7 +69,7 @@ def _get_requested_host(request: IRequest) -> bytes:
         return hostname
 
     # no Host header, use the address/port that the request arrived on
-    host = request.getHost()  # type: Union[address.IPv4Address, address.IPv6Address]
+    host: Union[address.IPv4Address, address.IPv6Address] = request.getHost()
 
     hostname = host.host.encode("ascii")
 
diff --git a/synapse/http/additional_resource.py b/synapse/http/additional_resource.py
index 479746c9c5..55ea97a07f 100644
--- a/synapse/http/additional_resource.py
+++ b/synapse/http/additional_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/http/client.py b/synapse/http/client.py
index a0caba84e4..c2ea51ee16 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -34,6 +33,7 @@
 from canonicaljson import encode_canonical_json
 from netaddr import AddrFormatError, IPAddress, IPSet
 from prometheus_client import Counter
+from typing_extensions import Protocol
 from zope.interface import implementer, provider
 
 from OpenSSL import SSL
@@ -160,7 +160,7 @@ def __init__(
     def resolveHostName(
         self, recv: IResolutionReceiver, hostname: str, portNumber: int = 0
     ) -> IResolutionReceiver:
-        addresses = []  # type: List[IAddress]
+        addresses: List[IAddress] = []
 
         def _callback() -> None:
             has_bad_ip = False
@@ -297,7 +297,7 @@ class SimpleHttpClient:
     def __init__(
         self,
         hs: "HomeServer",
-        treq_args: Dict[str, Any] = {},
+        treq_args: Optional[Dict[str, Any]] = None,
         ip_whitelist: Optional[IPSet] = None,
         ip_blacklist: Optional[IPSet] = None,
         use_proxy: bool = False,
@@ -317,7 +317,7 @@ def __init__(
 
         self._ip_whitelist = ip_whitelist
         self._ip_blacklist = ip_blacklist
-        self._extra_treq_args = treq_args
+        self._extra_treq_args = treq_args or {}
 
         self.user_agent = hs.version_string
         self.clock = hs.get_clock()
@@ -333,9 +333,9 @@ def __init__(
         if self._ip_blacklist:
             # If we have an IP blacklist, we need to use a DNS resolver which
             # filters out blacklisted IP addresses, to prevent DNS rebinding.
-            self.reactor = BlacklistingReactorWrapper(
+            self.reactor: ISynapseReactor = BlacklistingReactorWrapper(
                 hs.get_reactor(), self._ip_whitelist, self._ip_blacklist
-            )  # type: ISynapseReactor
+            )
         else:
             self.reactor = hs.get_reactor()
 
@@ -349,14 +349,14 @@ def __init__(
         pool.maxPersistentPerHost = max((100 * hs.config.caches.global_factor, 5))
         pool.cachedConnectionTimeout = 2 * 60
 
-        self.agent = ProxyAgent(
+        self.agent: IAgent = ProxyAgent(
             self.reactor,
             hs.get_reactor(),
             connectTimeout=15,
             contextFactory=self.hs.get_http_client_context_factory(),
             pool=pool,
             use_proxy=use_proxy,
-        )  # type: IAgent
+        )
 
         if self._ip_blacklist:
             # If we have an IP blacklist, we then install the blacklisting Agent
@@ -411,7 +411,7 @@ async def request(
                         cooperator=self._cooperator,
                     )
 
-                request_deferred = treq.request(
+                request_deferred: defer.Deferred = treq.request(
                     method,
                     uri,
                     agent=self.agent,
@@ -421,7 +421,7 @@ async def request(
                     # response bodies.
                     unbuffered=True,
                     **self._extra_treq_args,
-                )  # type: defer.Deferred
+                )
 
                 # we use our own timeout mechanism rather than treq's as a workaround
                 # for https://twistedmatrix.com/trac/ticket/9534.
@@ -590,7 +590,7 @@ async def put_json(
         uri: str,
         json_body: Any,
         args: Optional[QueryParams] = None,
-        headers: RawHeaders = None,
+        headers: Optional[RawHeaders] = None,
     ) -> Any:
         """Puts some json to the given URI.
 
@@ -755,6 +755,16 @@ def _timeout_to_request_timed_out_error(f: Failure):
     return f
 
 
+class ByteWriteable(Protocol):
+    """The type of object which must be passed into read_body_with_max_size.
+
+    Typically this is a file object.
+    """
+
+    def write(self, data: bytes) -> int:
+        pass
+
+
 class BodyExceededMaxSize(Exception):
     """The maximum allowed size of the HTTP body was exceeded."""
 
@@ -762,7 +772,7 @@ class BodyExceededMaxSize(Exception):
 class _DiscardBodyWithMaxSizeProtocol(protocol.Protocol):
     """A protocol which immediately errors upon receiving data."""
 
-    transport = None  # type: Optional[ITCPTransport]
+    transport: Optional[ITCPTransport] = None
 
     def __init__(self, deferred: defer.Deferred):
         self.deferred = deferred
@@ -788,10 +798,10 @@ def connectionLost(self, reason: Failure = connectionDone) -> None:
 class _ReadBodyWithMaxSizeProtocol(protocol.Protocol):
     """A protocol which reads body to a stream, erroring if the body exceeds a maximum size."""
 
-    transport = None  # type: Optional[ITCPTransport]
+    transport: Optional[ITCPTransport] = None
 
     def __init__(
-        self, stream: BinaryIO, deferred: defer.Deferred, max_size: Optional[int]
+        self, stream: ByteWriteable, deferred: defer.Deferred, max_size: Optional[int]
     ):
         self.stream = stream
         self.deferred = deferred
@@ -803,7 +813,12 @@ def dataReceived(self, data: bytes) -> None:
         if self.deferred.called:
             return
 
-        self.stream.write(data)
+        try:
+            self.stream.write(data)
+        except Exception:
+            self.deferred.errback()
+            return
+
         self.length += len(data)
         # The first time the maximum size is exceeded, error and cancel the
         # connection. dataReceived might be called again if data was received
@@ -831,8 +846,8 @@ def connectionLost(self, reason: Failure = connectionDone) -> None:
 
 
 def read_body_with_max_size(
-    response: IResponse, stream: BinaryIO, max_size: Optional[int]
-) -> defer.Deferred:
+    response: IResponse, stream: ByteWriteable, max_size: Optional[int]
+) -> "defer.Deferred[int]":
     """
     Read a HTTP response body to a file-object. Optionally enforcing a maximum file size.
 
@@ -847,7 +862,7 @@ def read_body_with_max_size(
     Returns:
         A Deferred which resolves to the length of the read body.
     """
-    d = defer.Deferred()
+    d: "defer.Deferred[int]" = defer.Deferred()
 
     # If the Content-Length header gives a size larger than the maximum allowed
     # size, do not bother downloading the body.
diff --git a/synapse/http/connectproxyclient.py b/synapse/http/connectproxyclient.py
index b797e3ce80..c577142268 100644
--- a/synapse/http/connectproxyclient.py
+++ b/synapse/http/connectproxyclient.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,8 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
 import logging
+from typing import Optional
 
+import attr
 from zope.interface import implementer
 
 from twisted.internet import defer, protocol
@@ -22,7 +24,6 @@
 from twisted.internet.interfaces import IReactorCore, IStreamClientEndpoint
 from twisted.internet.protocol import ClientFactory, Protocol, connectionDone
 from twisted.web import http
-from twisted.web.http_headers import Headers
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +32,22 @@ class ProxyConnectError(ConnectError):
     pass
 
 
+@attr.s
+class ProxyCredentials:
+    username_password = attr.ib(type=bytes)
+
+    def as_proxy_authorization_value(self) -> bytes:
+        """
+        Return the value for a Proxy-Authorization header (i.e. 'Basic abdef==').
+
+        Returns:
+            A transformation of the authentication string the encoded value for
+            a Proxy-Authorization header.
+        """
+        # Encode as base64 and prepend the authorization type
+        return b"Basic " + base64.encodebytes(self.username_password)
+
+
 @implementer(IStreamClientEndpoint)
 class HTTPConnectProxyEndpoint:
     """An Endpoint implementation which will send a CONNECT request to an http proxy
@@ -47,7 +64,7 @@ class HTTPConnectProxyEndpoint:
         proxy_endpoint: the endpoint to use to connect to the proxy
         host: hostname that we want to CONNECT to
         port: port that we want to connect to
-        headers: Extra HTTP headers to include in the CONNECT request
+        proxy_creds: credentials to authenticate at proxy
     """
 
     def __init__(
@@ -56,20 +73,20 @@ def __init__(
         proxy_endpoint: IStreamClientEndpoint,
         host: bytes,
         port: int,
-        headers: Headers,
+        proxy_creds: Optional[ProxyCredentials],
     ):
         self._reactor = reactor
         self._proxy_endpoint = proxy_endpoint
         self._host = host
         self._port = port
-        self._headers = headers
+        self._proxy_creds = proxy_creds
 
     def __repr__(self):
         return "<HTTPConnectProxyEndpoint %s>" % (self._proxy_endpoint,)
 
     def connect(self, protocolFactory: ClientFactory):
         f = HTTPProxiedClientFactory(
-            self._host, self._port, protocolFactory, self._headers
+            self._host, self._port, protocolFactory, self._proxy_creds
         )
         d = self._proxy_endpoint.connect(f)
         # once the tcp socket connects successfully, we need to wait for the
@@ -88,7 +105,7 @@ class HTTPProxiedClientFactory(protocol.ClientFactory):
         dst_host: hostname that we want to CONNECT to
         dst_port: port that we want to connect to
         wrapped_factory: The original Factory
-        headers: Extra HTTP headers to include in the CONNECT request
+        proxy_creds: credentials to authenticate at proxy
     """
 
     def __init__(
@@ -96,12 +113,12 @@ def __init__(
         dst_host: bytes,
         dst_port: int,
         wrapped_factory: ClientFactory,
-        headers: Headers,
+        proxy_creds: Optional[ProxyCredentials],
     ):
         self.dst_host = dst_host
         self.dst_port = dst_port
         self.wrapped_factory = wrapped_factory
-        self.headers = headers
+        self.proxy_creds = proxy_creds
         self.on_connection = defer.Deferred()
 
     def startedConnecting(self, connector):
@@ -115,7 +132,7 @@ def buildProtocol(self, addr):
             self.dst_port,
             wrapped_protocol,
             self.on_connection,
-            self.headers,
+            self.proxy_creds,
         )
 
     def clientConnectionFailed(self, connector, reason):
@@ -146,7 +163,7 @@ class HTTPConnectProtocol(protocol.Protocol):
         connected_deferred: a Deferred which will be callbacked with
             wrapped_protocol when the CONNECT completes
 
-        headers: Extra HTTP headers to include in the CONNECT request
+        proxy_creds: credentials to authenticate at proxy
     """
 
     def __init__(
@@ -155,16 +172,16 @@ def __init__(
         port: int,
         wrapped_protocol: Protocol,
         connected_deferred: defer.Deferred,
-        headers: Headers,
+        proxy_creds: Optional[ProxyCredentials],
     ):
         self.host = host
         self.port = port
         self.wrapped_protocol = wrapped_protocol
         self.connected_deferred = connected_deferred
-        self.headers = headers
+        self.proxy_creds = proxy_creds
 
         self.http_setup_client = HTTPConnectSetupClient(
-            self.host, self.port, self.headers
+            self.host, self.port, self.proxy_creds
         )
         self.http_setup_client.on_connected.addCallback(self.proxyConnected)
 
@@ -206,30 +223,38 @@ class HTTPConnectSetupClient(http.HTTPClient):
     Args:
         host: The hostname to send in the CONNECT message
         port: The port to send in the CONNECT message
-        headers: Extra headers to send with the CONNECT message
+        proxy_creds: credentials to authenticate at proxy
     """
 
-    def __init__(self, host: bytes, port: int, headers: Headers):
+    def __init__(
+        self,
+        host: bytes,
+        port: int,
+        proxy_creds: Optional[ProxyCredentials],
+    ):
         self.host = host
         self.port = port
-        self.headers = headers
+        self.proxy_creds = proxy_creds
         self.on_connected = defer.Deferred()
 
     def connectionMade(self):
         logger.debug("Connected to proxy, sending CONNECT")
         self.sendCommand(b"CONNECT", b"%s:%d" % (self.host, self.port))
 
-        # Send any additional specified headers
-        for name, values in self.headers.getAllRawHeaders():
-            for value in values:
-                self.sendHeader(name, value)
+        # Determine whether we need to set Proxy-Authorization headers
+        if self.proxy_creds:
+            # Set a Proxy-Authorization header
+            self.sendHeader(
+                b"Proxy-Authorization",
+                self.proxy_creds.as_proxy_authorization_value(),
+            )
 
         self.endHeaders()
 
     def handleStatus(self, version: bytes, status: bytes, message: bytes):
         logger.debug("Got Status: %s %s %s", status, message, version)
         if status != b"200":
-            raise ProxyConnectError("Unexpected status on CONNECT: %s" % status)
+            raise ProxyConnectError(f"Unexpected status on CONNECT: {status!s}")
 
     def handleEndHeaders(self):
         logger.debug("End Headers")
diff --git a/synapse/http/federation/__init__.py b/synapse/http/federation/__init__.py
index 1453d04571..743fb9904a 100644
--- a/synapse/http/federation/__init__.py
+++ b/synapse/http/federation/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py
index 5935a125fd..1238bfd287 100644
--- a/synapse/http/federation/matrix_federation_agent.py
+++ b/synapse/http/federation/matrix_federation_agent.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,6 +14,10 @@
 import logging
 import urllib.parse
 from typing import Any, Generator, List, Optional
+from urllib.request import (  # type: ignore[attr-defined]
+    getproxies_environment,
+    proxy_bypass_environment,
+)
 
 from netaddr import AddrFormatError, IPAddress, IPSet
 from zope.interface import implementer
@@ -28,12 +31,15 @@
 )
 from twisted.web.client import URI, Agent, HTTPConnectionPool
 from twisted.web.http_headers import Headers
-from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer
+from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer, IResponse
 
 from synapse.crypto.context_factory import FederationPolicyForHTTPS
-from synapse.http.client import BlacklistingAgentWrapper
+from synapse.http import proxyagent
+from synapse.http.client import BlacklistingAgentWrapper, BlacklistingReactorWrapper
+from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint
 from synapse.http.federation.srv_resolver import Server, SrvResolver
 from synapse.http.federation.well_known_resolver import WellKnownResolver
+from synapse.http.proxyagent import ProxyAgent
 from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.types import ISynapseReactor
 from synapse.util import Clock
@@ -58,6 +64,14 @@ class MatrixFederationAgent:
         user_agent:
             The user agent header to use for federation requests.
 
+        ip_whitelist: Allowed IP addresses.
+
+        ip_blacklist: Disallowed IP addresses.
+
+        proxy_reactor: twisted reactor to use for connections to the proxy server
+           reactor might have some blacklisting applied (i.e. for DNS queries),
+           but we need unblocked access to the proxy.
+
         _srv_resolver:
             SrvResolver implementation to use for looking up SRV records. None
             to use a default implementation.
@@ -72,11 +86,18 @@ def __init__(
         reactor: ISynapseReactor,
         tls_client_options_factory: Optional[FederationPolicyForHTTPS],
         user_agent: bytes,
+        ip_whitelist: IPSet,
         ip_blacklist: IPSet,
         _srv_resolver: Optional[SrvResolver] = None,
         _well_known_resolver: Optional[WellKnownResolver] = None,
     ):
-        self._reactor = reactor
+        # proxy_reactor is not blacklisted
+        proxy_reactor = reactor
+
+        # We need to use a DNS resolver which filters out blacklisted IP
+        # addresses, to prevent DNS rebinding.
+        reactor = BlacklistingReactorWrapper(reactor, ip_whitelist, ip_blacklist)
+
         self._clock = Clock(reactor)
         self._pool = HTTPConnectionPool(reactor)
         self._pool.retryAutomatically = False
@@ -84,24 +105,27 @@ def __init__(
         self._pool.cachedConnectionTimeout = 2 * 60
 
         self._agent = Agent.usingEndpointFactory(
-            self._reactor,
+            reactor,
             MatrixHostnameEndpointFactory(
-                reactor, tls_client_options_factory, _srv_resolver
+                reactor,
+                proxy_reactor,
+                tls_client_options_factory,
+                _srv_resolver,
             ),
             pool=self._pool,
         )
         self.user_agent = user_agent
 
         if _well_known_resolver is None:
-            # Note that the name resolver has already been wrapped in a
-            # IPBlacklistingResolver by MatrixFederationHttpClient.
             _well_known_resolver = WellKnownResolver(
-                self._reactor,
+                reactor,
                 agent=BlacklistingAgentWrapper(
-                    Agent(
-                        self._reactor,
+                    ProxyAgent(
+                        reactor,
+                        proxy_reactor,
                         pool=self._pool,
                         contextFactory=tls_client_options_factory,
+                        use_proxy=True,
                     ),
                     ip_blacklist=ip_blacklist,
                 ),
@@ -117,7 +141,7 @@ def request(
         uri: bytes,
         headers: Optional[Headers] = None,
         bodyProducer: Optional[IBodyProducer] = None,
-    ) -> Generator[defer.Deferred, Any, defer.Deferred]:
+    ) -> Generator[defer.Deferred, Any, IResponse]:
         """
         Args:
             method: HTTP method: GET/POST/etc
@@ -201,10 +225,12 @@ class MatrixHostnameEndpointFactory:
     def __init__(
         self,
         reactor: IReactorCore,
+        proxy_reactor: IReactorCore,
         tls_client_options_factory: Optional[FederationPolicyForHTTPS],
         srv_resolver: Optional[SrvResolver],
     ):
         self._reactor = reactor
+        self._proxy_reactor = proxy_reactor
         self._tls_client_options_factory = tls_client_options_factory
 
         if srv_resolver is None:
@@ -212,9 +238,10 @@ def __init__(
 
         self._srv_resolver = srv_resolver
 
-    def endpointForURI(self, parsed_uri):
+    def endpointForURI(self, parsed_uri: URI):
         return MatrixHostnameEndpoint(
             self._reactor,
+            self._proxy_reactor,
             self._tls_client_options_factory,
             self._srv_resolver,
             parsed_uri,
@@ -228,23 +255,45 @@ class MatrixHostnameEndpoint:
 
     Args:
         reactor: twisted reactor to use for underlying requests
+        proxy_reactor: twisted reactor to use for connections to the proxy server.
+           'reactor' might have some blacklisting applied (i.e. for DNS queries),
+           but we need unblocked access to the proxy.
         tls_client_options_factory:
             factory to use for fetching client tls options, or none to disable TLS.
         srv_resolver: The SRV resolver to use
         parsed_uri: The parsed URI that we're wanting to connect to.
+
+    Raises:
+        ValueError if the environment variables contain an invalid proxy specification.
+        RuntimeError if no tls_options_factory is given for a https connection
     """
 
     def __init__(
         self,
         reactor: IReactorCore,
+        proxy_reactor: IReactorCore,
         tls_client_options_factory: Optional[FederationPolicyForHTTPS],
         srv_resolver: SrvResolver,
         parsed_uri: URI,
     ):
         self._reactor = reactor
-
         self._parsed_uri = parsed_uri
 
+        # http_proxy is not needed because federation is always over TLS
+        proxies = getproxies_environment()
+        https_proxy = proxies["https"].encode() if "https" in proxies else None
+        self.no_proxy = proxies["no"] if "no" in proxies else None
+
+        # endpoint and credentials to use to connect to the outbound https proxy, if any.
+        (
+            self._https_proxy_endpoint,
+            self._https_proxy_creds,
+        ) = proxyagent.http_proxy_endpoint(
+            https_proxy,
+            proxy_reactor,
+            tls_client_options_factory,
+        )
+
         # set up the TLS connection params
         #
         # XXX disabling TLS is really only supported here for the benefit of the
@@ -274,9 +323,33 @@ async def _do_connect(self, protocol_factory: IProtocolFactory) -> None:
             host = server.host
             port = server.port
 
+            should_skip_proxy = False
+            if self.no_proxy is not None:
+                should_skip_proxy = proxy_bypass_environment(
+                    host.decode(),
+                    proxies={"no": self.no_proxy},
+                )
+
+            endpoint: IStreamClientEndpoint
             try:
-                logger.debug("Connecting to %s:%i", host.decode("ascii"), port)
-                endpoint = HostnameEndpoint(self._reactor, host, port)
+                if self._https_proxy_endpoint and not should_skip_proxy:
+                    logger.debug(
+                        "Connecting to %s:%i via %s",
+                        host.decode("ascii"),
+                        port,
+                        self._https_proxy_endpoint,
+                    )
+                    endpoint = HTTPConnectProxyEndpoint(
+                        self._reactor,
+                        self._https_proxy_endpoint,
+                        host,
+                        port,
+                        proxy_creds=self._https_proxy_creds,
+                    )
+                else:
+                    logger.debug("Connecting to %s:%i", host.decode("ascii"), port)
+                    # not using a proxy
+                    endpoint = HostnameEndpoint(self._reactor, host, port)
                 if self._tls_options:
                     endpoint = wrapClientTLS(self._tls_options, endpoint)
                 result = await make_deferred_yieldable(
diff --git a/synapse/http/federation/srv_resolver.py b/synapse/http/federation/srv_resolver.py
index d9620032d2..b8ed4ec905 100644
--- a/synapse/http/federation/srv_resolver.py
+++ b/synapse/http/federation/srv_resolver.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 #
diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py
index ce4079f15c..43f2140429 100644
--- a/synapse/http/federation/well_known_resolver.py
+++ b/synapse/http/federation/well_known_resolver.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -71,10 +70,8 @@
 logger = logging.getLogger(__name__)
 
 
-_well_known_cache = TTLCache("well-known")  # type: TTLCache[bytes, Optional[bytes]]
-_had_valid_well_known_cache = TTLCache(
-    "had-valid-well-known"
-)  # type: TTLCache[bytes, bool]
+_well_known_cache: TTLCache[bytes, Optional[bytes]] = TTLCache("well-known")
+_had_valid_well_known_cache: TTLCache[bytes, bool] = TTLCache("had-valid-well-known")
 
 
 @attr.s(slots=True, frozen=True)
@@ -131,9 +128,10 @@ async def get_well_known(self, server_name: bytes) -> WellKnownLookupResult:
         # requests for the same server in parallel?
         try:
             with Measure(self._clock, "get_well_known"):
-                result, cache_period = await self._fetch_well_known(
-                    server_name
-                )  # type: Optional[bytes], float
+                result: Optional[bytes]
+                cache_period: float
+
+                result, cache_period = await self._fetch_well_known(server_name)
 
         except _FetchWellKnownFailure as e:
             if prev_result and e.temporary:
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 5f01ebd3d4..2e9898997c 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
+# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,24 +11,39 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import abc
 import cgi
+import codecs
 import logging
 import random
 import sys
+import typing
 import urllib.parse
-from io import BytesIO
-from typing import Callable, Dict, List, Optional, Tuple, Union
+from io import BytesIO, StringIO
+from typing import (
+    Callable,
+    Dict,
+    Generic,
+    List,
+    Optional,
+    Tuple,
+    TypeVar,
+    Union,
+    overload,
+)
 
 import attr
 import treq
 from canonicaljson import encode_canonical_json
 from prometheus_client import Counter
 from signedjson.sign import sign_json
+from typing_extensions import Literal
 
 from twisted.internet import defer
 from twisted.internet.error import DNSLookupError
 from twisted.internet.interfaces import IReactorTime
 from twisted.internet.task import _EPSILON, Cooperator
+from twisted.web.client import ResponseFailed
 from twisted.web.http_headers import Headers
 from twisted.web.iweb import IBodyProducer, IResponse
 
@@ -46,20 +59,16 @@
 from synapse.http import QuieterFileBodyProducer
 from synapse.http.client import (
     BlacklistingAgentWrapper,
-    BlacklistingReactorWrapper,
     BodyExceededMaxSize,
+    ByteWriteable,
     encode_query_args,
     read_body_with_max_size,
 )
 from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent
+from synapse.logging import opentracing
 from synapse.logging.context import make_deferred_yieldable
-from synapse.logging.opentracing import (
-    inject_active_span_byte_dict,
-    set_tag,
-    start_active_span,
-    tags,
-)
-from synapse.types import ISynapseReactor, JsonDict
+from synapse.logging.opentracing import set_tag, start_active_span, tags
+from synapse.types import JsonDict
 from synapse.util import json_decoder
 from synapse.util.async_helpers import timeout_deferred
 from synapse.util.metrics import Measure
@@ -73,6 +82,9 @@
     "synapse_http_matrixfederationclient_responses", "", ["method", "code"]
 )
 
+# a federation response can be rather large (eg a big state_ids is 50M or so), so we
+# need a generous limit here.
+MAX_RESPONSE_SIZE = 100 * 1024 * 1024
 
 MAX_LONG_RETRIES = 10
 MAX_SHORT_RETRIES = 3
@@ -85,6 +97,27 @@
 QueryArgs = Dict[str, Union[str, List[str]]]
 
 
+T = TypeVar("T")
+
+
+class ByteParser(ByteWriteable, Generic[T], abc.ABC):
+    """A `ByteWriteable` that has an additional `finish` function that returns
+    the parsed data.
+    """
+
+    CONTENT_TYPE: str = abc.abstractproperty()  # type: ignore
+    """The expected content type of the response, e.g. `application/json`. If
+    the content type doesn't match we fail the request.
+    """
+
+    @abc.abstractmethod
+    def finish(self) -> T:
+        """Called when response has finished streaming and the parser should
+        return the final result (or error).
+        """
+        pass
+
+
 @attr.s(slots=True, frozen=True)
 class MatrixFederationRequest:
     method = attr.ib(type=str)
@@ -145,15 +178,33 @@ def get_json(self) -> Optional[JsonDict]:
         return self.json
 
 
-async def _handle_json_response(
+class JsonParser(ByteParser[Union[JsonDict, list]]):
+    """A parser that buffers the response and tries to parse it as JSON."""
+
+    CONTENT_TYPE = "application/json"
+
+    def __init__(self):
+        self._buffer = StringIO()
+        self._binary_wrapper = BinaryIOWrapper(self._buffer)
+
+    def write(self, data: bytes) -> int:
+        return self._binary_wrapper.write(data)
+
+    def finish(self) -> Union[JsonDict, list]:
+        return json_decoder.decode(self._buffer.getvalue())
+
+
+async def _handle_response(
     reactor: IReactorTime,
     timeout_sec: float,
     request: MatrixFederationRequest,
     response: IResponse,
     start_ms: int,
-) -> JsonDict:
+    parser: ByteParser[T],
+    max_response_size: Optional[int] = None,
+) -> T:
     """
-    Reads the JSON body of a response, with a timeout
+    Reads the body of a response with a timeout and sends it to a parser
 
     Args:
         reactor: twisted reactor, for the timeout
@@ -161,23 +212,41 @@ async def _handle_json_response(
         request: the request that triggered the response
         response: response to the request
         start_ms: Timestamp when request was made
+        parser: The parser for the response
+        max_response_size: The maximum size to read from the response, if None
+            uses the default.
 
     Returns:
-        The parsed JSON response
+        The parsed response
     """
+
+    if max_response_size is None:
+        max_response_size = MAX_RESPONSE_SIZE
+
     try:
-        check_content_type_is_json(response.headers)
+        check_content_type_is(response.headers, parser.CONTENT_TYPE)
 
-        # Use the custom JSON decoder (partially re-implements treq.json_content).
-        d = treq.text_content(response, encoding="utf-8")
-        d.addCallback(json_decoder.decode)
+        d = read_body_with_max_size(response, parser, max_response_size)
         d = timeout_deferred(d, timeout=timeout_sec, reactor=reactor)
 
-        body = await make_deferred_yieldable(d)
+        length = await make_deferred_yieldable(d)
+
+        value = parser.finish()
+    except BodyExceededMaxSize as e:
+        # The response was too big.
+        logger.warning(
+            "{%s} [%s] JSON response exceeded max size %i - %s %s",
+            request.txn_id,
+            request.destination,
+            MAX_RESPONSE_SIZE,
+            request.method,
+            request.uri.decode("ascii"),
+        )
+        raise RequestSendFailed(e, can_retry=False) from e
     except ValueError as e:
-        # The JSON content was invalid.
+        # The content was invalid.
         logger.warning(
-            "{%s} [%s] Failed to parse JSON response - %s %s",
+            "{%s} [%s] Failed to parse response - %s %s",
             request.txn_id,
             request.destination,
             request.method,
@@ -193,6 +262,15 @@ async def _handle_json_response(
             request.uri.decode("ascii"),
         )
         raise RequestSendFailed(e, can_retry=True) from e
+    except ResponseFailed as e:
+        logger.warning(
+            "{%s} [%s] Failed to read response - %s %s",
+            request.txn_id,
+            request.destination,
+            request.method,
+            request.uri.decode("ascii"),
+        )
+        raise RequestSendFailed(e, can_retry=True) from e
     except Exception as e:
         logger.warning(
             "{%s} [%s] Error reading response %s %s: %s",
@@ -207,16 +285,29 @@ async def _handle_json_response(
     time_taken_secs = reactor.seconds() - start_ms / 1000
 
     logger.info(
-        "{%s} [%s] Completed request: %d %s in %.2f secs - %s %s",
+        "{%s} [%s] Completed request: %d %s in %.2f secs, got %d bytes - %s %s",
         request.txn_id,
         request.destination,
         response.code,
         response.phrase.decode("ascii", errors="replace"),
         time_taken_secs,
+        length,
         request.method,
         request.uri.decode("ascii"),
     )
-    return body
+    return value
+
+
+class BinaryIOWrapper:
+    """A wrapper for a TextIO which converts from bytes on the fly."""
+
+    def __init__(self, file: typing.TextIO, encoding="utf-8", errors="strict"):
+        self.decoder = codecs.getincrementaldecoder(encoding)(errors)
+        self.file = file
+
+    def write(self, b: Union[bytes, bytearray]) -> int:
+        self.file.write(self.decoder.decode(b))
+        return len(b)
 
 
 class MatrixFederationHttpClient:
@@ -233,11 +324,7 @@ def __init__(self, hs, tls_client_options_factory):
         self.signing_key = hs.signing_key
         self.server_name = hs.hostname
 
-        # We need to use a DNS resolver which filters out blacklisted IP
-        # addresses, to prevent DNS rebinding.
-        self.reactor = BlacklistingReactorWrapper(
-            hs.get_reactor(), None, hs.config.federation_ip_range_blacklist
-        )  # type: ISynapseReactor
+        self.reactor = hs.get_reactor()
 
         user_agent = hs.version_string
         if hs.config.user_agent_suffix:
@@ -248,6 +335,7 @@ def __init__(self, hs, tls_client_options_factory):
             self.reactor,
             tls_client_options_factory,
             user_agent,
+            hs.config.federation_ip_range_whitelist,
             hs.config.federation_ip_range_blacklist,
         )
 
@@ -272,7 +360,7 @@ async def _send_request_with_optional_trailing_slash(
         self,
         request: MatrixFederationRequest,
         try_trailing_slash_on_400: bool = False,
-        **send_request_args
+        **send_request_args,
     ) -> IResponse:
         """Wrapper for _send_request which can optionally retry the request
         upon receiving a combination of a 400 HTTP response code and a
@@ -410,8 +498,8 @@ async def _send_request(
         )
 
         # Inject the span into the headers
-        headers_dict = {}  # type: Dict[bytes, List[bytes]]
-        inject_active_span_byte_dict(headers_dict, request.destination)
+        headers_dict: Dict[bytes, List[bytes]] = {}
+        opentracing.inject_header_dict(headers_dict, request.destination)
 
         headers_dict[b"User-Agent"] = [self.version_string_bytes]
 
@@ -439,9 +527,9 @@ async def _send_request(
                             destination_bytes, method_bytes, url_to_sign_bytes, json
                         )
                         data = encode_canonical_json(json)
-                        producer = QuieterFileBodyProducer(
+                        producer: Optional[IBodyProducer] = QuieterFileBodyProducer(
                             BytesIO(data), cooperator=self._cooperator
-                        )  # type: Optional[IBodyProducer]
+                        )
                     else:
                         producer = None
                         auth_headers = self.build_auth_headers(
@@ -641,6 +729,7 @@ def build_auth_headers(
             )
         return auth_headers
 
+    @overload
     async def put_json(
         self,
         destination: str,
@@ -653,7 +742,44 @@ async def put_json(
         ignore_backoff: bool = False,
         backoff_on_404: bool = False,
         try_trailing_slash_on_400: bool = False,
+        parser: Literal[None] = None,
+        max_response_size: Optional[int] = None,
     ) -> Union[JsonDict, list]:
+        ...
+
+    @overload
+    async def put_json(
+        self,
+        destination: str,
+        path: str,
+        args: Optional[QueryArgs] = None,
+        data: Optional[JsonDict] = None,
+        json_data_callback: Optional[Callable[[], JsonDict]] = None,
+        long_retries: bool = False,
+        timeout: Optional[int] = None,
+        ignore_backoff: bool = False,
+        backoff_on_404: bool = False,
+        try_trailing_slash_on_400: bool = False,
+        parser: Optional[ByteParser[T]] = None,
+        max_response_size: Optional[int] = None,
+    ) -> T:
+        ...
+
+    async def put_json(
+        self,
+        destination: str,
+        path: str,
+        args: Optional[QueryArgs] = None,
+        data: Optional[JsonDict] = None,
+        json_data_callback: Optional[Callable[[], JsonDict]] = None,
+        long_retries: bool = False,
+        timeout: Optional[int] = None,
+        ignore_backoff: bool = False,
+        backoff_on_404: bool = False,
+        try_trailing_slash_on_400: bool = False,
+        parser: Optional[ByteParser] = None,
+        max_response_size: Optional[int] = None,
+    ):
         """Sends the specified json data using PUT
 
         Args:
@@ -686,6 +812,10 @@ async def put_json(
                 of the request. Workaround for #3622 in Synapse <= v0.99.3. This
                 will be attempted before backing off if backing off has been
                 enabled.
+            parser: The parser to use to decode the response. Defaults to
+                parsing as JSON.
+            max_response_size: The maximum size to read from the response, if None
+                uses the default.
 
         Returns:
             Succeeds when we get a 2xx HTTP response. The
@@ -726,8 +856,17 @@ async def put_json(
         else:
             _sec_timeout = self.default_timeout
 
-        body = await _handle_json_response(
-            self.reactor, _sec_timeout, request, response, start_ms
+        if parser is None:
+            parser = JsonParser()
+
+        body = await _handle_response(
+            self.reactor,
+            _sec_timeout,
+            request,
+            response,
+            start_ms,
+            parser=parser,
+            max_response_size=max_response_size,
         )
 
         return body
@@ -800,12 +939,8 @@ async def post_json(
         else:
             _sec_timeout = self.default_timeout
 
-        body = await _handle_json_response(
-            self.reactor,
-            _sec_timeout,
-            request,
-            response,
-            start_ms,
+        body = await _handle_response(
+            self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser()
         )
         return body
 
@@ -877,8 +1012,8 @@ async def get_json(
         else:
             _sec_timeout = self.default_timeout
 
-        body = await _handle_json_response(
-            self.reactor, _sec_timeout, request, response, start_ms
+        body = await _handle_response(
+            self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser()
         )
 
         return body
@@ -945,8 +1080,8 @@ async def delete_json(
         else:
             _sec_timeout = self.default_timeout
 
-        body = await _handle_json_response(
-            self.reactor, _sec_timeout, request, response, start_ms
+        body = await _handle_response(
+            self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser()
         )
         return body
 
@@ -1006,6 +1141,24 @@ async def get_file(
                 msg,
             )
             raise SynapseError(502, msg, Codes.TOO_LARGE)
+        except defer.TimeoutError as e:
+            logger.warning(
+                "{%s} [%s] Timed out reading response - %s %s",
+                request.txn_id,
+                request.destination,
+                request.method,
+                request.uri.decode("ascii"),
+            )
+            raise RequestSendFailed(e, can_retry=True) from e
+        except ResponseFailed as e:
+            logger.warning(
+                "{%s} [%s] Failed to read response - %s %s",
+                request.txn_id,
+                request.destination,
+                request.method,
+                request.uri.decode("ascii"),
+            )
+            raise RequestSendFailed(e, can_retry=True) from e
         except Exception as e:
             logger.warning(
                 "{%s} [%s] Error reading response: %s",
@@ -1038,16 +1191,16 @@ def _flatten_response_never_received(e):
         return repr(e)
 
 
-def check_content_type_is_json(headers: Headers) -> None:
+def check_content_type_is(headers: Headers, expected_content_type: str) -> None:
     """
     Check that a set of HTTP headers have a Content-Type header, and that it
-    is application/json.
+    is the expected value..
 
     Args:
         headers: headers to check
 
     Raises:
-        RequestSendFailed: if the Content-Type header is missing or isn't JSON
+        RequestSendFailed: if the Content-Type header is missing or doesn't match
 
     """
     content_type_headers = headers.getRawHeaders(b"Content-Type")
@@ -1059,11 +1212,10 @@ def check_content_type_is_json(headers: Headers) -> None:
 
     c_type = content_type_headers[0].decode("ascii")  # only the first header
     val, options = cgi.parse_header(c_type)
-    if val != "application/json":
+    if val != expected_content_type:
         raise RequestSendFailed(
             RuntimeError(
-                "Remote server sent Content-Type header of '%s', not 'application/json'"
-                % c_type,
+                f"Remote server sent Content-Type header of '{c_type}', not '{expected_content_type}'",
             ),
             can_retry=False,
         )
diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py
index 16ec850064..a3f31452d0 100644
--- a/synapse/http/proxyagent.py
+++ b/synapse/http/proxyagent.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,46 +11,39 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import base64
 import logging
 import re
-from typing import Optional, Tuple
-from urllib.request import getproxies_environment, proxy_bypass_environment
+from typing import Any, Dict, Optional, Tuple
+from urllib.parse import urlparse
+from urllib.request import (  # type: ignore[attr-defined]
+    getproxies_environment,
+    proxy_bypass_environment,
+)
 
-import attr
 from zope.interface import implementer
 
 from twisted.internet import defer
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
+from twisted.internet.interfaces import IReactorCore, IStreamClientEndpoint
 from twisted.python.failure import Failure
-from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _AgentBase
+from twisted.web.client import (
+    URI,
+    BrowserLikePolicyForHTTPS,
+    HTTPConnectionPool,
+    _AgentBase,
+)
 from twisted.web.error import SchemeNotSupported
 from twisted.web.http_headers import Headers
-from twisted.web.iweb import IAgent
+from twisted.web.iweb import IAgent, IBodyProducer, IPolicyForHTTPS
 
-from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint
+from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint, ProxyCredentials
+from synapse.types import ISynapseReactor
 
 logger = logging.getLogger(__name__)
 
 _VALID_URI = re.compile(br"\A[\x21-\x7e]+\Z")
 
 
-@attr.s
-class ProxyCredentials:
-    username_password = attr.ib(type=bytes)
-
-    def as_proxy_authorization_value(self) -> bytes:
-        """
-        Return the value for a Proxy-Authorization header (i.e. 'Basic abdef==').
-
-        Returns:
-            A transformation of the authentication string the encoded value for
-            a Proxy-Authorization header.
-        """
-        # Encode as base64 and prepend the authorization type
-        return b"Basic " + base64.encodebytes(self.username_password)
-
-
 @implementer(IAgent)
 class ProxyAgent(_AgentBase):
     """An Agent implementation which will use an HTTP proxy if one was requested
@@ -64,36 +56,42 @@ class ProxyAgent(_AgentBase):
                        reactor might have some blacklisting applied (i.e. for DNS queries),
                        but we need unblocked access to the proxy.
 
-        contextFactory (IPolicyForHTTPS): A factory for TLS contexts, to control the
+        contextFactory: A factory for TLS contexts, to control the
             verification parameters of OpenSSL.  The default is to use a
             `BrowserLikePolicyForHTTPS`, so unless you have special
             requirements you can leave this as-is.
 
-        connectTimeout (Optional[float]): The amount of time that this Agent will wait
+        connectTimeout: The amount of time that this Agent will wait
             for the peer to accept a connection, in seconds. If 'None',
             HostnameEndpoint's default (30s) will be used.
-
             This is used for connections to both proxies and destination servers.
 
-        bindAddress (bytes): The local address for client sockets to bind to.
+        bindAddress: The local address for client sockets to bind to.
 
-        pool (HTTPConnectionPool|None): connection pool to be used. If None, a
+        pool: connection pool to be used. If None, a
             non-persistent pool instance will be created.
 
-        use_proxy (bool): Whether proxy settings should be discovered and used
+        use_proxy: Whether proxy settings should be discovered and used
             from conventional environment variables.
+
+    Raises:
+        ValueError if use_proxy is set and the environment variables
+            contain an invalid proxy specification.
+        RuntimeError if no tls_options_factory is given for a https connection
     """
 
     def __init__(
         self,
-        reactor,
-        proxy_reactor=None,
-        contextFactory=BrowserLikePolicyForHTTPS(),
-        connectTimeout=None,
-        bindAddress=None,
-        pool=None,
-        use_proxy=False,
+        reactor: IReactorCore,
+        proxy_reactor: Optional[ISynapseReactor] = None,
+        contextFactory: Optional[IPolicyForHTTPS] = None,
+        connectTimeout: Optional[float] = None,
+        bindAddress: Optional[bytes] = None,
+        pool: Optional[HTTPConnectionPool] = None,
+        use_proxy: bool = False,
     ):
+        contextFactory = contextFactory or BrowserLikePolicyForHTTPS()
+
         _AgentBase.__init__(self, reactor, pool)
 
         if proxy_reactor is None:
@@ -101,7 +99,7 @@ def __init__(
         else:
             self.proxy_reactor = proxy_reactor
 
-        self._endpoint_kwargs = {}
+        self._endpoint_kwargs: Dict[str, Any] = {}
         if connectTimeout is not None:
             self._endpoint_kwargs["timeout"] = connectTimeout
         if bindAddress is not None:
@@ -116,15 +114,12 @@ def __init__(
             https_proxy = proxies["https"].encode() if "https" in proxies else None
             no_proxy = proxies["no"] if "no" in proxies else None
 
-        # Parse credentials from https proxy connection string if present
-        self.https_proxy_creds, https_proxy = parse_username_password(https_proxy)
-
-        self.http_proxy_endpoint = _http_proxy_endpoint(
-            http_proxy, self.proxy_reactor, **self._endpoint_kwargs
+        self.http_proxy_endpoint, self.http_proxy_creds = http_proxy_endpoint(
+            http_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs
         )
 
-        self.https_proxy_endpoint = _http_proxy_endpoint(
-            https_proxy, self.proxy_reactor, **self._endpoint_kwargs
+        self.https_proxy_endpoint, self.https_proxy_creds = http_proxy_endpoint(
+            https_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs
         )
 
         self.no_proxy = no_proxy
@@ -132,7 +127,13 @@ def __init__(
         self._policy_for_https = contextFactory
         self._reactor = reactor
 
-    def request(self, method, uri, headers=None, bodyProducer=None):
+    def request(
+        self,
+        method: bytes,
+        uri: bytes,
+        headers: Optional[Headers] = None,
+        bodyProducer: Optional[IBodyProducer] = None,
+    ) -> defer.Deferred:
         """
         Issue a request to the server indicated by the given uri.
 
@@ -144,16 +145,15 @@ def request(self, method, uri, headers=None, bodyProducer=None):
         See also: twisted.web.iweb.IAgent.request
 
         Args:
-            method (bytes): The request method to use, such as `GET`, `POST`, etc
+            method: The request method to use, such as `GET`, `POST`, etc
 
-            uri (bytes): The location of the resource to request.
+            uri: The location of the resource to request.
 
-            headers (Headers|None): Extra headers to send with the request
+            headers: Extra headers to send with the request
 
-            bodyProducer (IBodyProducer|None): An object which can generate bytes to
-                make up the body of this request (for example, the properly encoded
-                contents of a file for a file upload). Or, None if the request is to
-                have no body.
+            bodyProducer: An object which can generate bytes to make up the body of
+                this request (for example, the properly encoded contents of a file for
+                a file upload). Or, None if the request is to have no body.
 
         Returns:
             Deferred[IResponse]: completes when the header of the response has
@@ -170,7 +170,7 @@ def request(self, method, uri, headers=None, bodyProducer=None):
         """
         uri = uri.strip()
         if not _VALID_URI.match(uri):
-            raise ValueError("Invalid URI {!r}".format(uri))
+            raise ValueError(f"Invalid URI {uri!r}")
 
         parsed_uri = URI.fromBytes(uri)
         pool_key = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port)
@@ -188,6 +188,15 @@ def request(self, method, uri, headers=None, bodyProducer=None):
             and self.http_proxy_endpoint
             and not should_skip_proxy
         ):
+            # Determine whether we need to set Proxy-Authorization headers
+            if self.http_proxy_creds:
+                # Set a Proxy-Authorization header
+                if headers is None:
+                    headers = Headers()
+                headers.addRawHeader(
+                    b"Proxy-Authorization",
+                    self.http_proxy_creds.as_proxy_authorization_value(),
+                )
             # Cache *all* connections under the same key, since we are only
             # connecting to a single destination, the proxy:
             pool_key = ("http-proxy", self.http_proxy_endpoint)
@@ -198,22 +207,12 @@ def request(self, method, uri, headers=None, bodyProducer=None):
             and self.https_proxy_endpoint
             and not should_skip_proxy
         ):
-            connect_headers = Headers()
-
-            # Determine whether we need to set Proxy-Authorization headers
-            if self.https_proxy_creds:
-                # Set a Proxy-Authorization header
-                connect_headers.addRawHeader(
-                    b"Proxy-Authorization",
-                    self.https_proxy_creds.as_proxy_authorization_value(),
-                )
-
             endpoint = HTTPConnectProxyEndpoint(
                 self.proxy_reactor,
                 self.https_proxy_endpoint,
                 parsed_uri.host,
                 parsed_uri.port,
-                headers=connect_headers,
+                self.https_proxy_creds,
             )
         else:
             # not using a proxy
@@ -242,70 +241,95 @@ def request(self, method, uri, headers=None, bodyProducer=None):
         )
 
 
-def _http_proxy_endpoint(proxy: Optional[bytes], reactor, **kwargs):
+def http_proxy_endpoint(
+    proxy: Optional[bytes],
+    reactor: IReactorCore,
+    tls_options_factory: Optional[IPolicyForHTTPS],
+    **kwargs,
+) -> Tuple[Optional[IStreamClientEndpoint], Optional[ProxyCredentials]]:
     """Parses an http proxy setting and returns an endpoint for the proxy
 
     Args:
-        proxy: the proxy setting in the form: [<username>:<password>@]<host>[:<port>]
-            Note that compared to other apps, this function currently lacks support
-            for specifying a protocol schema (i.e. protocol://...).
+        proxy: the proxy setting in the form: [scheme://][<username>:<password>@]<host>[:<port>]
+            This currently supports http:// and https:// proxies.
+            A hostname without scheme is assumed to be http.
 
         reactor: reactor to be used to connect to the proxy
 
+        tls_options_factory: the TLS options to use when connecting through a https proxy
+
         kwargs: other args to be passed to HostnameEndpoint
 
     Returns:
-        interfaces.IStreamClientEndpoint|None: endpoint to use to connect to the proxy,
-            or None
+        a tuple of
+            endpoint to use to connect to the proxy, or None
+            ProxyCredentials or if no credentials were found, or None
+
+    Raise:
+        ValueError if proxy has no hostname or unsupported scheme.
+        RuntimeError if no tls_options_factory is given for a https connection
     """
     if proxy is None:
-        return None
+        return None, None
 
-    # Parse the connection string
-    host, port = parse_host_port(proxy, default_port=1080)
-    return HostnameEndpoint(reactor, host, port, **kwargs)
+    # Note: urlsplit/urlparse cannot be used here as that does not work (for Python
+    # 3.9+) on scheme-less proxies, e.g. host:port.
+    scheme, host, port, credentials = parse_proxy(proxy)
 
+    proxy_endpoint = HostnameEndpoint(reactor, host, port, **kwargs)
 
-def parse_username_password(proxy: bytes) -> Tuple[Optional[ProxyCredentials], bytes]:
-    """
-    Parses the username and password from a proxy declaration e.g
-    username:password@hostname:port.
+    if scheme == b"https":
+        if tls_options_factory:
+            tls_options = tls_options_factory.creatorForNetloc(host, port)
+            proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint)
+        else:
+            raise RuntimeError(
+                f"No TLS options for a https connection via proxy {proxy!s}"
+            )
 
-    Args:
-        proxy: The proxy connection string.
+    return proxy_endpoint, credentials
 
-    Returns
-        An instance of ProxyCredentials and the proxy connection string with any credentials
-        stripped, i.e u:p@host:port -> host:port. If no credentials were found, the
-        ProxyCredentials instance is replaced with None.
-    """
-    if proxy and b"@" in proxy:
-        # We use rsplit here as the password could contain an @ character
-        credentials, proxy_without_credentials = proxy.rsplit(b"@", 1)
-        return ProxyCredentials(credentials), proxy_without_credentials
 
-    return None, proxy
+def parse_proxy(
+    proxy: bytes, default_scheme: bytes = b"http", default_port: int = 1080
+) -> Tuple[bytes, bytes, int, Optional[ProxyCredentials]]:
+    """
+    Parse a proxy connection string.
 
+    Given a HTTP proxy URL, breaks it down into components and checks that it
+    has a hostname (otherwise it is not useful to us when trying to find a
+    proxy) and asserts that the URL has a scheme we support.
 
-def parse_host_port(hostport: bytes, default_port: int = None) -> Tuple[bytes, int]:
-    """
-    Parse the hostname and port from a proxy connection byte string.
 
     Args:
-        hostport: The proxy connection string. Must be in the form 'host[:port]'.
-        default_port: The default port to return if one is not found in `hostport`.
+        proxy: The proxy connection string. Must be in the form '[scheme://][<username>:<password>@]host[:port]'.
+        default_scheme: The default scheme to return if one is not found in `proxy`. Defaults to http
+        default_port: The default port to return if one is not found in `proxy`. Defaults to 1080
 
     Returns:
-        A tuple containing the hostname and port. Uses `default_port` if one was not found.
+        A tuple containing the scheme, hostname, port and ProxyCredentials.
+            If no credentials were found, the ProxyCredentials instance is replaced with None.
+
+    Raise:
+        ValueError if proxy has no hostname or unsupported scheme.
     """
-    if b":" in hostport:
-        host, port = hostport.rsplit(b":", 1)
-        try:
-            port = int(port)
-            return host, port
-        except ValueError:
-            # the thing after the : wasn't a valid port; presumably this is an
-            # IPv6 address.
-            pass
+    # First check if we have a scheme present
+    # Note: urlsplit/urlparse cannot be used (for Python # 3.9+) on scheme-less proxies, e.g. host:port.
+    if b"://" not in proxy:
+        proxy = b"".join([default_scheme, b"://", proxy])
+
+    url = urlparse(proxy)
+
+    if not url.hostname:
+        raise ValueError("Proxy URL did not contain a hostname! Please specify one.")
+
+    if url.scheme not in (b"http", b"https"):
+        raise ValueError(
+            f"Unknown proxy scheme {url.scheme!s}; only 'http' and 'https' is supported."
+        )
+
+    credentials = None
+    if url.username and url.password:
+        credentials = ProxyCredentials(b"".join([url.username, b":", url.password]))
 
-    return hostport, default_port
+    return url.scheme, url.hostname, url.port or default_port, credentials
diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 0ec5d941b8..602f93c497 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/http/server.py b/synapse/http/server.py
index fa89260850..b79fa722e9 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -82,7 +81,7 @@ def return_json_error(f: failure.Failure, request: SynapseRequest) -> None:
 
     if f.check(SynapseError):
         # mypy doesn't understand that f.check asserts the type.
-        exc = f.value  # type: SynapseError  # type: ignore
+        exc: SynapseError = f.value  # type: ignore
         error_code = exc.code
         error_dict = exc.error_dict()
 
@@ -133,7 +132,7 @@ def return_html_error(
     """
     if f.check(CodeMessageException):
         # mypy doesn't understand that f.check asserts the type.
-        cme = f.value  # type: CodeMessageException  # type: ignore
+        cme: CodeMessageException = f.value  # type: ignore
         code = cme.code
         msg = cme.msg
 
@@ -405,7 +404,7 @@ def _get_handler_for_request(
             key word arguments to pass to the callback
         """
         # At this point the path must be bytes.
-        request_path_bytes = request.path  # type: bytes  # type: ignore
+        request_path_bytes: bytes = request.path  # type: ignore
         request_path = request_path_bytes.decode("ascii")
         # Treat HEAD requests as GET requests.
         request_method = request.method
@@ -558,7 +557,7 @@ def __init__(
         request: Request,
         iterator: Iterator[bytes],
     ):
-        self._request = request  # type: Optional[Request]
+        self._request: Optional[Request] = request
         self._iterator = iterator
         self._paused = False
 
@@ -729,7 +728,7 @@ def set_cors_headers(request: Request):
     )
     request.setHeader(
         b"Access-Control-Allow-Headers",
-        b"Origin, X-Requested-With, Content-Type, Accept, Authorization, Date",
+        b"X-Requested-With, Content-Type, Authorization, Date",
     )
 
 
diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py
index 839d58d0d4..8942f29f96 100644
--- a/synapse/http/servlet.py
+++ b/synapse/http/servlet.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,43 +14,99 @@
 
 """ This module contains base REST classes for constructing REST servlets. """
 import logging
-from typing import Dict, List, Optional, Union
+from typing import (
+    TYPE_CHECKING,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    overload,
+)
+
+from typing_extensions import Literal
+
+from twisted.web.server import Request
 
 from synapse.api.errors import Codes, SynapseError
+from synapse.types import JsonDict, RoomAlias, RoomID
 from synapse.util import json_decoder
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
-def parse_integer(request, name, default=None, required=False):
+@overload
+def parse_integer(request: Request, name: str, default: int) -> int:
+    ...
+
+
+@overload
+def parse_integer(request: Request, name: str, *, required: Literal[True]) -> int:
+    ...
+
+
+@overload
+def parse_integer(
+    request: Request, name: str, default: Optional[int] = None, required: bool = False
+) -> Optional[int]:
+    ...
+
+
+def parse_integer(
+    request: Request, name: str, default: Optional[int] = None, required: bool = False
+) -> Optional[int]:
     """Parse an integer parameter from the request string
 
     Args:
         request: the twisted HTTP request.
-        name (bytes/unicode): the name of the query parameter.
-        default (int|None): value to use if the parameter is absent, defaults
-            to None.
-        required (bool): whether to raise a 400 SynapseError if the
-            parameter is absent, defaults to False.
+        name: the name of the query parameter.
+        default: value to use if the parameter is absent, defaults to None.
+        required: whether to raise a 400 SynapseError if the parameter is absent,
+            defaults to False.
 
     Returns:
-        int|None: An int value or the default.
+        An int value or the default.
 
     Raises:
         SynapseError: if the parameter is absent and required, or if the
             parameter is present and not an integer.
     """
-    return parse_integer_from_args(request.args, name, default, required)
+    args: Mapping[bytes, Sequence[bytes]] = request.args  # type: ignore
+    return parse_integer_from_args(args, name, default, required)
 
 
-def parse_integer_from_args(args, name, default=None, required=False):
+def parse_integer_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[int] = None,
+    required: bool = False,
+) -> Optional[int]:
+    """Parse an integer parameter from the request string
 
-    if not isinstance(name, bytes):
-        name = name.encode("ascii")
+    Args:
+        args: A mapping of request args as bytes to a list of bytes (e.g. request.args).
+        name: the name of the query parameter.
+        default: value to use if the parameter is absent, defaults to None.
+        required: whether to raise a 400 SynapseError if the parameter is absent,
+            defaults to False.
 
-    if name in args:
+    Returns:
+        An int value or the default.
+
+    Raises:
+        SynapseError: if the parameter is absent and required, or if the
+            parameter is present and not an integer.
+    """
+    name_bytes = name.encode("ascii")
+
+    if name_bytes in args:
         try:
-            return int(args[name][0])
+            return int(args[name_bytes][0])
         except Exception:
             message = "Query parameter %r must be an integer" % (name,)
             raise SynapseError(400, message, errcode=Codes.INVALID_PARAM)
@@ -63,36 +118,102 @@ def parse_integer_from_args(args, name, default=None, required=False):
             return default
 
 
-def parse_boolean(request, name, default=None, required=False):
+@overload
+def parse_boolean(request: Request, name: str, default: bool) -> bool:
+    ...
+
+
+@overload
+def parse_boolean(request: Request, name: str, *, required: Literal[True]) -> bool:
+    ...
+
+
+@overload
+def parse_boolean(
+    request: Request, name: str, default: Optional[bool] = None, required: bool = False
+) -> Optional[bool]:
+    ...
+
+
+def parse_boolean(
+    request: Request, name: str, default: Optional[bool] = None, required: bool = False
+) -> Optional[bool]:
     """Parse a boolean parameter from the request query string
 
     Args:
         request: the twisted HTTP request.
-        name (bytes/unicode): the name of the query parameter.
-        default (bool|None): value to use if the parameter is absent, defaults
-            to None.
-        required (bool): whether to raise a 400 SynapseError if the
-            parameter is absent, defaults to False.
+        name: the name of the query parameter.
+        default: value to use if the parameter is absent, defaults to None.
+        required: whether to raise a 400 SynapseError if the parameter is absent,
+            defaults to False.
 
     Returns:
-        bool|None: A bool value or the default.
+        A bool value or the default.
 
     Raises:
         SynapseError: if the parameter is absent and required, or if the
             parameter is present and not one of "true" or "false".
     """
+    args: Mapping[bytes, Sequence[bytes]] = request.args  # type: ignore
+    return parse_boolean_from_args(args, name, default, required)
+
+
+@overload
+def parse_boolean_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: bool,
+) -> bool:
+    ...
+
+
+@overload
+def parse_boolean_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    *,
+    required: Literal[True],
+) -> bool:
+    ...
+
+
+@overload
+def parse_boolean_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[bool] = None,
+    required: bool = False,
+) -> Optional[bool]:
+    ...
+
+
+def parse_boolean_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[bool] = None,
+    required: bool = False,
+) -> Optional[bool]:
+    """Parse a boolean parameter from the request query string
 
-    return parse_boolean_from_args(request.args, name, default, required)
-
+    Args:
+        args: A mapping of request args as bytes to a list of bytes (e.g. request.args).
+        name: the name of the query parameter.
+        default: value to use if the parameter is absent, defaults to None.
+        required: whether to raise a 400 SynapseError if the parameter is absent,
+            defaults to False.
 
-def parse_boolean_from_args(args, name, default=None, required=False):
+    Returns:
+        A bool value or the default.
 
-    if not isinstance(name, bytes):
-        name = name.encode("ascii")
+    Raises:
+        SynapseError: if the parameter is absent and required, or if the
+            parameter is present and not one of "true" or "false".
+    """
+    name_bytes = name.encode("ascii")
 
-    if name in args:
+    if name_bytes in args:
         try:
-            return {b"true": True, b"false": False}[args[name][0]]
+            return {b"true": True, b"false": False}[args[name_bytes][0]]
         except Exception:
             message = (
                 "Boolean query parameter %r must be one of ['true', 'false']"
@@ -106,15 +227,113 @@ def parse_boolean_from_args(args, name, default=None, required=False):
             return default
 
 
+@overload
+def parse_bytes_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[bytes] = None,
+) -> Optional[bytes]:
+    ...
+
+
+@overload
+def parse_bytes_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Literal[None] = None,
+    *,
+    required: Literal[True],
+) -> bytes:
+    ...
+
+
+@overload
+def parse_bytes_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[bytes] = None,
+    required: bool = False,
+) -> Optional[bytes]:
+    ...
+
+
+def parse_bytes_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[bytes] = None,
+    required: bool = False,
+) -> Optional[bytes]:
+    """
+    Parse a string parameter as bytes from the request query string.
+
+    Args:
+        args: A mapping of request args as bytes to a list of bytes (e.g. request.args).
+        name: the name of the query parameter.
+        default: value to use if the parameter is absent,
+            defaults to None. Must be bytes if encoding is None.
+        required: whether to raise a 400 SynapseError if the
+            parameter is absent, defaults to False.
+    Returns:
+        Bytes or the default value.
+
+    Raises:
+        SynapseError if the parameter is absent and required.
+    """
+    name_bytes = name.encode("ascii")
+
+    if name_bytes in args:
+        return args[name_bytes][0]
+    elif required:
+        message = "Missing string query parameter %s" % (name,)
+        raise SynapseError(400, message, errcode=Codes.MISSING_PARAM)
+
+    return default
+
+
+@overload
+def parse_string(
+    request: Request,
+    name: str,
+    default: str,
+    *,
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> str:
+    ...
+
+
+@overload
+def parse_string(
+    request: Request,
+    name: str,
+    *,
+    required: Literal[True],
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> str:
+    ...
+
+
+@overload
+def parse_string(
+    request: Request,
+    name: str,
+    *,
+    required: bool = False,
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> Optional[str]:
+    ...
+
+
 def parse_string(
-    request,
-    name,
-    default=None,
-    required=False,
-    allowed_values=None,
-    param_type="string",
-    encoding="ascii",
-):
+    request: Request,
+    name: str,
+    default: Optional[str] = None,
+    required: bool = False,
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> Optional[str]:
     """
     Parse a string parameter from the request query string.
 
@@ -123,132 +342,245 @@ def parse_string(
 
     Args:
         request: the twisted HTTP request.
-        name (bytes|unicode): the name of the query parameter.
-        default (bytes|unicode|None): value to use if the parameter is absent,
-            defaults to None. Must be bytes if encoding is None.
-        required (bool): whether to raise a 400 SynapseError if the
+        name: the name of the query parameter.
+        default: value to use if the parameter is absent, defaults to None.
+        required: whether to raise a 400 SynapseError if the
             parameter is absent, defaults to False.
-        allowed_values (list[bytes|unicode]): List of allowed values for the
+        allowed_values: List of allowed values for the
             string, or None if any value is allowed, defaults to None. Must be
             the same type as name, if given.
-        encoding (str|None): The encoding to decode the string content with.
+        encoding: The encoding to decode the string content with.
 
     Returns:
-        bytes/unicode|None: A string value or the default. Unicode if encoding
-        was given, bytes otherwise.
+        A string value or the default.
 
     Raises:
         SynapseError if the parameter is absent and required, or if the
             parameter is present, must be one of a list of allowed values and
             is not one of those allowed values.
     """
+    args: Mapping[bytes, Sequence[bytes]] = request.args  # type: ignore
     return parse_string_from_args(
-        request.args, name, default, required, allowed_values, param_type, encoding
+        args,
+        name,
+        default,
+        required=required,
+        allowed_values=allowed_values,
+        encoding=encoding,
     )
 
 
-def parse_list_from_args(
-    args: Dict[bytes, List[bytes]],
-    name: Union[bytes, str],
-    encoding: Optional[str] = "ascii",
-):
-    """Parse and optionally decode a list of values from request query parameters.
+def _parse_string_value(
+    value: bytes,
+    allowed_values: Optional[Iterable[str]],
+    name: str,
+    encoding: str,
+) -> str:
+    try:
+        value_str = value.decode(encoding)
+    except ValueError:
+        raise SynapseError(400, "Query parameter %r must be %s" % (name, encoding))
+
+    if allowed_values is not None and value_str not in allowed_values:
+        message = "Query parameter %r must be one of [%s]" % (
+            name,
+            ", ".join(repr(v) for v in allowed_values),
+        )
+        raise SynapseError(400, message)
+    else:
+        return value_str
+
+
+@overload
+def parse_strings_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    *,
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> Optional[List[str]]:
+    ...
+
+
+@overload
+def parse_strings_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: List[str],
+    *,
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> List[str]:
+    ...
+
+
+@overload
+def parse_strings_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    *,
+    required: Literal[True],
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> List[str]:
+    ...
+
+
+@overload
+def parse_strings_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[List[str]] = None,
+    *,
+    required: bool = False,
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> Optional[List[str]]:
+    ...
+
+
+def parse_strings_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[List[str]] = None,
+    required: bool = False,
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> Optional[List[str]]:
+    """
+    Parse a string parameter from the request query string list.
+
+    The content of the query param will be decoded to Unicode using the encoding.
 
     Args:
-        args: A dictionary of query parameters from a request.
-        name: The name of the query parameter to extract values from. If given as bytes,
-            will be decoded as "ascii".
-        encoding: An optional encoding that is used to decode each parameter value with.
+        args: A mapping of request args as bytes to a list of bytes (e.g. request.args).
+        name: the name of the query parameter.
+        default: value to use if the parameter is absent, defaults to None.
+        required: whether to raise a 400 SynapseError if the
+            parameter is absent, defaults to False.
+        allowed_values: List of allowed values for the
+            string, or None if any value is allowed, defaults to None.
+        encoding: The encoding to decode the string content with.
+
+    Returns:
+        A string value or the default.
 
     Raises:
-        KeyError: If the given `name` does not exist in `args`.
-        SynapseError: If an argument was not encoded with the specified `encoding`.
+        SynapseError if the parameter is absent and required, or if the
+            parameter is present, must be one of a list of allowed values and
+            is not one of those allowed values.
     """
-    if not isinstance(name, bytes):
-        name = name.encode("ascii")
-    args_list = args[name]
+    name_bytes = name.encode("ascii")
+
+    if name_bytes in args:
+        values = args[name_bytes]
+
+        return [
+            _parse_string_value(value, allowed_values, name=name, encoding=encoding)
+            for value in values
+        ]
+    else:
+        if required:
+            message = "Missing string query parameter %r" % (name,)
+            raise SynapseError(400, message, errcode=Codes.MISSING_PARAM)
+
+        return default
+
+
+@overload
+def parse_string_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[str] = None,
+    *,
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> Optional[str]:
+    ...
+
+
+@overload
+def parse_string_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[str] = None,
+    *,
+    required: Literal[True],
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> str:
+    ...
 
-    if encoding:
-        # Decode each argument value
-        try:
-            args_list = [value.decode(encoding) for value in args_list]
-        except ValueError:
-            raise SynapseError(400, "Query parameter %r must be %s" % (name, encoding))
 
-    return args_list
+@overload
+def parse_string_from_args(
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
+    default: Optional[str] = None,
+    required: bool = False,
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> Optional[str]:
+    ...
 
 
 def parse_string_from_args(
-    args: Dict[bytes, List[bytes]],
-    name: Union[bytes, str],
+    args: Mapping[bytes, Sequence[bytes]],
+    name: str,
     default: Optional[str] = None,
-    required: Optional[bool] = False,
-    allowed_values: Optional[List[bytes]] = None,
-    param_type: Optional[str] = "string",
-    encoding: Optional[str] = "ascii",
-):
-    """Parse and optionally decode a single value from request query parameters.
+    required: bool = False,
+    allowed_values: Optional[Iterable[str]] = None,
+    encoding: str = "ascii",
+) -> Optional[str]:
+    """
+    Parse the string parameter from the request query string list
+    and return the first result.
+
+    The content of the query param will be decoded to Unicode using the encoding.
 
     Args:
-        args: A dictionary of query parameters from a request.
-        name: The name of the query parameter to extract values from. If given as bytes,
-            will be decoded as "ascii".
-        default: A default value to return if the given argument `name` was not found.
-        required: If this is True, no `default` is provided and the given argument `name`
-            was not found then a SynapseError is raised.
-        allowed_values: A list of allowed values. If specified and the found str is
-            not in this list, a SynapseError is raised.
-        param_type: The expected type of the query parameter's value.
-        encoding: An optional encoding that is used to decode each parameter value with.
+        args: A mapping of request args as bytes to a list of bytes (e.g. request.args).
+        name: the name of the query parameter.
+        default: value to use if the parameter is absent, defaults to None.
+        required: whether to raise a 400 SynapseError if the
+            parameter is absent, defaults to False.
+        allowed_values: List of allowed values for the
+            string, or None if any value is allowed, defaults to None. Must be
+            the same type as name, if given.
+        encoding: The encoding to decode the string content with.
 
     Returns:
-        The found argument value.
+        A string value or the default.
 
     Raises:
-        SynapseError: If the given name was not found in the request arguments,
-        the argument's values were encoded incorrectly or a required value was missing.
+        SynapseError if the parameter is absent and required, or if the
+            parameter is present, must be one of a list of allowed values and
+            is not one of those allowed values.
     """
-    if not isinstance(name, bytes):
-        name = name.encode("ascii")
-
-    if name in args:
-        value = args[name][0]
-
-        if encoding:
-            try:
-                value = value.decode(encoding)
-            except ValueError:
-                raise SynapseError(
-                    400, "Query parameter %r must be %s" % (name, encoding)
-                )
-
-        if allowed_values is not None and value not in allowed_values:
-            message = "Query parameter %r must be one of [%s]" % (
-                name,
-                ", ".join(repr(v) for v in allowed_values),
-            )
-            raise SynapseError(400, message)
-        else:
-            return value
-    else:
-        if required:
-            message = "Missing %s query parameter %r" % (param_type, name)
-            raise SynapseError(400, message, errcode=Codes.MISSING_PARAM)
-        else:
 
-            if encoding and isinstance(default, bytes):
-                return default.decode(encoding)
+    strings = parse_strings_from_args(
+        args,
+        name,
+        default=[default] if default is not None else None,
+        required=required,
+        allowed_values=allowed_values,
+        encoding=encoding,
+    )
 
-            return default
+    if strings is None:
+        return None
+
+    return strings[0]
 
 
-def parse_json_value_from_request(request, allow_empty_body=False):
+def parse_json_value_from_request(
+    request: Request, allow_empty_body: bool = False
+) -> Optional[JsonDict]:
     """Parse a JSON value from the body of a twisted HTTP request.
 
     Args:
         request: the twisted HTTP request.
-        allow_empty_body (bool): if True, an empty body will be accepted and
-            turned into None
+        allow_empty_body: if True, an empty body will be accepted and turned into None
 
     Returns:
         The JSON value.
@@ -257,7 +589,7 @@ def parse_json_value_from_request(request, allow_empty_body=False):
         SynapseError if the request body couldn't be decoded as JSON.
     """
     try:
-        content_bytes = request.content.read()
+        content_bytes = request.content.read()  # type: ignore
     except Exception:
         raise SynapseError(400, "Error reading JSON content.")
 
@@ -267,19 +599,21 @@ def parse_json_value_from_request(request, allow_empty_body=False):
     try:
         content = json_decoder.decode(content_bytes.decode("utf-8"))
     except Exception as e:
-        logger.warning("Unable to parse JSON: %s", e)
+        logger.warning("Unable to parse JSON: %s (%s)", e, content_bytes)
         raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON)
 
     return content
 
 
-def parse_json_object_from_request(request, allow_empty_body=False):
+def parse_json_object_from_request(
+    request: Request, allow_empty_body: bool = False
+) -> JsonDict:
     """Parse a JSON object from the body of a twisted HTTP request.
 
     Args:
         request: the twisted HTTP request.
-        allow_empty_body (bool): if True, an empty body will be accepted and
-            turned into an empty dict.
+        allow_empty_body: if True, an empty body will be accepted and turned into
+            an empty dict.
 
     Raises:
         SynapseError if the request body couldn't be decoded as JSON or
@@ -290,14 +624,16 @@ def parse_json_object_from_request(request, allow_empty_body=False):
     if allow_empty_body and content is None:
         return {}
 
-    if type(content) != dict:
+    if not isinstance(content, dict):
         message = "Content must be a JSON object."
         raise SynapseError(400, message, errcode=Codes.BAD_JSON)
 
     return content
 
 
-def assert_params_in_dict(body, required):
+def assert_params_in_dict(
+    body: JsonDict, required: Iterable[Union[str, bytes]]
+) -> None:
     absent = []
     for k in required:
         if k not in body:
@@ -329,10 +665,9 @@ class attribute containing a pre-compiled regular expression. The automatic
     """
 
     def register(self, http_server):
-        """ Register this servlet with the given HTTP server. """
-        if hasattr(self, "PATTERNS"):
-            patterns = self.PATTERNS
-
+        """Register this servlet with the given HTTP server."""
+        patterns = getattr(self, "PATTERNS", None)
+        if patterns:
             for method in ("GET", "PUT", "POST", "DELETE"):
                 if hasattr(self, "on_%s" % (method,)):
                     servlet_classname = self.__class__.__name__
@@ -343,3 +678,45 @@ def register(self, http_server):
 
         else:
             raise NotImplementedError("RestServlet must register something.")
+
+
+class ResolveRoomIdMixin:
+    def __init__(self, hs: "HomeServer"):
+        self.room_member_handler = hs.get_room_member_handler()
+
+    async def resolve_room_id(
+        self, room_identifier: str, remote_room_hosts: Optional[List[str]] = None
+    ) -> Tuple[str, Optional[List[str]]]:
+        """
+        Resolve a room identifier to a room ID, if necessary.
+
+        This also performanes checks to ensure the room ID is of the proper form.
+
+        Args:
+            room_identifier: The room ID or alias.
+            remote_room_hosts: The potential remote room hosts to use.
+
+        Returns:
+            The resolved room ID.
+
+        Raises:
+            SynapseError if the room ID is of the wrong form.
+        """
+        if RoomID.is_valid(room_identifier):
+            resolved_room_id = room_identifier
+        elif RoomAlias.is_valid(room_identifier):
+            room_alias = RoomAlias.from_string(room_identifier)
+            (
+                room_id,
+                remote_room_hosts,
+            ) = await self.room_member_handler.lookup_room_alias(room_alias)
+            resolved_room_id = room_id.to_string()
+        else:
+            raise SynapseError(
+                400, "%s was not legal room ID or room alias" % (room_identifier,)
+            )
+        if not resolved_room_id:
+            raise SynapseError(
+                400, "Unknown room ID or room alias %s" % room_identifier
+            )
+        return resolved_room_id, remote_room_hosts
diff --git a/synapse/http/site.py b/synapse/http/site.py
index 47754aff43..190084e8aa 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -14,19 +14,24 @@
 import contextlib
 import logging
 import time
-from typing import Optional, Type, Union
+from typing import Optional, Tuple, Union
 
 import attr
 from zope.interface import implementer
 
-from twisted.internet.interfaces import IAddress
+from twisted.internet.interfaces import IAddress, IReactorTime
 from twisted.python.failure import Failure
+from twisted.web.resource import IResource
 from twisted.web.server import Request, Site
 
 from synapse.config.server import ListenerConfig
 from synapse.http import get_request_user_agent, redact_uri
 from synapse.http.request_metrics import RequestMetrics, requests_counter
-from synapse.logging.context import LoggingContext, PreserveLoggingContext
+from synapse.logging.context import (
+    ContextRequest,
+    LoggingContext,
+    PreserveLoggingContext,
+)
 from synapse.types import Requester
 
 logger = logging.getLogger(__name__)
@@ -45,6 +50,7 @@ class SynapseRequest(Request):
      * Redaction of access_token query-params in __repr__
      * Logging at start and end
      * Metrics to record CPU, wallclock and DB time by endpoint.
+     * A limit to the size of request which will be accepted
 
     It also provides a method `processing`, which returns a context manager. If this
     method is called, the request won't be logged until the context manager is closed;
@@ -55,18 +61,19 @@ class SynapseRequest(Request):
         logcontext: the log context for this request
     """
 
-    def __init__(self, channel, *args, **kw):
+    def __init__(self, channel, *args, max_request_body_size=1024, **kw):
         Request.__init__(self, channel, *args, **kw)
-        self.site = channel.site  # type: SynapseSite
+        self._max_request_body_size = max_request_body_size
+        self.site: SynapseSite = channel.site
         self._channel = channel  # this is used by the tests
         self.start_time = 0.0
 
         # The requester, if authenticated. For federation requests this is the
         # server name, for client requests this is the Requester object.
-        self.requester = None  # type: Optional[Union[Requester, str]]
+        self._requester: Optional[Union[Requester, str]] = None
 
         # we can't yet create the logcontext, as we don't know the method.
-        self.logcontext = None  # type: Optional[LoggingContext]
+        self.logcontext: Optional[LoggingContext] = None
 
         global _next_request_seq
         self.request_seq = _next_request_seq
@@ -93,6 +100,45 @@ def __repr__(self):
             self.site.site_tag,
         )
 
+    def handleContentChunk(self, data):
+        # we should have a `content` by now.
+        assert self.content, "handleContentChunk() called before gotLength()"
+        if self.content.tell() + len(data) > self._max_request_body_size:
+            logger.warning(
+                "Aborting connection from %s because the request exceeds maximum size: %s %s",
+                self.client,
+                self.get_method(),
+                self.get_redacted_uri(),
+            )
+            self.transport.abortConnection()
+            return
+        super().handleContentChunk(data)
+
+    @property
+    def requester(self) -> Optional[Union[Requester, str]]:
+        return self._requester
+
+    @requester.setter
+    def requester(self, value: Union[Requester, str]) -> None:
+        # Store the requester, and update some properties based on it.
+
+        # This should only be called once.
+        assert self._requester is None
+
+        self._requester = value
+
+        # A logging context should exist by now (and have a ContextRequest).
+        assert self.logcontext is not None
+        assert self.logcontext.request is not None
+
+        (
+            requester,
+            authenticated_entity,
+        ) = self.get_authenticated_entity()
+        self.logcontext.request.requester = requester
+        # If there's no authenticated entity, it was the requester.
+        self.logcontext.request.authenticated_entity = authenticated_entity or requester
+
     def get_request_id(self):
         return "%s-%i" % (self.get_method(), self.request_seq)
 
@@ -106,7 +152,7 @@ def get_redacted_uri(self) -> str:
         Returns:
             The redacted URI as a string.
         """
-        uri = self.uri  # type: Union[bytes, str]
+        uri: Union[bytes, str] = self.uri
         if isinstance(uri, bytes):
             uri = uri.decode("ascii", errors="replace")
         return redact_uri(uri)
@@ -121,18 +167,65 @@ def get_method(self) -> str:
         Returns:
             The request method as a string.
         """
-        method = self.method  # type: Union[bytes, str]
+        method: Union[bytes, str] = self.method
         if isinstance(method, bytes):
             return self.method.decode("ascii")
         return method
 
+    def get_authenticated_entity(self) -> Tuple[Optional[str], Optional[str]]:
+        """
+        Get the "authenticated" entity of the request, which might be the user
+        performing the action, or a user being puppeted by a server admin.
+
+        Returns:
+            A tuple:
+                The first item is a string representing the user making the request.
+
+                The second item is a string or None representing the user who
+                authenticated when making this request. See
+                Requester.authenticated_entity.
+        """
+        # Convert the requester into a string that we can log
+        if isinstance(self._requester, str):
+            return self._requester, None
+        elif isinstance(self._requester, Requester):
+            requester = self._requester.user.to_string()
+            authenticated_entity = self._requester.authenticated_entity
+
+            # If this is a request where the target user doesn't match the user who
+            # authenticated (e.g. and admin is puppetting a user) then we return both.
+            if self._requester.user.to_string() != authenticated_entity:
+                return requester, authenticated_entity
+
+            return requester, None
+        elif self._requester is not None:
+            # This shouldn't happen, but we log it so we don't lose information
+            # and can see that we're doing something wrong.
+            return repr(self._requester), None  # type: ignore[unreachable]
+
+        return None, None
+
     def render(self, resrc):
         # this is called once a Resource has been found to serve the request; in our
         # case the Resource in question will normally be a JsonResource.
 
         # create a LogContext for this request
         request_id = self.get_request_id()
-        self.logcontext = LoggingContext(request_id, request=request_id)
+        self.logcontext = LoggingContext(
+            request_id,
+            request=ContextRequest(
+                request_id=request_id,
+                ip_address=self.getClientIP(),
+                site_tag=self.site.site_tag,
+                # The requester is going to be unknown at this point.
+                requester=None,
+                authenticated_entity=None,
+                method=self.get_method(),
+                url=self.get_redacted_uri(),
+                protocol=self.clientproto.decode("ascii", errors="replace"),
+                user_agent=get_request_user_agent(self),
+            ),
+        )
 
         # override the Server header which is set by twisted
         self.setHeader("Server", self.site.server_version_string)
@@ -277,25 +370,6 @@ def _finished_processing(self):
         # to the client (nb may be negative)
         response_send_time = self.finish_time - self._processing_finished_time
 
-        # Convert the requester into a string that we can log
-        authenticated_entity = None
-        if isinstance(self.requester, str):
-            authenticated_entity = self.requester
-        elif isinstance(self.requester, Requester):
-            authenticated_entity = self.requester.authenticated_entity
-
-            # If this is a request where the target user doesn't match the user who
-            # authenticated (e.g. and admin is puppetting a user) then we log both.
-            if self.requester.user.to_string() != authenticated_entity:
-                authenticated_entity = "{},{}".format(
-                    authenticated_entity,
-                    self.requester.user.to_string(),
-                )
-        elif self.requester is not None:
-            # This shouldn't happen, but we log it so we don't lose information
-            # and can see that we're doing something wrong.
-            authenticated_entity = repr(self.requester)  # type: ignore[unreachable]
-
         user_agent = get_request_user_agent(self, "-")
 
         code = str(self.code)
@@ -305,6 +379,13 @@ def _finished_processing(self):
             code += "!"
 
         log_level = logging.INFO if self._should_log_request() else logging.DEBUG
+
+        # If this is a request where the target user doesn't match the user who
+        # authenticated (e.g. and admin is puppetting a user) then we log both.
+        requester, authenticated_entity = self.get_authenticated_entity()
+        if authenticated_entity:
+            requester = f"{authenticated_entity}.{requester}"
+
         self.site.access_logger.log(
             log_level,
             "%s - %s - {%s}"
@@ -312,7 +393,7 @@ def _finished_processing(self):
             ' %sB %s "%s %s %s" "%s" [%d dbevts]',
             self.getClientIP(),
             self.site.site_tag,
-            authenticated_entity,
+            requester,
             processing_time,
             response_send_time,
             usage.ru_utime,
@@ -353,8 +434,8 @@ class XForwardedForRequest(SynapseRequest):
     """
 
     # the client IP and ssl flag, as extracted from the headers.
-    _forwarded_for = None  # type: Optional[_XForwardedForAddress]
-    _forwarded_https = False  # type: bool
+    _forwarded_for: "Optional[_XForwardedForAddress]" = None
+    _forwarded_https: bool = False
 
     def requestReceived(self, command, path, version):
         # this method is called by the Channel once the full request has been
@@ -421,29 +502,55 @@ class _XForwardedForAddress:
 
 class SynapseSite(Site):
     """
-    Subclass of a twisted http Site that does access logging with python's
-    standard logging
+    Synapse-specific twisted http Site
+
+    This does two main things.
+
+    First, it replaces the requestFactory in use so that we build SynapseRequests
+    instead of regular t.w.server.Requests. All of the  constructor params are really
+    just parameters for SynapseRequest.
+
+    Second, it inhibits the log() method called by Request.finish, since SynapseRequest
+    does its own logging.
     """
 
     def __init__(
         self,
-        logger_name,
-        site_tag,
+        logger_name: str,
+        site_tag: str,
         config: ListenerConfig,
-        resource,
+        resource: IResource,
         server_version_string,
-        *args,
-        **kwargs
+        max_request_body_size: int,
+        reactor: IReactorTime,
     ):
-        Site.__init__(self, resource, *args, **kwargs)
+        """
+
+        Args:
+            logger_name:  The name of the logger to use for access logs.
+            site_tag:  A tag to use for this site - mostly in access logs.
+            config:  Configuration for the HTTP listener corresponding to this site
+            resource:  The base of the resource tree to be used for serving requests on
+                this site
+            server_version_string: A string to present for the Server header
+            max_request_body_size: Maximum request body length to allow before
+                dropping the connection
+            reactor: reactor to be used to manage connection timeouts
+        """
+        Site.__init__(self, resource, reactor=reactor)
 
         self.site_tag = site_tag
 
         assert config.http_options is not None
         proxied = config.http_options.x_forwarded
-        self.requestFactory = (
-            XForwardedForRequest if proxied else SynapseRequest
-        )  # type: Type[Request]
+        request_class = XForwardedForRequest if proxied else SynapseRequest
+
+        def request_factory(channel, queued) -> Request:
+            return request_class(
+                channel, max_request_body_size=max_request_body_size, queued=queued
+            )
+
+        self.requestFactory = request_factory  # type: ignore
         self.access_logger = logging.getLogger(logger_name)
         self.server_version_string = server_version_string.encode("ascii")
 
diff --git a/synapse/logging/__init__.py b/synapse/logging/__init__.py
index b28b7b2ef7..b50a4f95eb 100644
--- a/synapse/logging/__init__.py
+++ b/synapse/logging/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,8 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# These are imported to allow for nicer logging configuration files.
+import logging
+
 from synapse.logging._remote import RemoteHandler
 from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter
 
+# These are imported to allow for nicer logging configuration files.
 __all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter"]
+
+# Debug logger for https://github.com/matrix-org/synapse/issues/9533 etc
+issue9533_logger = logging.getLogger("synapse.9533_debug")
diff --git a/synapse/logging/_remote.py b/synapse/logging/_remote.py
index 643492ceaf..8202d0494d 100644
--- a/synapse/logging/_remote.py
+++ b/synapse/logging/_remote.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -111,9 +110,9 @@ def __init__(
         self.port = port
         self.maximum_buffer = maximum_buffer
 
-        self._buffer = deque()  # type: Deque[logging.LogRecord]
-        self._connection_waiter = None  # type: Optional[Deferred]
-        self._producer = None  # type: Optional[LogProducer]
+        self._buffer: Deque[logging.LogRecord] = deque()
+        self._connection_waiter: Optional[Deferred] = None
+        self._producer: Optional[LogProducer] = None
 
         # Connect without DNS lookups if it's a direct IP.
         if _reactor is None:
@@ -124,9 +123,9 @@ def __init__(
         try:
             ip = ip_address(self.host)
             if isinstance(ip, IPv4Address):
-                endpoint = TCP4ClientEndpoint(
+                endpoint: IStreamClientEndpoint = TCP4ClientEndpoint(
                     _reactor, self.host, self.port
-                )  # type: IStreamClientEndpoint
+                )
             elif isinstance(ip, IPv6Address):
                 endpoint = TCP6ClientEndpoint(_reactor, self.host, self.port)
             else:
@@ -166,7 +165,7 @@ def fail(failure: Failure) -> None:
         def writer(result: Protocol) -> None:
             # Force recognising transport as a Connection and not the more
             # generic ITransport.
-            transport = result.transport  # type: Connection  # type: ignore
+            transport: Connection = result.transport  # type: ignore
 
             # We have a connection. If we already have a producer, and its
             # transport is the same, just trigger a resumeProducing.
@@ -189,7 +188,7 @@ def writer(result: Protocol) -> None:
             self._producer.resumeProducing()
             self._connection_waiter = None
 
-        deferred = self._service.whenConnected(failAfterFailures=1)  # type: Deferred
+        deferred: Deferred = self._service.whenConnected(failAfterFailures=1)
         deferred.addCallbacks(writer, fail)
         self._connection_waiter = deferred
 
@@ -227,11 +226,11 @@ def _handle_pressure(self) -> None:
         old_buffer = self._buffer
         self._buffer = deque()
 
-        for i in range(buffer_split):
+        for _ in range(buffer_split):
             self._buffer.append(old_buffer.popleft())
 
         end_buffer = []
-        for i in range(buffer_split):
+        for _ in range(buffer_split):
             end_buffer.append(old_buffer.pop())
 
         self._buffer.extend(reversed(end_buffer))
diff --git a/synapse/logging/_structured.py b/synapse/logging/_structured.py
index 3e054f615c..b9933a1528 100644
--- a/synapse/logging/_structured.py
+++ b/synapse/logging/_structured.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -64,7 +63,7 @@ def parse_drain_configs(
             DrainType.CONSOLE_JSON,
             DrainType.FILE_JSON,
         ):
-            formatter = "json"  # type: Optional[str]
+            formatter: Optional[str] = "json"
         elif logging_type in (
             DrainType.CONSOLE_JSON_TERSE,
             DrainType.NETWORK_JSON_TERSE,
diff --git a/synapse/logging/_terse_json.py b/synapse/logging/_terse_json.py
index 2fbf5549a1..6e82f7c7f1 100644
--- a/synapse/logging/_terse_json.py
+++ b/synapse/logging/_terse_json.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,8 +20,9 @@
 
 _encoder = json.JSONEncoder(ensure_ascii=False, separators=(",", ":"))
 
-# The properties of a standard LogRecord.
-_LOG_RECORD_ATTRIBUTES = {
+# The properties of a standard LogRecord that should be ignored when generating
+# JSON logs.
+_IGNORED_LOG_RECORD_ATTRIBUTES = {
     "args",
     "asctime",
     "created",
@@ -60,9 +60,9 @@ def format(self, record: logging.LogRecord) -> str:
         return self._format(record, event)
 
     def _format(self, record: logging.LogRecord, event: dict) -> str:
-        # Add any extra attributes to the event.
+        # Add attributes specified via the extra keyword to the logged event.
         for key, value in record.__dict__.items():
-            if key not in _LOG_RECORD_ATTRIBUTES:
+            if key not in _IGNORED_LOG_RECORD_ATTRIBUTES:
                 event[key] = value
 
         return _encoder.encode(event)
diff --git a/synapse/logging/context.py b/synapse/logging/context.py
index 03cf3c2b8e..02e5ddd2ef 100644
--- a/synapse/logging/context.py
+++ b/synapse/logging/context.py
@@ -22,14 +22,14 @@
 
 See doc/log_contexts.rst for details on how this works.
 """
-
 import inspect
 import logging
 import threading
-import types
+import typing
 import warnings
 from typing import TYPE_CHECKING, Optional, Tuple, TypeVar, Union
 
+import attr
 from typing_extensions import Literal
 
 from twisted.internet import defer, threads
@@ -113,13 +113,13 @@ def __init__(self, copy_from: "Optional[ContextResourceUsage]" = None) -> None:
             self.reset()
         else:
             # FIXME: mypy can't infer the types set via reset() above, so specify explicitly for now
-            self.ru_utime = copy_from.ru_utime  # type: float
-            self.ru_stime = copy_from.ru_stime  # type: float
-            self.db_txn_count = copy_from.db_txn_count  # type: int
+            self.ru_utime: float = copy_from.ru_utime
+            self.ru_stime: float = copy_from.ru_stime
+            self.db_txn_count: int = copy_from.db_txn_count
 
-            self.db_txn_duration_sec = copy_from.db_txn_duration_sec  # type: float
-            self.db_sched_duration_sec = copy_from.db_sched_duration_sec  # type: float
-            self.evt_db_fetch_count = copy_from.evt_db_fetch_count  # type: int
+            self.db_txn_duration_sec: float = copy_from.db_txn_duration_sec
+            self.db_sched_duration_sec: float = copy_from.db_sched_duration_sec
+            self.evt_db_fetch_count: int = copy_from.evt_db_fetch_count
 
     def copy(self) -> "ContextResourceUsage":
         return ContextResourceUsage(copy_from=self)
@@ -181,6 +181,29 @@ def __sub__(self, other: "ContextResourceUsage") -> "ContextResourceUsage":
         return res
 
 
+@attr.s(slots=True)
+class ContextRequest:
+    """
+    A bundle of attributes from the SynapseRequest object.
+
+    This exists to:
+
+    * Avoid a cycle between LoggingContext and SynapseRequest.
+    * Be a single variable that can be passed from parent LoggingContexts to
+      their children.
+    """
+
+    request_id = attr.ib(type=str)
+    ip_address = attr.ib(type=str)
+    site_tag = attr.ib(type=str)
+    requester = attr.ib(type=Optional[str])
+    authenticated_entity = attr.ib(type=Optional[str])
+    method = attr.ib(type=str)
+    url = attr.ib(type=str)
+    protocol = attr.ib(type=str)
+    user_agent = attr.ib(type=str)
+
+
 LoggingContextOrSentinel = Union["LoggingContext", "_Sentinel"]
 
 
@@ -235,7 +258,8 @@ class LoggingContext:
           child to the parent
 
     Args:
-        name (str): Name for the context for debugging.
+        name: Name for the context for logging. If this is omitted, it is
+           inherited from the parent context.
         parent_context (LoggingContext|None): The parent of the new context
     """
 
@@ -256,22 +280,21 @@ def __init__(
         self,
         name: Optional[str] = None,
         parent_context: "Optional[LoggingContext]" = None,
-        request: Optional[str] = None,
+        request: Optional[ContextRequest] = None,
     ) -> None:
         self.previous_context = current_context()
-        self.name = name
 
         # track the resources used by this context so far
         self._resource_usage = ContextResourceUsage()
 
         # The thread resource usage when the logcontext became active. None
         # if the context is not currently active.
-        self.usage_start = None  # type: Optional[resource._RUsage]
+        self.usage_start: Optional[resource._RUsage] = None
 
         self.main_thread = get_thread_id()
         self.request = None
         self.tag = ""
-        self.scope = None  # type: Optional[_LogContextScope]
+        self.scope: Optional["_LogContextScope"] = None
 
         # keep track of whether we have hit the __exit__ block for this context
         # (suggesting that the the thing that created the context thinks it should
@@ -281,16 +304,27 @@ def __init__(
         self.parent_context = parent_context
 
         if self.parent_context is not None:
-            self.parent_context.copy_to(self)
+            # we track the current request_id
+            self.request = self.parent_context.request
+
+            # we also track the current scope:
+            self.scope = self.parent_context.scope
 
         if request is not None:
             # the request param overrides the request from the parent context
             self.request = request
 
+        # if we don't have a `name`, but do have a parent context, use its name.
+        if self.parent_context and name is None:
+            name = str(self.parent_context)
+        if name is None:
+            raise ValueError(
+                "LoggingContext must be given either a name or a parent context"
+            )
+        self.name = name
+
     def __str__(self) -> str:
-        if self.request:
-            return str(self.request)
-        return "%s@%x" % (self.name, id(self))
+        return self.name
 
     @classmethod
     def current_context(cls) -> LoggingContextOrSentinel:
@@ -556,8 +590,23 @@ def filter(self, record: logging.LogRecord) -> Literal[True]:
         # we end up in a death spiral of infinite loops, so let's check, for
         # robustness' sake.
         if context is not None:
-            # Logging is interested in the request.
-            record.request = context.request  # type: ignore
+            # Logging is interested in the request ID. Note that for backwards
+            # compatibility this is stored as the "request" on the record.
+            record.request = str(context)  # type: ignore
+
+            # Add some data from the HTTP request.
+            request = context.request
+            if request is None:
+                return True
+
+            record.ip_address = request.ip_address  # type: ignore
+            record.site_tag = request.site_tag  # type: ignore
+            record.requester = request.requester  # type: ignore
+            record.authenticated_entity = request.authenticated_entity  # type: ignore
+            record.method = request.method  # type: ignore
+            record.url = request.url  # type: ignore
+            record.protocol = request.protocol  # type: ignore
+            record.user_agent = request.user_agent  # type: ignore
 
         return True
 
@@ -630,8 +679,8 @@ def set_current_context(context: LoggingContextOrSentinel) -> LoggingContextOrSe
 def nested_logging_context(suffix: str) -> LoggingContext:
     """Creates a new logging context as a child of another.
 
-    The nested logging context will have a 'request' made up of the parent context's
-    request, plus the given suffix.
+    The nested logging context will have a 'name' made up of the parent context's
+    name, plus the given suffix.
 
     CPU/db usage stats will be added to the parent context's on exit.
 
@@ -641,7 +690,7 @@ def nested_logging_context(suffix: str) -> LoggingContext:
             # ... do stuff
 
     Args:
-        suffix: suffix to add to the parent context's 'request'.
+        suffix: suffix to add to the parent context's 'name'.
 
     Returns:
         LoggingContext: new logging context.
@@ -652,12 +701,14 @@ def nested_logging_context(suffix: str) -> LoggingContext:
             "Starting nested logging context from sentinel context: metrics will be lost"
         )
         parent_context = None
-        prefix = ""
     else:
         assert isinstance(curr_context, LoggingContext)
         parent_context = curr_context
-        prefix = str(parent_context.request)
-    return LoggingContext(parent_context=parent_context, request=prefix + "-" + suffix)
+    prefix = str(curr_context)
+    return LoggingContext(
+        prefix + "-" + suffix,
+        parent_context=parent_context,
+    )
 
 
 def preserve_fn(f):
@@ -694,7 +745,7 @@ def run_in_background(f, *args, **kwargs) -> defer.Deferred:
         # by synchronous exceptions, so let's turn them into Failures.
         return defer.fail()
 
-    if isinstance(res, types.CoroutineType):
+    if isinstance(res, typing.Coroutine):
         res = defer.ensureDeferred(res)
 
     # At this point we should have a Deferred, if not then f was a synchronous
@@ -847,7 +898,7 @@ def defer_to_threadpool(reactor, threadpool, f, *args, **kwargs):
         parent_context = curr_context
 
     def g():
-        with LoggingContext(parent_context=parent_context):
+        with LoggingContext(str(curr_context), parent_context=parent_context):
             return f(*args, **kwargs)
 
     return make_deferred_yieldable(threads.deferToThreadPool(reactor, threadpool, g))
diff --git a/synapse/logging/filter.py b/synapse/logging/filter.py
index 1baf8dd679..ed51a4726c 100644
--- a/synapse/logging/filter.py
+++ b/synapse/logging/filter.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/logging/formatter.py b/synapse/logging/formatter.py
index 11f60a77f7..c0f12ecd15 100644
--- a/synapse/logging/formatter.py
+++ b/synapse/logging/formatter.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/logging/handlers.py b/synapse/logging/handlers.py
new file mode 100644
index 0000000000..af5fc407a8
--- /dev/null
+++ b/synapse/logging/handlers.py
@@ -0,0 +1,89 @@
+import logging
+import time
+from logging import Handler, LogRecord
+from logging.handlers import MemoryHandler
+from threading import Thread
+from typing import Optional
+
+from twisted.internet.interfaces import IReactorCore
+
+
+class PeriodicallyFlushingMemoryHandler(MemoryHandler):
+    """
+    This is a subclass of MemoryHandler that additionally spawns a background
+    thread to periodically flush the buffer.
+
+    This prevents messages from being buffered for too long.
+
+    Additionally, all messages will be immediately flushed if the reactor has
+    not yet been started.
+    """
+
+    def __init__(
+        self,
+        capacity: int,
+        flushLevel: int = logging.ERROR,
+        target: Optional[Handler] = None,
+        flushOnClose: bool = True,
+        period: float = 5.0,
+        reactor: Optional[IReactorCore] = None,
+    ) -> None:
+        """
+        period: the period between automatic flushes
+
+        reactor: if specified, a custom reactor to use. If not specifies,
+            defaults to the globally-installed reactor.
+            Log entries will be flushed immediately until this reactor has
+            started.
+        """
+        super().__init__(capacity, flushLevel, target, flushOnClose)
+
+        self._flush_period: float = period
+        self._active: bool = True
+        self._reactor_started = False
+
+        self._flushing_thread: Thread = Thread(
+            name="PeriodicallyFlushingMemoryHandler flushing thread",
+            target=self._flush_periodically,
+            daemon=True,
+        )
+        self._flushing_thread.start()
+
+        def on_reactor_running():
+            self._reactor_started = True
+
+        reactor_to_use: IReactorCore
+        if reactor is None:
+            from twisted.internet import reactor as global_reactor
+
+            reactor_to_use = global_reactor  # type: ignore[assignment]
+        else:
+            reactor_to_use = reactor
+
+        # call our hook when the reactor start up
+        reactor_to_use.callWhenRunning(on_reactor_running)
+
+    def shouldFlush(self, record: LogRecord) -> bool:
+        """
+        Before reactor start-up, log everything immediately.
+        Otherwise, fall back to original behaviour of waiting for the buffer to fill.
+        """
+
+        if self._reactor_started:
+            return super().shouldFlush(record)
+        else:
+            return True
+
+    def _flush_periodically(self):
+        """
+        Whilst this handler is active, flush the handler periodically.
+        """
+
+        while self._active:
+            # flush is thread-safe; it acquires and releases the lock internally
+            self.flush()
+            time.sleep(self._flush_period)
+
+    def close(self) -> None:
+        self._active = False
+        super().close()
diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py
index aa146e8bb8..ecd51f1b4a 100644
--- a/synapse/logging/opentracing.py
+++ b/synapse/logging/opentracing.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -169,11 +168,12 @@ def set_fates(clotho, lachesis, atropos, father="Zues", mother="Themis"):
 import logging
 import re
 from functools import wraps
-from typing import TYPE_CHECKING, Dict, Optional, Pattern, Type
+from typing import TYPE_CHECKING, Collection, Dict, List, Optional, Pattern, Type
 
 import attr
 
 from twisted.internet import defer
+from twisted.web.http_headers import Headers
 
 from synapse.config import ConfigError
 from synapse.util import json_decoder, json_encoder
@@ -251,7 +251,7 @@ def report_span(self, span):
             except Exception:
                 logger.exception("Failed to report span")
 
-    RustReporter = _WrappedRustReporter  # type: Optional[Type[_WrappedRustReporter]]
+    RustReporter: Optional[Type[_WrappedRustReporter]] = _WrappedRustReporter
 except ImportError:
     RustReporter = None
 
@@ -259,13 +259,39 @@ def report_span(self, span):
 logger = logging.getLogger(__name__)
 
 
+class SynapseTags:
+    # The message ID of any to_device message processed
+    TO_DEVICE_MESSAGE_ID = "to_device.message_id"
+
+    # Whether the sync response has new data to be returned to the client.
+    SYNC_RESULT = "sync.new_data"
+
+    # incoming HTTP request ID  (as written in the logs)
+    REQUEST_ID = "request_id"
+
+    # HTTP request tag (used to distinguish full vs incremental syncs, etc)
+    REQUEST_TAG = "request_tag"
+
+    # Text description of a database transaction
+    DB_TXN_DESC = "db.txn_desc"
+
+    # Uniqueish ID of a database transaction
+    DB_TXN_ID = "db.txn_id"
+
+
+class SynapseBaggage:
+    FORCE_TRACING = "synapse-force-tracing"
+
+
 # Block everything by default
 # A regex which matches the server_names to expose traces for.
 # None means 'block everything'.
-_homeserver_whitelist = None  # type: Optional[Pattern[str]]
+_homeserver_whitelist: Optional[Pattern[str]] = None
 
 # Util methods
 
+Sentinel = object()
+
 
 def only_if_tracing(func):
     """Executes the function only if we're tracing. Otherwise returns None."""
@@ -318,6 +344,7 @@ def ensure_active_span_inner_2(*args, **kwargs):
 @contextlib.contextmanager
 def noop_context_manager(*args, **kwargs):
     """Does exactly what it says on the tin"""
+    # TODO: replace with contextlib.nullcontext once we drop support for Python 3.6
     yield
 
 
@@ -343,10 +370,13 @@ def init_tracer(hs: "HomeServer"):
 
     set_homeserver_whitelist(hs.config.opentracer_whitelist)
 
+    from jaeger_client.metrics.prometheus import PrometheusMetricsFactory
+
     config = JaegerConfig(
         config=hs.config.jaeger_config,
-        service_name="{} {}".format(hs.config.server_name, hs.get_instance_name()),
+        service_name=f"{hs.config.server_name} {hs.get_instance_name()}",
         scope_manager=LogContextScopeManager(hs.config),
+        metrics_factory=PrometheusMetricsFactory(),
     )
 
     # If we have the rust jaeger reporter available let's use that.
@@ -424,12 +454,28 @@ def start_active_span(
     )
 
 
-def start_active_span_follows_from(operation_name, contexts):
+def start_active_span_follows_from(
+    operation_name: str, contexts: Collection, inherit_force_tracing=False
+):
+    """Starts an active opentracing span, with additional references to previous spans
+
+    Args:
+        operation_name: name of the operation represented by the new span
+        contexts: the previous spans to inherit from
+        inherit_force_tracing: if set, and any of the previous contexts have had tracing
+           forced, the new span will also have tracing forced.
+    """
     if opentracing is None:
         return noop_context_manager()
 
     references = [opentracing.follows_from(context) for context in contexts]
     scope = start_active_span(operation_name, references=references)
+
+    if inherit_force_tracing and any(
+        is_context_forced_tracing(ctx) for ctx in contexts
+    ):
+        force_tracing(scope.span)
+
     return scope
 
 
@@ -478,7 +524,7 @@ def start_active_span_from_request(
 def start_active_span_from_edu(
     edu_content,
     operation_name,
-    references=[],
+    references: Optional[list] = None,
     tags=None,
     start_time=None,
     ignore_active_span=False,
@@ -493,6 +539,7 @@ def start_active_span_from_edu(
 
         For the other args see opentracing.tracer
     """
+    references = references or []
 
     if opentracing is None:
         return noop_context_manager()
@@ -527,6 +574,10 @@ def start_active_span_from_edu(
 
 
 # Opentracing setters for tags, logs, etc
+@only_if_tracing
+def active_span():
+    """Get the currently active span, if any"""
+    return opentracing.tracer.active_span
 
 
 @ensure_active_span("set a tag")
@@ -547,62 +598,52 @@ def set_operation_name(operation_name):
     opentracing.tracer.active_span.set_operation_name(operation_name)
 
 
-# Injection and extraction
-
+@only_if_tracing
+def force_tracing(span=Sentinel) -> None:
+    """Force sampling for the active/given span and its children.
 
-@ensure_active_span("inject the span into a header")
-def inject_active_span_twisted_headers(headers, destination, check_destination=True):
+    Args:
+        span: span to force tracing for. By default, the active span.
     """
-    Injects a span context into twisted headers in-place
+    if span is Sentinel:
+        span = opentracing.tracer.active_span
+    if span is None:
+        logger.error("No active span in force_tracing")
+        return
 
-    Args:
-        headers (twisted.web.http_headers.Headers)
-        destination (str): address of entity receiving the span context. If check_destination
-            is true the context will only be injected if the destination matches the
-            opentracing whitelist
-        check_destination (bool): If false, destination will be ignored and the context
-            will always be injected.
-        span (opentracing.Span)
+    span.set_tag(opentracing.tags.SAMPLING_PRIORITY, 1)
 
-    Returns:
-        In-place modification of headers
+    # also set a bit of baggage, so that we have a way of figuring out if
+    # it is enabled later
+    span.set_baggage_item(SynapseBaggage.FORCE_TRACING, "1")
 
-    Note:
-        The headers set by the tracer are custom to the tracer implementation which
-        should be unique enough that they don't interfere with any headers set by
-        synapse or twisted. If we're still using jaeger these headers would be those
-        here:
-        https://github.com/jaegertracing/jaeger-client-python/blob/master/jaeger_client/constants.py
-    """
 
-    if check_destination and not whitelisted_homeserver(destination):
-        return
+def is_context_forced_tracing(span_context) -> bool:
+    """Check if sampling has been force for the given span context."""
+    if span_context is None:
+        return False
+    return span_context.baggage.get(SynapseBaggage.FORCE_TRACING) is not None
 
-    span = opentracing.tracer.active_span
-    carrier = {}  # type: Dict[str, str]
-    opentracing.tracer.inject(span, opentracing.Format.HTTP_HEADERS, carrier)
 
-    for key, value in carrier.items():
-        headers.addRawHeaders(key, value)
+# Injection and extraction
 
 
-@ensure_active_span("inject the span into a byte dict")
-def inject_active_span_byte_dict(headers, destination, check_destination=True):
+@ensure_active_span("inject the span into a header dict")
+def inject_header_dict(
+    headers: Dict[bytes, List[bytes]],
+    destination: Optional[str] = None,
+    check_destination: bool = True,
+) -> None:
     """
-    Injects a span context into a dict where the headers are encoded as byte
-    strings
+    Injects a span context into a dict of HTTP headers
 
     Args:
-        headers (dict)
-        destination (str): address of entity receiving the span context. If check_destination
-            is true the context will only be injected if the destination matches the
-            opentracing whitelist
+        headers: the dict to inject headers into
+        destination: address of entity receiving the span context. Must be given unless
+            check_destination is False. The context will only be injected if the
+            destination matches the opentracing whitelist
         check_destination (bool): If false, destination will be ignored and the context
             will always be injected.
-        span (opentracing.Span)
-
-    Returns:
-        In-place modification of headers
 
     Note:
         The headers set by the tracer are custom to the tracer implementation which
@@ -611,48 +652,40 @@ def inject_active_span_byte_dict(headers, destination, check_destination=True):
         here:
         https://github.com/jaegertracing/jaeger-client-python/blob/master/jaeger_client/constants.py
     """
-    if check_destination and not whitelisted_homeserver(destination):
-        return
+    if check_destination:
+        if destination is None:
+            raise ValueError(
+                "destination must be given unless check_destination is False"
+            )
+        if not whitelisted_homeserver(destination):
+            return
 
     span = opentracing.tracer.active_span
 
-    carrier = {}  # type: Dict[str, str]
-    opentracing.tracer.inject(span, opentracing.Format.HTTP_HEADERS, carrier)
+    carrier: Dict[str, str] = {}
+    opentracing.tracer.inject(span.context, opentracing.Format.HTTP_HEADERS, carrier)
 
     for key, value in carrier.items():
         headers[key.encode()] = [value.encode()]
 
 
-@ensure_active_span("inject the span into a text map")
-def inject_active_span_text_map(carrier, destination, check_destination=True):
-    """
-    Injects a span context into a dict
-
-    Args:
-        carrier (dict)
-        destination (str): address of entity receiving the span context. If check_destination
-            is true the context will only be injected if the destination matches the
-            opentracing whitelist
-        check_destination (bool): If false, destination will be ignored and the context
-            will always be injected.
-
-    Returns:
-        In-place modification of carrier
-
-    Note:
-        The headers set by the tracer are custom to the tracer implementation which
-        should be unique enough that they don't interfere with any headers set by
-        synapse or twisted. If we're still using jaeger these headers would be those
-        here:
-        https://github.com/jaegertracing/jaeger-client-python/blob/master/jaeger_client/constants.py
-    """
-
-    if check_destination and not whitelisted_homeserver(destination):
+def inject_response_headers(response_headers: Headers) -> None:
+    """Inject the current trace id into the HTTP response headers"""
+    if not opentracing:
+        return
+    span = opentracing.tracer.active_span
+    if not span:
         return
 
-    opentracing.tracer.inject(
-        opentracing.tracer.active_span, opentracing.Format.TEXT_MAP, carrier
-    )
+    # This is a bit implementation-specific.
+    #
+    # Jaeger's Spans have a trace_id property; other implementations (including the
+    # dummy opentracing.span.Span which we use if init_tracer is not called) do not
+    # expose it
+    trace_id = getattr(span, "trace_id", None)
+
+    if trace_id is not None:
+        response_headers.addRawHeader("Synapse-Trace-Id", f"{trace_id:x}")
 
 
 @ensure_active_span("get the active span context as a dict", ret={})
@@ -671,9 +704,9 @@ def get_active_span_text_map(destination=None):
     if destination and not whitelisted_homeserver(destination):
         return {}
 
-    carrier = {}  # type: Dict[str, str]
+    carrier: Dict[str, str] = {}
     opentracing.tracer.inject(
-        opentracing.tracer.active_span, opentracing.Format.TEXT_MAP, carrier
+        opentracing.tracer.active_span.context, opentracing.Format.TEXT_MAP, carrier
     )
 
     return carrier
@@ -685,10 +718,10 @@ def active_span_context_as_string():
     Returns:
         The active span context encoded as a string.
     """
-    carrier = {}  # type: Dict[str, str]
+    carrier: Dict[str, str] = {}
     if opentracing:
         opentracing.tracer.inject(
-            opentracing.tracer.active_span, opentracing.Format.TEXT_MAP, carrier
+            opentracing.tracer.active_span.context, opentracing.Format.TEXT_MAP, carrier
         )
     return json_encoder.encode(carrier)
 
@@ -816,7 +849,7 @@ def trace_servlet(request: "SynapseRequest", extract_context: bool = False):
         return
 
     request_tags = {
-        "request_id": request.get_request_id(),
+        SynapseTags.REQUEST_ID: request.get_request_id(),
         tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER,
         tags.HTTP_METHOD: request.get_method(),
         tags.HTTP_URL: request.get_redacted_uri(),
@@ -825,11 +858,12 @@ def trace_servlet(request: "SynapseRequest", extract_context: bool = False):
 
     request_name = request.request_metrics.name
     if extract_context:
-        scope = start_active_span_from_request(request, request_name, tags=request_tags)
+        scope = start_active_span_from_request(request, request_name)
     else:
-        scope = start_active_span(request_name, tags=request_tags)
+        scope = start_active_span(request_name)
 
     with scope:
+        inject_response_headers(request.responseHeaders)
         try:
             yield
         finally:
@@ -837,4 +871,11 @@ def trace_servlet(request: "SynapseRequest", extract_context: bool = False):
             # with JsonResource).
             scope.span.set_operation_name(request.request_metrics.name)
 
-            scope.span.set_tag("request_tag", request.request_metrics.start_context.tag)
+            # set the tags *after* the servlet completes, in case it decided to
+            # prioritise the span (tags will get dropped on unprioritised spans)
+            request_tags[
+                SynapseTags.REQUEST_TAG
+            ] = request.request_metrics.start_context.tag
+
+            for k, v in request_tags.items():
+                scope.span.set_tag(k, v)
diff --git a/synapse/logging/scopecontextmanager.py b/synapse/logging/scopecontextmanager.py
index 7b9c657456..b1e8e08fe9 100644
--- a/synapse/logging/scopecontextmanager.py
+++ b/synapse/logging/scopecontextmanager.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/logging/utils.py b/synapse/logging/utils.py
index fd3543ab04..08895e72ee 100644
--- a/synapse/logging/utils.py
+++ b/synapse/logging/utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 3b499efc07..f237b8a236 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -47,7 +46,7 @@
 METRICS_PREFIX = "/_synapse/metrics"
 
 running_on_pypy = platform.python_implementation() == "PyPy"
-all_gauges = {}  # type: Dict[str, Union[LaterGauge, InFlightGauge]]
+all_gauges: "Dict[str, Union[LaterGauge, InFlightGauge]]" = {}
 
 HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat")
 
@@ -131,7 +130,7 @@ def __init__(self, name, desc, labels, sub_metrics):
         )
 
         # Counts number of in flight blocks for a given set of label values
-        self._registrations = {}  # type: Dict
+        self._registrations: Dict = {}
 
         # Protects access to _registrations
         self._lock = threading.Lock()
@@ -214,7 +213,12 @@ class GaugeBucketCollector:
     Prometheus, and optimise for that case.
     """
 
-    __slots__ = ("_name", "_documentation", "_bucket_bounds", "_metric")
+    __slots__ = (
+        "_name",
+        "_documentation",
+        "_bucket_bounds",
+        "_metric",
+    )
 
     def __init__(
         self,
@@ -242,11 +246,16 @@ def __init__(
         if self._bucket_bounds[-1] != float("inf"):
             self._bucket_bounds.append(float("inf"))
 
-        self._metric = self._values_to_metric([])
+        # We initially set this to None. We won't report metrics until
+        # this has been initialised after a successful data update
+        self._metric: Optional[GaugeHistogramMetricFamily] = None
+
         registry.register(self)
 
     def collect(self):
-        yield self._metric
+        # Don't report metrics unless we've already collected some data
+        if self._metric is not None:
+            yield self._metric
 
     def update_data(self, values: Iterable[float]):
         """Update the data to be reported by the metric
@@ -526,6 +535,13 @@ def collect(self):
 
 REGISTRY.register(ReactorLastSeenMetric())
 
+# The minimum time in seconds between GCs for each generation, regardless of the current GC
+# thresholds and counts.
+MIN_TIME_BETWEEN_GCS = (1.0, 10.0, 30.0)
+
+# The time (in seconds since the epoch) of the last time we did a GC for each generation.
+_last_gc = [0.0, 0.0, 0.0]
+
 
 def runUntilCurrentTimer(reactor, func):
     @functools.wraps(func)
@@ -566,11 +582,16 @@ def f(*args, **kwargs):
             return ret
 
         # Check if we need to do a manual GC (since its been disabled), and do
-        # one if necessary.
+        # one if necessary. Note we go in reverse order as e.g. a gen 1 GC may
+        # promote an object into gen 2, and we don't want to handle the same
+        # object multiple times.
         threshold = gc.get_threshold()
         counts = gc.get_count()
         for i in (2, 1, 0):
-            if threshold[i] < counts[i]:
+            # We check if we need to do one based on a straightforward
+            # comparison between the threshold and count. We also do an extra
+            # check to make sure that we don't a GC too often.
+            if threshold[i] < counts[i] and MIN_TIME_BETWEEN_GCS[i] < end - _last_gc[i]:
                 if i == 0:
                     logger.debug("Collecting gc %d", i)
                 else:
@@ -580,6 +601,8 @@ def f(*args, **kwargs):
                 unreachable = gc.collect(i)
                 end = time.time()
 
+                _last_gc[i] = end
+
                 gc_time.labels(i).observe(end - start)
                 gc_unreachable.labels(i).set(unreachable)
 
@@ -606,6 +629,7 @@ def f(*args, **kwargs):
 except AttributeError:
     pass
 
+
 __all__ = [
     "MetricsResource",
     "generate_latest",
diff --git a/synapse/metrics/_exposition.py b/synapse/metrics/_exposition.py
index 71320a1402..bb9bcb5592 100644
--- a/synapse/metrics/_exposition.py
+++ b/synapse/metrics/_exposition.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015-2019 Prometheus Python Client Developers
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
@@ -35,7 +34,7 @@
 
 from synapse.util import caches
 
-CONTENT_TYPE_LATEST = str("text/plain; version=0.0.4; charset=utf-8")
+CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8"
 
 
 INF = float("inf")
@@ -56,8 +55,8 @@ def floatToGoString(d):
         # Go switches to exponents sooner than Python.
         # We only need to care about positive values for le/quantile.
         if d > 0 and dot > 6:
-            mantissa = "{0}.{1}{2}".format(s[0], s[1:dot], s[dot + 1 :]).rstrip("0.")
-            return "{0}e+0{1}".format(mantissa, dot - 1)
+            mantissa = f"{s[0]}.{s[1:dot]}{s[dot + 1 :]}".rstrip("0.")
+            return f"{mantissa}e+0{dot - 1}"
         return s
 
 
@@ -66,7 +65,7 @@ def sample_line(line, name):
         labelstr = "{{{0}}}".format(
             ",".join(
                 [
-                    '{0}="{1}"'.format(
+                    '{}="{}"'.format(
                         k,
                         v.replace("\\", r"\\").replace("\n", r"\n").replace('"', r"\""),
                     )
@@ -79,10 +78,8 @@ def sample_line(line, name):
     timestamp = ""
     if line.timestamp is not None:
         # Convert to milliseconds.
-        timestamp = " {0:d}".format(int(float(line.timestamp) * 1000))
-    return "{0}{1} {2}{3}\n".format(
-        name, labelstr, floatToGoString(line.value), timestamp
-    )
+        timestamp = f" {int(float(line.timestamp) * 1000):d}"
+    return "{}{} {}{}\n".format(name, labelstr, floatToGoString(line.value), timestamp)
 
 
 def generate_latest(registry, emit_help=False):
@@ -119,14 +116,14 @@ def generate_latest(registry, emit_help=False):
         # Output in the old format for compatibility.
         if emit_help:
             output.append(
-                "# HELP {0} {1}\n".format(
+                "# HELP {} {}\n".format(
                     mname,
                     metric.documentation.replace("\\", r"\\").replace("\n", r"\n"),
                 )
             )
-        output.append("# TYPE {0} {1}\n".format(mname, mtype))
+        output.append(f"# TYPE {mname} {mtype}\n")
 
-        om_samples = {}  # type: Dict[str, List[str]]
+        om_samples: Dict[str, List[str]] = {}
         for s in metric.samples:
             for suffix in ["_created", "_gsum", "_gcount"]:
                 if s.name == metric.name + suffix:
@@ -144,13 +141,13 @@ def generate_latest(registry, emit_help=False):
         for suffix, lines in sorted(om_samples.items()):
             if emit_help:
                 output.append(
-                    "# HELP {0}{1} {2}\n".format(
+                    "# HELP {}{} {}\n".format(
                         metric.name,
                         suffix,
                         metric.documentation.replace("\\", r"\\").replace("\n", r"\n"),
                     )
                 )
-            output.append("# TYPE {0}{1} gauge\n".format(metric.name, suffix))
+            output.append(f"# TYPE {metric.name}{suffix} gauge\n")
             output.extend(lines)
 
         # Get rid of the weird colon things while we're at it
@@ -164,12 +161,12 @@ def generate_latest(registry, emit_help=False):
         # Also output in the new format, if it's different.
         if emit_help:
             output.append(
-                "# HELP {0} {1}\n".format(
+                "# HELP {} {}\n".format(
                     mnewname,
                     metric.documentation.replace("\\", r"\\").replace("\n", r"\n"),
                 )
             )
-        output.append("# TYPE {0} {1}\n".format(mnewname, mtype))
+        output.append(f"# TYPE {mnewname} {mtype}\n")
 
         for s in metric.samples:
             # Get rid of the OpenMetrics specific samples (we should already have
diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py
index b56986d8e7..3a14260752 100644
--- a/synapse/metrics/background_process_metrics.py
+++ b/synapse/metrics/background_process_metrics.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,14 +15,18 @@
 import logging
 import threading
 from functools import wraps
-from typing import TYPE_CHECKING, Dict, Optional, Set
+from typing import TYPE_CHECKING, Dict, Optional, Set, Union
 
 from prometheus_client.core import REGISTRY, Counter, Gauge
 
 from twisted.internet import defer
 
 from synapse.logging.context import LoggingContext, PreserveLoggingContext
-from synapse.logging.opentracing import noop_context_manager, start_active_span
+from synapse.logging.opentracing import (
+    SynapseTags,
+    noop_context_manager,
+    start_active_span,
+)
 from synapse.util.async_helpers import maybe_awaitable
 
 if TYPE_CHECKING:
@@ -90,7 +93,7 @@
 # map from description to a counter, so that we can name our logcontexts
 # incrementally. (It actually duplicates _background_process_start_count, but
 # it's much simpler to do so than to try to combine them.)
-_background_process_counts = {}  # type: Dict[str, int]
+_background_process_counts: Dict[str, int] = {}
 
 # Set of all running background processes that became active active since the
 # last time metrics were scraped (i.e. background processes that performed some
@@ -100,7 +103,7 @@
 # background processes stacking up behind a lock or linearizer, where we then
 # only need to iterate over and update metrics for the process that have
 # actually been active and can ignore the idle ones.
-_background_processes_active_since_last_scrape = set()  # type: Set[_BackgroundProcess]
+_background_processes_active_since_last_scrape: "Set[_BackgroundProcess]" = set()
 
 # A lock that covers the above set and dict
 _bg_metrics_lock = threading.Lock()
@@ -134,8 +137,7 @@ def collect(self):
             _background_process_db_txn_duration,
             _background_process_db_sched_duration,
         ):
-            for r in m.collect():
-                yield r
+            yield from m.collect()
 
 
 REGISTRY.register(_Collector())
@@ -199,11 +201,14 @@ async def run():
         _background_process_start_count.labels(desc).inc()
         _background_process_in_flight_count.labels(desc).inc()
 
-        with BackgroundProcessLoggingContext(desc, "%s-%i" % (desc, count)) as context:
+        with BackgroundProcessLoggingContext(desc, count) as context:
             try:
-                ctx = noop_context_manager()
                 if bg_start_span:
-                    ctx = start_active_span(desc, tags={"request_id": context.request})
+                    ctx = start_active_span(
+                        f"bgproc.{desc}", tags={SynapseTags.REQUEST_ID: str(context)}
+                    )
+                else:
+                    ctx = noop_context_manager()
                 with ctx:
                     return await maybe_awaitable(func(*args, **kwargs))
             except Exception:
@@ -244,9 +249,20 @@ class BackgroundProcessLoggingContext(LoggingContext):
 
     __slots__ = ["_proc"]
 
-    def __init__(self, name: str, request: Optional[str] = None):
-        super().__init__(name, request=request)
+    def __init__(self, name: str, instance_id: Optional[Union[int, str]] = None):
+        """
+
+        Args:
+            name: The name of the background process. Each distinct `name` gets a
+                separate prometheus time series.
 
+            instance_id: an identifer to add to `name` to distinguish this instance of
+                the named background process in the logs. If this is `None`, one is
+                made up based on id(self).
+        """
+        if instance_id is None:
+            instance_id = id(self)
+        super().__init__("%s-%s" % (name, instance_id))
         self._proc = _BackgroundProcess(name, self)
 
     def start(self, rusage: "Optional[resource._RUsage]"):
diff --git a/synapse/metrics/jemalloc.py b/synapse/metrics/jemalloc.py
new file mode 100644
index 0000000000..29ab6c0229
--- /dev/null
+++ b/synapse/metrics/jemalloc.py
@@ -0,0 +1,196 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ctypes
+import logging
+import os
+import re
+from typing import Optional
+
+from synapse.metrics import REGISTRY, GaugeMetricFamily
+
+logger = logging.getLogger(__name__)
+
+
+def _setup_jemalloc_stats():
+    """Checks to see if jemalloc is loaded, and hooks up a collector to record
+    statistics exposed by jemalloc.
+    """
+
+    # Try to find the loaded jemalloc shared library, if any. We need to
+    # introspect into what is loaded, rather than loading whatever is on the
+    # path, as if we load a *different* jemalloc version things will seg fault.
+
+    # We look in `/proc/self/maps`, which only exists on linux.
+    if not os.path.exists("/proc/self/maps"):
+        logger.debug("Not looking for jemalloc as no /proc/self/maps exist")
+        return
+
+    # We're looking for a path at the end of the line that includes
+    # "libjemalloc".
+    regex = re.compile(r"/\S+/libjemalloc.*$")
+
+    jemalloc_path = None
+    with open("/proc/self/maps") as f:
+        for line in f:
+            match = regex.search(line.strip())
+            if match:
+                jemalloc_path = match.group()
+
+    if not jemalloc_path:
+        # No loaded jemalloc was found.
+        logger.debug("jemalloc not found")
+        return
+
+    logger.debug("Found jemalloc at %s", jemalloc_path)
+
+    jemalloc = ctypes.CDLL(jemalloc_path)
+
+    def _mallctl(
+        name: str, read: bool = True, write: Optional[int] = None
+    ) -> Optional[int]:
+        """Wrapper around `mallctl` for reading and writing integers to
+        jemalloc.
+
+        Args:
+            name: The name of the option to read from/write to.
+            read: Whether to try and read the value.
+            write: The value to write, if given.
+
+        Returns:
+            The value read if `read` is True, otherwise None.
+
+        Raises:
+            An exception if `mallctl` returns a non-zero error code.
+        """
+
+        input_var = None
+        input_var_ref = None
+        input_len_ref = None
+        if read:
+            input_var = ctypes.c_size_t(0)
+            input_len = ctypes.c_size_t(ctypes.sizeof(input_var))
+
+            input_var_ref = ctypes.byref(input_var)
+            input_len_ref = ctypes.byref(input_len)
+
+        write_var_ref = None
+        write_len = ctypes.c_size_t(0)
+        if write is not None:
+            write_var = ctypes.c_size_t(write)
+            write_len = ctypes.c_size_t(ctypes.sizeof(write_var))
+
+            write_var_ref = ctypes.byref(write_var)
+
+        # The interface is:
+        #
+        #   int mallctl(
+        #       const char *name,
+        #       void *oldp,
+        #       size_t *oldlenp,
+        #       void *newp,
+        #       size_t newlen
+        #   )
+        #
+        # Where oldp/oldlenp is a buffer where the old value will be written to
+        # (if not null), and newp/newlen is the buffer with the new value to set
+        # (if not null). Note that they're all references *except* newlen.
+        result = jemalloc.mallctl(
+            name.encode("ascii"),
+            input_var_ref,
+            input_len_ref,
+            write_var_ref,
+            write_len,
+        )
+
+        if result != 0:
+            raise Exception("Failed to call mallctl")
+
+        if input_var is None:
+            return None
+
+        return input_var.value
+
+    def _jemalloc_refresh_stats() -> None:
+        """Request that jemalloc updates its internal statistics. This needs to
+        be called before querying for stats, otherwise it will return stale
+        values.
+        """
+        try:
+            _mallctl("epoch", read=False, write=1)
+        except Exception as e:
+            logger.warning("Failed to reload jemalloc stats: %s", e)
+
+    class JemallocCollector:
+        """Metrics for internal jemalloc stats."""
+
+        def collect(self):
+            _jemalloc_refresh_stats()
+
+            g = GaugeMetricFamily(
+                "jemalloc_stats_app_memory_bytes",
+                "The stats reported by jemalloc",
+                labels=["type"],
+            )
+
+            # Read the relevant global stats from jemalloc. Note that these may
+            # not be accurate if python is configured to use its internal small
+            # object allocator (which is on by default, disable by setting the
+            # env `PYTHONMALLOC=malloc`).
+            #
+            # See the jemalloc manpage for details about what each value means,
+            # roughly:
+            #   - allocated ─ Total number of bytes allocated by the app
+            #   - active ─ Total number of bytes in active pages allocated by
+            #     the application, this is bigger than `allocated`.
+            #   - resident ─ Maximum number of bytes in physically resident data
+            #     pages mapped by the allocator, comprising all pages dedicated
+            #     to allocator metadata, pages backing active allocations, and
+            #     unused dirty pages. This is bigger than `active`.
+            #   - mapped ─ Total number of bytes in active extents mapped by the
+            #     allocator.
+            #   - metadata ─ Total number of bytes dedicated to jemalloc
+            #     metadata.
+            for t in (
+                "allocated",
+                "active",
+                "resident",
+                "mapped",
+                "metadata",
+            ):
+                try:
+                    value = _mallctl(f"stats.{t}")
+                except Exception as e:
+                    # There was an error fetching the value, skip.
+                    logger.warning("Failed to read jemalloc stats.%s: %s", t, e)
+                    continue
+
+                g.add_metric([t], value=value)
+
+            yield g
+
+    REGISTRY.register(JemallocCollector())
+
+    logger.debug("Added jemalloc stats")
+
+
+def setup_jemalloc_stats():
+    """Try to setup jemalloc stats, if jemalloc is loaded."""
+
+    try:
+        _setup_jemalloc_stats()
+    except Exception as e:
+        # This should only happen if we find the loaded jemalloc library, but
+        # fail to load it somehow (e.g. we somehow picked the wrong version).
+        logger.info("Failed to setup collector to record jemalloc stats: %s", e)
diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py
index 2e38150eac..2d2ed229e2 100644
--- a/synapse/module_api/__init__.py
+++ b/synapse/module_api/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -13,17 +12,42 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import email.utils
 import logging
-from typing import TYPE_CHECKING, Any, Generator, Iterable, Optional, Tuple
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+)
+
+import jinja2
 
 from twisted.internet import defer
+from twisted.web.resource import IResource
 
 from synapse.events import EventBase
 from synapse.http.client import SimpleHttpClient
+from synapse.http.server import (
+    DirectServeHtmlResource,
+    DirectServeJsonResource,
+    respond_with_html,
+)
+from synapse.http.servlet import parse_json_object_from_request
 from synapse.http.site import SynapseRequest
 from synapse.logging.context import make_deferred_yieldable, run_in_background
+from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.storage.database import DatabasePool, LoggingTransaction
+from synapse.storage.databases.main.roommember import ProfileInfo
 from synapse.storage.state import StateFilter
-from synapse.types import JsonDict, UserID, create_requester
+from synapse.types import JsonDict, Requester, UserID, UserInfo, create_requester
+from synapse.util import Clock
+from synapse.util.caches.descriptors import cached
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -33,7 +57,20 @@
 are loaded into Synapse.
 """
 
-__all__ = ["errors", "make_deferred_yieldable", "run_in_background", "ModuleApi"]
+__all__ = [
+    "errors",
+    "make_deferred_yieldable",
+    "parse_json_object_from_request",
+    "respond_with_html",
+    "run_in_background",
+    "cached",
+    "UserID",
+    "DatabasePool",
+    "LoggingTransaction",
+    "DirectServeHtmlResource",
+    "DirectServeJsonResource",
+    "ModuleApi",
+]
 
 logger = logging.getLogger(__name__)
 
@@ -50,11 +87,67 @@ def __init__(self, hs: "HomeServer", auth_handler):
         self._auth = hs.get_auth()
         self._auth_handler = auth_handler
         self._server_name = hs.hostname
+        self._presence_stream = hs.get_event_sources().sources["presence"]
+        self._state = hs.get_state_handler()
+        self._clock: Clock = hs.get_clock()
+        self._send_email_handler = hs.get_send_email_handler()
+        self.custom_template_dir = hs.config.server.custom_template_directory
+
+        try:
+            app_name = self._hs.config.email_app_name
+
+            self._from_string = self._hs.config.email_notif_from % {"app": app_name}
+        except (KeyError, TypeError):
+            # If substitution failed (which can happen if the string contains
+            # placeholders other than just "app", or if the type of the placeholder is
+            # not a string), fall back to the bare strings.
+            self._from_string = self._hs.config.email_notif_from
+
+        self._raw_from = email.utils.parseaddr(self._from_string)[1]
 
         # We expose these as properties below in order to attach a helpful docstring.
-        self._http_client = hs.get_simple_http_client()  # type: SimpleHttpClient
+        self._http_client: SimpleHttpClient = hs.get_simple_http_client()
         self._public_room_list_manager = PublicRoomListManager(hs)
 
+        self._spam_checker = hs.get_spam_checker()
+        self._account_validity_handler = hs.get_account_validity_handler()
+        self._third_party_event_rules = hs.get_third_party_event_rules()
+
+    #################################################################################
+    # The following methods should only be called during the module's initialisation.
+
+    @property
+    def register_spam_checker_callbacks(self):
+        """Registers callbacks for spam checking capabilities."""
+        return self._spam_checker.register_callbacks
+
+    @property
+    def register_account_validity_callbacks(self):
+        """Registers callbacks for account validity capabilities."""
+        return self._account_validity_handler.register_account_validity_callbacks
+
+    @property
+    def register_third_party_rules_callbacks(self):
+        """Registers callbacks for third party event rules capabilities."""
+        return self._third_party_event_rules.register_third_party_rules_callbacks
+
+    def register_web_resource(self, path: str, resource: IResource):
+        """Registers a web resource to be served at the given path.
+
+        This function should be called during initialisation of the module.
+
+        If multiple modules register a resource for the same path, the module that
+        appears the highest in the configuration file takes priority.
+
+        Args:
+            path: The path to register the resource for.
+            resource: The resource to attach to this path.
+        """
+        self._hs.register_module_web_resource(path, resource)
+
+    #########################################################################
+    # The following methods can be called by the module at any point in time.
+
     @property
     def http_client(self):
         """Allows making outbound HTTP requests to remote resources.
@@ -72,22 +165,66 @@ def public_room_list_manager(self):
         """
         return self._public_room_list_manager
 
-    def get_user_by_req(self, req, allow_guest=False):
+    @property
+    def public_baseurl(self) -> str:
+        """The configured public base URL for this homeserver."""
+        return self._hs.config.public_baseurl
+
+    @property
+    def email_app_name(self) -> str:
+        """The application name configured in the homeserver's configuration."""
+        return self._hs.config.email.email_app_name
+
+    async def get_userinfo_by_id(self, user_id: str) -> Optional[UserInfo]:
+        """Get user info by user_id
+
+        Args:
+            user_id: Fully qualified user id.
+        Returns:
+            UserInfo object if a user was found, otherwise None
+        """
+        return await self._store.get_userinfo_by_id(user_id)
+
+    async def get_user_by_req(
+        self,
+        req: SynapseRequest,
+        allow_guest: bool = False,
+        allow_expired: bool = False,
+    ) -> Requester:
         """Check the access_token provided for a request
 
         Args:
-            req (twisted.web.server.Request): Incoming HTTP request
-            allow_guest (bool): True if guest users should be allowed. If this
+            req: Incoming HTTP request
+            allow_guest: True if guest users should be allowed. If this
                 is False, and the access token is for a guest user, an
                 AuthError will be thrown
+            allow_expired: True if expired users should be allowed. If this
+                is False, and the access token is for an expired user, an
+                AuthError will be thrown
+
         Returns:
-            twisted.internet.defer.Deferred[synapse.types.Requester]:
-                the requester for this request
+            The requester for this request
+
         Raises:
-            synapse.api.errors.AuthError: if no user by that token exists,
+            InvalidClientCredentialsError: if no user by that token exists,
                 or the token is invalid.
         """
-        return self._auth.get_user_by_req(req, allow_guest)
+        return await self._auth.get_user_by_req(
+            req,
+            allow_guest,
+            allow_expired=allow_expired,
+        )
+
+    async def is_user_admin(self, user_id: str) -> bool:
+        """Checks if a user is a server admin.
+
+        Args:
+            user_id: The Matrix ID of the user to check.
+
+        Returns:
+            True if the user is a server admin, False otherwise.
+        """
+        return await self._store.is_server_admin(UserID.from_string(user_id))
 
     def get_qualified_user_id(self, username):
         """Qualify a user id, if necessary
@@ -105,6 +242,32 @@ def get_qualified_user_id(self, username):
             return username
         return UserID(username, self._hs.hostname).to_string()
 
+    async def get_profile_for_user(self, localpart: str) -> ProfileInfo:
+        """Look up the profile info for the user with the given localpart.
+
+        Args:
+            localpart: The localpart to look up profile information for.
+
+        Returns:
+            The profile information (i.e. display name and avatar URL).
+        """
+        return await self._store.get_profileinfo(localpart)
+
+    async def get_threepids_for_user(self, user_id: str) -> List[Dict[str, str]]:
+        """Look up the threepids (email addresses and phone numbers) associated with the
+        given Matrix user ID.
+
+        Args:
+            user_id: The Matrix user ID to look up threepids for.
+
+        Returns:
+            A list of threepids, each threepid being represented by a dictionary
+            containing a "medium" key which value is "email" for email addresses and
+            "msisdn" for phone numbers, and an "address" key which value is the
+            threepid's address.
+        """
+        return await self._store.user_get_threepids(user_id)
+
     def check_user_exists(self, user_id):
         """Check if user exists.
 
@@ -118,7 +281,7 @@ def check_user_exists(self, user_id):
         return defer.ensureDeferred(self._auth_handler.check_user_exists(user_id))
 
     @defer.inlineCallbacks
-    def register(self, localpart, displayname=None, emails=[]):
+    def register(self, localpart, displayname=None, emails: Optional[List[str]] = None):
         """Registers a new user with given localpart and optional displayname, emails.
 
         Also returns an access token for the new user.
@@ -138,11 +301,13 @@ def register(self, localpart, displayname=None, emails=[]):
         logger.warning(
             "Using deprecated ModuleApi.register which creates a dummy user device."
         )
-        user_id = yield self.register_user(localpart, displayname, emails)
-        _, access_token = yield self.register_device(user_id)
+        user_id = yield self.register_user(localpart, displayname, emails or [])
+        _, access_token, _, _ = yield self.register_device(user_id)
         return user_id, access_token
 
-    def register_user(self, localpart, displayname=None, emails=[]):
+    def register_user(
+        self, localpart, displayname=None, emails: Optional[List[str]] = None
+    ):
         """Registers a new user with given localpart and optional displayname, emails.
 
         Args:
@@ -161,7 +326,7 @@ def register_user(self, localpart, displayname=None, emails=[]):
             self._hs.get_registration_handler().register_user(
                 localpart=localpart,
                 default_display_name=displayname,
-                bind_emails=emails,
+                bind_emails=emails or [],
             )
         )
 
@@ -330,7 +495,7 @@ async def complete_sso_login_async(
     @defer.inlineCallbacks
     def get_state_events_in_room(
         self, room_id: str, types: Iterable[Tuple[str, Optional[str]]]
-    ) -> Generator[defer.Deferred, Any, defer.Deferred]:
+    ) -> Generator[defer.Deferred, Any, Iterable[EventBase]]:
         """Gets current state events for the given room.
 
         (This is exposed for compatibility with the old SpamCheckerApi. We should
@@ -385,6 +550,146 @@ async def create_and_send_event_into_room(self, event_dict: JsonDict) -> EventBa
 
         return event
 
+    async def send_local_online_presence_to(self, users: Iterable[str]) -> None:
+        """
+        Forces the equivalent of a presence initial_sync for a set of local or remote
+        users. The users will receive presence for all currently online users that they
+        are considered interested in.
+
+        Updates to remote users will be sent immediately, whereas local users will receive
+        them on their next sync attempt.
+
+        Note that this method can only be run on the process that is configured to write to the
+        presence stream. By default this is the main process.
+        """
+        if self._hs._instance_name not in self._hs.config.worker.writers.presence:
+            raise Exception(
+                "send_local_online_presence_to can only be run "
+                "on the process that is configured to write to the "
+                "presence stream (by default this is the main process)",
+            )
+
+        local_users = set()
+        remote_users = set()
+        for user in users:
+            if self._hs.is_mine_id(user):
+                local_users.add(user)
+            else:
+                remote_users.add(user)
+
+        # We pull out the presence handler here to break a cyclic
+        # dependency between the presence router and module API.
+        presence_handler = self._hs.get_presence_handler()
+
+        if local_users:
+            # Force a presence initial_sync for these users next time they sync.
+            await presence_handler.send_full_presence_to_users(local_users)
+
+        for user in remote_users:
+            # Retrieve presence state for currently online users that this user
+            # is considered interested in.
+            presence_events, _ = await self._presence_stream.get_new_events(
+                UserID.from_string(user), from_key=None, include_offline=False
+            )
+
+            # Send to remote destinations.
+            destination = UserID.from_string(user).domain
+            presence_handler.get_federation_queue().send_presence_to_destinations(
+                presence_events, destination
+            )
+
+    def looping_background_call(
+        self,
+        f: Callable,
+        msec: float,
+        *args,
+        desc: Optional[str] = None,
+        run_on_all_instances: bool = False,
+        **kwargs,
+    ):
+        """Wraps a function as a background process and calls it repeatedly.
+
+        NOTE: Will only run on the instance that is configured to run
+        background processes (which is the main process by default), unless
+        `run_on_all_workers` is set.
+
+        Waits `msec` initially before calling `f` for the first time.
+
+        Args:
+            f: The function to call repeatedly. f can be either synchronous or
+                asynchronous, and must follow Synapse's logcontext rules.
+                More info about logcontexts is available at
+                https://matrix-org.github.io/synapse/latest/log_contexts.html
+            msec: How long to wait between calls in milliseconds.
+            *args: Positional arguments to pass to function.
+            desc: The background task's description. Default to the function's name.
+            run_on_all_instances: Whether to run this on all instances, rather
+                than just the instance configured to run background tasks.
+            **kwargs: Key arguments to pass to function.
+        """
+        if desc is None:
+            desc = f.__name__
+
+        if self._hs.config.run_background_tasks or run_on_all_instances:
+            self._clock.looping_call(
+                run_as_background_process,
+                msec,
+                desc,
+                f,
+                *args,
+                **kwargs,
+            )
+        else:
+            logger.warning(
+                "Not running looping call %s as the configuration forbids it",
+                f,
+            )
+
+    async def send_mail(
+        self,
+        recipient: str,
+        subject: str,
+        html: str,
+        text: str,
+    ):
+        """Send an email on behalf of the homeserver.
+
+        Args:
+            recipient: The email address for the recipient.
+            subject: The email's subject.
+            html: The email's HTML content.
+            text: The email's text content.
+        """
+        await self._send_email_handler.send_email(
+            email_address=recipient,
+            subject=subject,
+            app_name=self.email_app_name,
+            html=html,
+            text=text,
+        )
+
+    def read_templates(
+        self,
+        filenames: List[str],
+        custom_template_directory: Optional[str] = None,
+    ) -> List[jinja2.Template]:
+        """Read and load the content of the template files at the given location.
+        By default, Synapse will look for these templates in its configured template
+        directory, but another directory to search in can be provided.
+
+        Args:
+            filenames: The name of the template files to look for.
+            custom_template_directory: An additional directory to look for the files in.
+
+        Returns:
+            A list containing the loaded templates, with the orders matching the one of
+            the filenames parameter.
+        """
+        return self._hs.config.read_templates(
+            filenames,
+            (td for td in (self.custom_template_dir, custom_template_directory) if td),
+        )
+
 
 class PublicRoomListManager:
     """Contains methods for adding to, removing from and querying whether a room
diff --git a/synapse/module_api/errors.py b/synapse/module_api/errors.py
index b15441772c..98ea911a81 100644
--- a/synapse/module_api/errors.py
+++ b/synapse/module_api/errors.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,4 +14,9 @@
 
 """Exception types which are exposed as part of the stable module API"""
 
-from synapse.api.errors import RedirectException, SynapseError  # noqa: F401
+from synapse.api.errors import (  # noqa: F401
+    InvalidClientCredentialsError,
+    RedirectException,
+    SynapseError,
+)
+from synapse.config._base import ConfigError  # noqa: F401
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 1374aae490..bbe337949a 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,6 +17,7 @@
 from typing import (
     Awaitable,
     Callable,
+    Collection,
     Dict,
     Iterable,
     List,
@@ -38,17 +38,13 @@
 from synapse.api.errors import AuthError
 from synapse.events import EventBase
 from synapse.handlers.presence import format_user_presence_state
+from synapse.logging import issue9533_logger
 from synapse.logging.context import PreserveLoggingContext
+from synapse.logging.opentracing import log_kv, start_active_span
 from synapse.logging.utils import log_function
 from synapse.metrics import LaterGauge
 from synapse.streams.config import PaginationConfig
-from synapse.types import (
-    Collection,
-    PersistedEventPosition,
-    RoomStreamToken,
-    StreamToken,
-    UserID,
-)
+from synapse.types import PersistedEventPosition, RoomStreamToken, StreamToken, UserID
 from synapse.util.async_helpers import ObservableDeferred, timeout_deferred
 from synapse.util.metrics import Measure
 from synapse.visibility import filter_events_for_client
@@ -115,8 +111,9 @@ def __init__(
         self.last_notified_token = current_token
         self.last_notified_ms = time_now_ms
 
-        with PreserveLoggingContext():
-            self.notify_deferred = ObservableDeferred(defer.Deferred())
+        self.notify_deferred: ObservableDeferred[StreamToken] = ObservableDeferred(
+            defer.Deferred()
+        )
 
     def notify(
         self,
@@ -136,6 +133,15 @@ def notify(
         self.last_notified_ms = time_now_ms
         noify_deferred = self.notify_deferred
 
+        log_kv(
+            {
+                "notify": self.user_id,
+                "stream": stream_key,
+                "stream_id": stream_id,
+                "listeners": self.count_listeners(),
+            }
+        )
+
         users_woken_by_stream_counter.labels(stream_key).inc()
 
         with PreserveLoggingContext():
@@ -198,21 +204,21 @@ class Notifier:
     UNUSED_STREAM_EXPIRY_MS = 10 * 60 * 1000
 
     def __init__(self, hs: "synapse.server.HomeServer"):
-        self.user_to_user_stream = {}  # type: Dict[str, _NotifierUserStream]
-        self.room_to_user_streams = {}  # type: Dict[str, Set[_NotifierUserStream]]
+        self.user_to_user_stream: Dict[str, _NotifierUserStream] = {}
+        self.room_to_user_streams: Dict[str, Set[_NotifierUserStream]] = {}
 
         self.hs = hs
         self.storage = hs.get_storage()
         self.event_sources = hs.get_event_sources()
         self.store = hs.get_datastore()
-        self.pending_new_room_events = []  # type: List[_PendingRoomEventEntry]
+        self.pending_new_room_events: List[_PendingRoomEventEntry] = []
 
         # Called when there are new things to stream over replication
-        self.replication_callbacks = []  # type: List[Callable[[], None]]
+        self.replication_callbacks: List[Callable[[], None]] = []
 
         # Called when remote servers have come back online after having been
         # down.
-        self.remote_server_up_callbacks = []  # type: List[Callable[[str], None]]
+        self.remote_server_up_callbacks: List[Callable[[str], None]] = []
 
         self.clock = hs.get_clock()
         self.appservice_handler = hs.get_application_service_handler()
@@ -232,7 +238,7 @@ def __init__(self, hs: "synapse.server.HomeServer"):
         # when rendering the metrics page, which is likely once per minute at
         # most when scraping it.
         def count_listeners():
-            all_user_streams = set()  # type: Set[_NotifierUserStream]
+            all_user_streams: Set[_NotifierUserStream] = set()
 
             for streams in list(self.room_to_user_streams.values()):
                 all_user_streams |= streams
@@ -266,7 +272,7 @@ def on_new_room_event(
         event: EventBase,
         event_pos: PersistedEventPosition,
         max_room_stream_token: RoomStreamToken,
-        extra_users: Collection[UserID] = [],
+        extra_users: Optional[Collection[UserID]] = None,
     ):
         """Unwraps event and calls `on_new_room_event_args`."""
         self.on_new_room_event_args(
@@ -276,7 +282,7 @@ def on_new_room_event(
             state_key=event.get("state_key"),
             membership=event.content.get("membership"),
             max_room_stream_token=max_room_stream_token,
-            extra_users=extra_users,
+            extra_users=extra_users or [],
         )
 
     def on_new_room_event_args(
@@ -287,7 +293,7 @@ def on_new_room_event_args(
         membership: Optional[str],
         event_pos: PersistedEventPosition,
         max_room_stream_token: RoomStreamToken,
-        extra_users: Collection[UserID] = [],
+        extra_users: Optional[Collection[UserID]] = None,
     ):
         """Used by handlers to inform the notifier something has happened
         in the room, room event wise.
@@ -303,7 +309,7 @@ def on_new_room_event_args(
         self.pending_new_room_events.append(
             _PendingRoomEventEntry(
                 event_pos=event_pos,
-                extra_users=extra_users,
+                extra_users=extra_users or [],
                 room_id=room_id,
                 type=event_type,
                 state_key=state_key,
@@ -324,8 +330,8 @@ def _notify_pending_new_room_events(self, max_room_stream_token: RoomStreamToken
         pending = self.pending_new_room_events
         self.pending_new_room_events = []
 
-        users = set()  # type: Set[UserID]
-        rooms = set()  # type: Set[str]
+        users: Set[UserID] = set()
+        rooms: Set[str] = set()
 
         for entry in pending:
             if entry.event_pos.persisted_after(max_room_stream_token):
@@ -372,14 +378,14 @@ def _notify_app_services_ephemeral(
         self,
         stream_key: str,
         new_token: Union[int, RoomStreamToken],
-        users: Collection[Union[str, UserID]] = [],
+        users: Optional[Collection[Union[str, UserID]]] = None,
     ):
         try:
             stream_token = None
             if isinstance(new_token, int):
                 stream_token = new_token
             self.appservice_handler.notify_interested_services_ephemeral(
-                stream_key, stream_token, users
+                stream_key, stream_token, users or []
             )
         except Exception:
             logger.exception("Error notifying application services of event")
@@ -394,16 +400,26 @@ def on_new_event(
         self,
         stream_key: str,
         new_token: Union[int, RoomStreamToken],
-        users: Collection[Union[str, UserID]] = [],
-        rooms: Collection[str] = [],
+        users: Optional[Collection[Union[str, UserID]]] = None,
+        rooms: Optional[Collection[str]] = None,
     ):
         """Used to inform listeners that something has happened event wise.
 
         Will wake up all listeners for the given users and rooms.
         """
+        users = users or []
+        rooms = rooms or []
+
         with Measure(self.clock, "on_new_event"):
             user_streams = set()
 
+            log_kv(
+                {
+                    "waking_up_explicit_users": len(users),
+                    "waking_up_explicit_rooms": len(rooms),
+                }
+            )
+
             for user in users:
                 user_stream = self.user_to_user_stream.get(str(user))
                 if user_stream is not None:
@@ -412,6 +428,13 @@ def on_new_event(
             for room in rooms:
                 user_streams |= self.room_to_user_streams.get(room, set())
 
+            if stream_key == "to_device_key":
+                issue9533_logger.debug(
+                    "to-device messages stream id %s, awaking streams for %s",
+                    new_token,
+                    users,
+                )
+
             time_now_ms = self.clock.time_msec()
             for user_stream in user_streams:
                 try:
@@ -463,36 +486,60 @@ async def wait_for_events(
             end_time = self.clock.time_msec() + timeout
 
             while not result:
-                try:
-                    now = self.clock.time_msec()
-                    if end_time <= now:
+                with start_active_span("wait_for_events"):
+                    try:
+                        now = self.clock.time_msec()
+                        if end_time <= now:
+                            break
+
+                        # Now we wait for the _NotifierUserStream to be told there
+                        # is a new token.
+                        listener = user_stream.new_listener(prev_token)
+                        listener.deferred = timeout_deferred(
+                            listener.deferred,
+                            (end_time - now) / 1000.0,
+                            self.hs.get_reactor(),
+                        )
+
+                        log_kv(
+                            {
+                                "wait_for_events": "sleep",
+                                "token": prev_token,
+                            }
+                        )
+
+                        with PreserveLoggingContext():
+                            await listener.deferred
+
+                        log_kv(
+                            {
+                                "wait_for_events": "woken",
+                                "token": user_stream.current_token,
+                            }
+                        )
+
+                        current_token = user_stream.current_token
+
+                        result = await callback(prev_token, current_token)
+                        log_kv(
+                            {
+                                "wait_for_events": "result",
+                                "result": bool(result),
+                            }
+                        )
+                        if result:
+                            break
+
+                        # Update the prev_token to the current_token since nothing
+                        # has happened between the old prev_token and the current_token
+                        prev_token = current_token
+                    except defer.TimeoutError:
+                        log_kv({"wait_for_events": "timeout"})
                         break
-
-                    # Now we wait for the _NotifierUserStream to be told there
-                    # is a new token.
-                    listener = user_stream.new_listener(prev_token)
-                    listener.deferred = timeout_deferred(
-                        listener.deferred,
-                        (end_time - now) / 1000.0,
-                        self.hs.get_reactor(),
-                    )
-                    with PreserveLoggingContext():
-                        await listener.deferred
-
-                    current_token = user_stream.current_token
-
-                    result = await callback(prev_token, current_token)
-                    if result:
+                    except defer.CancelledError:
+                        log_kv({"wait_for_events": "cancelled"})
                         break
 
-                    # Update the prev_token to the current_token since nothing
-                    # has happened between the old prev_token and the current_token
-                    prev_token = current_token
-                except defer.TimeoutError:
-                    break
-                except defer.CancelledError:
-                    break
-
         if result is None:
             # This happened if there was no timeout or if the timeout had
             # already expired.
@@ -507,7 +554,7 @@ async def get_events_for(
         pagination_config: PaginationConfig,
         timeout: int,
         is_guest: bool = False,
-        explicit_room_id: str = None,
+        explicit_room_id: Optional[str] = None,
     ) -> EventStreamResult:
         """For the given user and rooms, return any new events for them. If
         there are no new events wait for up to `timeout` milliseconds for any
@@ -534,7 +581,7 @@ async def check_for_updates(
             if after_token == before_token:
                 return EventStreamResult([], (from_token, from_token))
 
-            events = []  # type: List[EventBase]
+            events: List[EventBase] = []
             end_token = from_token
 
             for name, source in self.event_sources.sources.items():
diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py
index 9fc3da49a2..2c23afe8e3 100644
--- a/synapse/push/__init__.py
+++ b/synapse/push/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py
index 38a47a600f..60758df016 100644
--- a/synapse/push/action_generator.py
+++ b/synapse/push/action_generator.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 1897f59153..c337e530d3 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015 OpenMarket Ltd
 # Copyright 2017 New Vector Ltd
 #
@@ -105,7 +104,11 @@ class BulkPushRuleEvaluator:
     def __init__(self, hs: "HomeServer"):
         self.hs = hs
         self.store = hs.get_datastore()
-        self.auth = hs.get_auth()
+        self._event_auth_handler = hs.get_event_auth_handler()
+
+        # Used by `RulesForRoom` to ensure only one thing mutates the cache at a
+        # time. Keyed off room_id.
+        self._rules_linearizer = Linearizer(name="rules_for_room")
 
         self.room_push_rule_cache_metrics = register_cache(
             "cache",
@@ -124,7 +127,16 @@ async def _get_rules_for_event(
             dict of user_id -> push_rules
         """
         room_id = event.room_id
-        rules_for_room = self._get_rules_for_room(room_id)
+
+        rules_for_room_data = self._get_rules_for_room(room_id)
+        rules_for_room = RulesForRoom(
+            hs=self.hs,
+            room_id=room_id,
+            rules_for_room_cache=self._get_rules_for_room.cache,
+            room_push_rule_cache_metrics=self.room_push_rule_cache_metrics,
+            linearizer=self._rules_linearizer,
+            cached_data=rules_for_room_data,
+        )
 
         rules_by_user = await rules_for_room.get_rules(event, context)
 
@@ -143,17 +155,12 @@ async def _get_rules_for_event(
         return rules_by_user
 
     @lru_cache()
-    def _get_rules_for_room(self, room_id: str) -> "RulesForRoom":
-        """Get the current RulesForRoom object for the given room id"""
-        # It's important that RulesForRoom gets added to self._get_rules_for_room.cache
+    def _get_rules_for_room(self, room_id: str) -> "RulesForRoomData":
+        """Get the current RulesForRoomData object for the given room id"""
+        # It's important that the RulesForRoomData object gets added to self._get_rules_for_room.cache
         # before any lookup methods get called on it as otherwise there may be
         # a race if invalidate_all gets called (which assumes its in the cache)
-        return RulesForRoom(
-            self.hs,
-            room_id,
-            self._get_rules_for_room.cache,
-            self.room_push_rule_cache_metrics,
-        )
+        return RulesForRoomData()
 
     async def _get_power_levels_and_sender_level(
         self, event: EventBase, context: EventContext
@@ -165,7 +172,7 @@ async def _get_power_levels_and_sender_level(
             # not having a power level event is an extreme edge case
             auth_events = {POWER_KEY: await self.store.get_event(pl_event_id)}
         else:
-            auth_events_ids = self.auth.compute_auth_events(
+            auth_events_ids = self._event_auth_handler.compute_auth_events(
                 event, prev_state_ids, for_verification=False
             )
             auth_events_dict = await self.store.get_events(auth_events_ids)
@@ -187,7 +194,7 @@ async def action_for_event_by_user(
         count_as_unread = _should_count_as_unread(event, context)
 
         rules_by_user = await self._get_rules_for_event(event, context)
-        actions_by_user = {}  # type: Dict[str, List[Union[dict, str]]]
+        actions_by_user: Dict[str, List[Union[dict, str]]] = {}
 
         room_members = await self.store.get_joined_users_from_context(event, context)
 
@@ -200,7 +207,7 @@ async def action_for_event_by_user(
             event, len(room_members), sender_power_level, power_levels
         )
 
-        condition_cache = {}  # type: Dict[str, bool]
+        condition_cache: Dict[str, bool] = {}
 
         # If the event is not a state event check if any users ignore the sender.
         if not event.is_state():
@@ -283,11 +290,49 @@ def _condition_checker(
     return True
 
 
+@attr.s(slots=True)
+class RulesForRoomData:
+    """The data stored in the cache by `RulesForRoom`.
+
+    We don't store `RulesForRoom` directly in the cache as we want our caches to
+    *only* include data, and not references to e.g. the data stores.
+    """
+
+    # event_id -> (user_id, state)
+    member_map = attr.ib(type=Dict[str, Tuple[str, str]], factory=dict)
+    # user_id -> rules
+    rules_by_user = attr.ib(type=Dict[str, List[Dict[str, dict]]], factory=dict)
+
+    # The last state group we updated the caches for. If the state_group of
+    # a new event comes along, we know that we can just return the cached
+    # result.
+    # On invalidation of the rules themselves (if the user changes them),
+    # we invalidate everything and set state_group to `object()`
+    state_group = attr.ib(type=Union[object, int], factory=object)
+
+    # A sequence number to keep track of when we're allowed to update the
+    # cache. We bump the sequence number when we invalidate the cache. If
+    # the sequence number changes while we're calculating stuff we should
+    # not update the cache with it.
+    sequence = attr.ib(type=int, default=0)
+
+    # A cache of user_ids that we *know* aren't interesting, e.g. user_ids
+    # owned by AS's, or remote users, etc. (I.e. users we will never need to
+    # calculate push for)
+    # These never need to be invalidated as we will never set up push for
+    # them.
+    uninteresting_user_set = attr.ib(type=Set[str], factory=set)
+
+
 class RulesForRoom:
     """Caches push rules for users in a room.
 
     This efficiently handles users joining/leaving the room by not invalidating
     the entire cache for the room.
+
+    A new instance is constructed for each call to
+    `BulkPushRuleEvaluator._get_rules_for_event`, with the cached data from
+    previous calls passed in.
     """
 
     def __init__(
@@ -296,6 +341,8 @@ def __init__(
         room_id: str,
         rules_for_room_cache: LruCache,
         room_push_rule_cache_metrics: CacheMetric,
+        linearizer: Linearizer,
+        cached_data: RulesForRoomData,
     ):
         """
         Args:
@@ -304,38 +351,21 @@ def __init__(
             rules_for_room_cache: The cache object that caches these
                 RoomsForUser objects.
             room_push_rule_cache_metrics: The metrics object
+            linearizer: The linearizer used to ensure only one thing mutates
+                the cache at a time. Keyed off room_id
+            cached_data: Cached data from previous calls to `self.get_rules`,
+                can be mutated.
         """
         self.room_id = room_id
         self.is_mine_id = hs.is_mine_id
         self.store = hs.get_datastore()
         self.room_push_rule_cache_metrics = room_push_rule_cache_metrics
 
-        self.linearizer = Linearizer(name="rules_for_room")
-
-        # event_id -> (user_id, state)
-        self.member_map = {}  # type: Dict[str, Tuple[str, str]]
-        # user_id -> rules
-        self.rules_by_user = {}  # type: Dict[str, List[Dict[str, dict]]]
-
-        # The last state group we updated the caches for. If the state_group of
-        # a new event comes along, we know that we can just return the cached
-        # result.
-        # On invalidation of the rules themselves (if the user changes them),
-        # we invalidate everything and set state_group to `object()`
-        self.state_group = object()
-
-        # A sequence number to keep track of when we're allowed to update the
-        # cache. We bump the sequence number when we invalidate the cache. If
-        # the sequence number changes while we're calculating stuff we should
-        # not update the cache with it.
-        self.sequence = 0
-
-        # A cache of user_ids that we *know* aren't interesting, e.g. user_ids
-        # owned by AS's, or remote users, etc. (I.e. users we will never need to
-        # calculate push for)
-        # These never need to be invalidated as we will never set up push for
-        # them.
-        self.uninteresting_user_set = set()  # type: Set[str]
+        # Used to ensure only one thing mutates the cache at a time. Keyed off
+        # room_id.
+        self.linearizer = linearizer
+
+        self.data = cached_data
 
         # We need to be clever on the invalidating caches callbacks, as
         # otherwise the invalidation callback holds a reference to the object,
@@ -353,25 +383,25 @@ async def get_rules(
         """
         state_group = context.state_group
 
-        if state_group and self.state_group == state_group:
+        if state_group and self.data.state_group == state_group:
             logger.debug("Using cached rules for %r", self.room_id)
             self.room_push_rule_cache_metrics.inc_hits()
-            return self.rules_by_user
+            return self.data.rules_by_user
 
-        with (await self.linearizer.queue(())):
-            if state_group and self.state_group == state_group:
+        with (await self.linearizer.queue(self.room_id)):
+            if state_group and self.data.state_group == state_group:
                 logger.debug("Using cached rules for %r", self.room_id)
                 self.room_push_rule_cache_metrics.inc_hits()
-                return self.rules_by_user
+                return self.data.rules_by_user
 
             self.room_push_rule_cache_metrics.inc_misses()
 
             ret_rules_by_user = {}
             missing_member_event_ids = {}
-            if state_group and self.state_group == context.prev_group:
+            if state_group and self.data.state_group == context.prev_group:
                 # If we have a simple delta then we can reuse most of the previous
                 # results.
-                ret_rules_by_user = self.rules_by_user
+                ret_rules_by_user = self.data.rules_by_user
                 current_state_ids = context.delta_ids
 
                 push_rules_delta_state_cache_metric.inc_hits()
@@ -394,24 +424,24 @@ async def get_rules(
                 if typ != EventTypes.Member:
                     continue
 
-                if user_id in self.uninteresting_user_set:
+                if user_id in self.data.uninteresting_user_set:
                     continue
 
                 if not self.is_mine_id(user_id):
-                    self.uninteresting_user_set.add(user_id)
+                    self.data.uninteresting_user_set.add(user_id)
                     continue
 
                 if self.store.get_if_app_services_interested_in_user(user_id):
-                    self.uninteresting_user_set.add(user_id)
+                    self.data.uninteresting_user_set.add(user_id)
                     continue
 
                 event_id = current_state_ids[key]
 
-                res = self.member_map.get(event_id, None)
+                res = self.data.member_map.get(event_id, None)
                 if res:
                     user_id, state = res
                     if state == Membership.JOIN:
-                        rules = self.rules_by_user.get(user_id, None)
+                        rules = self.data.rules_by_user.get(user_id, None)
                         if rules:
                             ret_rules_by_user[user_id] = rules
                     continue
@@ -431,7 +461,7 @@ async def get_rules(
             else:
                 # The push rules didn't change but lets update the cache anyway
                 self.update_cache(
-                    self.sequence,
+                    self.data.sequence,
                     members={},  # There were no membership changes
                     rules_by_user=ret_rules_by_user,
                     state_group=state_group,
@@ -462,7 +492,7 @@ async def _update_rules_with_member_event_ids(
                 for. Used when updating the cache.
             event: The event we are currently computing push rules for.
         """
-        sequence = self.sequence
+        sequence = self.data.sequence
 
         rows = await self.store.get_membership_from_event_ids(member_event_ids.values())
 
@@ -502,23 +532,11 @@ async def _update_rules_with_member_event_ids(
 
         self.update_cache(sequence, members, ret_rules_by_user, state_group)
 
-    def invalidate_all(self) -> None:
-        # Note: Don't hand this function directly to an invalidation callback
-        # as it keeps a reference to self and will stop this instance from being
-        # GC'd if it gets dropped from the rules_to_user cache. Instead use
-        # `self.invalidate_all_cb`
-        logger.debug("Invalidating RulesForRoom for %r", self.room_id)
-        self.sequence += 1
-        self.state_group = object()
-        self.member_map = {}
-        self.rules_by_user = {}
-        push_rules_invalidation_counter.inc()
-
     def update_cache(self, sequence, members, rules_by_user, state_group) -> None:
-        if sequence == self.sequence:
-            self.member_map.update(members)
-            self.rules_by_user = rules_by_user
-            self.state_group = state_group
+        if sequence == self.data.sequence:
+            self.data.member_map.update(members)
+            self.data.rules_by_user = rules_by_user
+            self.data.state_group = state_group
 
 
 @attr.attrs(slots=True, frozen=True)
@@ -536,6 +554,10 @@ class _Invalidation:
     room_id = attr.ib(type=str)
 
     def __call__(self) -> None:
-        rules = self.cache.get(self.room_id, None, update_metrics=False)
-        if rules:
-            rules.invalidate_all()
+        rules_data = self.cache.get(self.room_id, None, update_metrics=False)
+        if rules_data:
+            rules_data.sequence += 1
+            rules_data.state_group = object()
+            rules_data.member_map = {}
+            rules_data.rules_by_user = {}
+            push_rules_invalidation_counter.inc()
diff --git a/synapse/push/clientformat.py b/synapse/push/clientformat.py
index 0cadba761a..1fc9716a34 100644
--- a/synapse/push/clientformat.py
+++ b/synapse/push/clientformat.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,10 +26,10 @@ def format_push_rules_for_user(user: UserID, ruleslist) -> Dict[str, Dict[str, l
     # We're going to be mutating this a lot, so do a deep copy
     ruleslist = copy.deepcopy(ruleslist)
 
-    rules = {
+    rules: Dict[str, Dict[str, List[Dict[str, Any]]]] = {
         "global": {},
         "device": {},
-    }  # type: Dict[str, Dict[str, List[Dict[str, Any]]]]
+    }
 
     rules["global"] = _add_empty_priority_class_arrays(rules["global"])
 
diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py
index c0968dc7a1..e08e125cb8 100644
--- a/synapse/push/emailpusher.py
+++ b/synapse/push/emailpusher.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,8 +19,9 @@
 from twisted.internet.interfaces import IDelayedCall
 
 from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.push import Pusher, PusherConfig, ThrottleParams
+from synapse.push import Pusher, PusherConfig, PusherConfigException, ThrottleParams
 from synapse.push.mailer import Mailer
+from synapse.util.threepids import validate_email
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -66,12 +66,18 @@ def __init__(self, hs: "HomeServer", pusher_config: PusherConfig, mailer: Mailer
 
         self.store = self.hs.get_datastore()
         self.email = pusher_config.pushkey
-        self.timed_call = None  # type: Optional[IDelayedCall]
-        self.throttle_params = {}  # type: Dict[str, ThrottleParams]
+        self.timed_call: Optional[IDelayedCall] = None
+        self.throttle_params: Dict[str, ThrottleParams] = {}
         self._inited = False
 
         self._is_processing = False
 
+        # Make sure that the email is valid.
+        try:
+            validate_email(self.email)
+        except ValueError:
+            raise PusherConfigException("Invalid email")
+
     def on_started(self, should_check_for_notifs: bool) -> None:
         """Called when this pusher has been started.
 
@@ -162,7 +168,7 @@ async def _unsafe_process(self) -> None:
             )
         )
 
-        soonest_due_at = None  # type: Optional[int]
+        soonest_due_at: Optional[int] = None
 
         if not unprocessed:
             await self.save_last_stream_ordering_and_success(self.max_stream_ordering)
diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index 26af5309c1..36aabd8422 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 New Vector Ltd
 #
@@ -72,7 +71,7 @@ def __init__(self, hs: "HomeServer", pusher_config: PusherConfig):
         self.data = pusher_config.data
         self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC
         self.failing_since = pusher_config.failing_since
-        self.timed_call = None  # type: Optional[IDelayedCall]
+        self.timed_call: Optional[IDelayedCall] = None
         self._is_processing = False
         self._group_unread_count_by_room = hs.config.push_group_unread_count_by_room
         self._pusherpool = hs.get_pusherpool()
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index 2e5161de2c..941fb238b7 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,22 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import email.mime.multipart
-import email.utils
 import logging
 import urllib.parse
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, TypeVar
 
 import bleach
 import jinja2
 
-from synapse.api.constants import EventTypes, Membership
+from synapse.api.constants import EventTypes, Membership, RoomTypes
 from synapse.api.errors import StoreError
 from synapse.config.emailconfig import EmailSubjectConfig
 from synapse.events import EventBase
-from synapse.logging.context import make_deferred_yieldable
 from synapse.push.presentable_names import (
     calculate_room_name,
     descriptor_from_member_events,
@@ -109,14 +103,14 @@ def __init__(
         self.template_html = template_html
         self.template_text = template_text
 
-        self.sendmail = self.hs.get_sendmail()
+        self.send_email_handler = hs.get_send_email_handler()
         self.store = self.hs.get_datastore()
         self.state_store = self.hs.get_storage().state
         self.macaroon_gen = self.hs.get_macaroon_generator()
         self.state_handler = self.hs.get_state_handler()
         self.storage = hs.get_storage()
         self.app_name = app_name
-        self.email_subjects = hs.config.email_subjects  # type: EmailSubjectConfig
+        self.email_subjects: EmailSubjectConfig = hs.config.email_subjects
 
         logger.info("Created Mailer for app_name %s" % app_name)
 
@@ -236,7 +230,7 @@ async def send_notification_mail(
             [pa["event_id"] for pa in push_actions]
         )
 
-        notifs_by_room = {}  # type: Dict[str, List[Dict[str, Any]]]
+        notifs_by_room: Dict[str, List[Dict[str, Any]]] = {}
         for pa in push_actions:
             notifs_by_room.setdefault(pa["room_id"], []).append(pa)
 
@@ -311,17 +305,6 @@ async def send_email(
         self, email_address: str, subject: str, extra_template_vars: Dict[str, Any]
     ) -> None:
         """Send an email with the given information and template text"""
-        try:
-            from_string = self.hs.config.email_notif_from % {"app": self.app_name}
-        except TypeError:
-            from_string = self.hs.config.email_notif_from
-
-        raw_from = email.utils.parseaddr(from_string)[1]
-        raw_to = email.utils.parseaddr(email_address)[1]
-
-        if raw_to == "":
-            raise RuntimeError("Invalid 'to' address")
-
         template_vars = {
             "app_name": self.app_name,
             "server_name": self.hs.config.server.server_name,
@@ -330,35 +313,14 @@ async def send_email(
         template_vars.update(extra_template_vars)
 
         html_text = self.template_html.render(**template_vars)
-        html_part = MIMEText(html_text, "html", "utf8")
-
         plain_text = self.template_text.render(**template_vars)
-        text_part = MIMEText(plain_text, "plain", "utf8")
-
-        multipart_msg = MIMEMultipart("alternative")
-        multipart_msg["Subject"] = subject
-        multipart_msg["From"] = from_string
-        multipart_msg["To"] = email_address
-        multipart_msg["Date"] = email.utils.formatdate()
-        multipart_msg["Message-ID"] = email.utils.make_msgid()
-        multipart_msg.attach(text_part)
-        multipart_msg.attach(html_part)
-
-        logger.info("Sending email to %s" % email_address)
-
-        await make_deferred_yieldable(
-            self.sendmail(
-                self.hs.config.email_smtp_host,
-                raw_from,
-                raw_to,
-                multipart_msg.as_string().encode("utf8"),
-                reactor=self.hs.get_reactor(),
-                port=self.hs.config.email_smtp_port,
-                requireAuthentication=self.hs.config.email_smtp_user is not None,
-                username=self.hs.config.email_smtp_user,
-                password=self.hs.config.email_smtp_pass,
-                requireTransportSecurity=self.hs.config.require_transport_security,
-            )
+
+        await self.send_email_handler.send_email(
+            email_address=email_address,
+            subject=subject,
+            app_name=self.app_name,
+            html=html_text,
+            text=plain_text,
         )
 
     async def _get_room_vars(
@@ -394,13 +356,13 @@ async def _get_room_vars(
 
         room_name = await calculate_room_name(self.store, room_state_ids, user_id)
 
-        room_vars = {
+        room_vars: Dict[str, Any] = {
             "title": room_name,
             "hash": string_ordinal_total(room_id),  # See sender avatar hash
             "notifs": [],
             "invite": is_invite,
             "link": self._make_room_link(room_id),
-        }  # type: Dict[str, Any]
+        }
 
         if not is_invite:
             for n in notifs:
@@ -498,9 +460,9 @@ async def _get_message_vars(
         type_state_key = ("m.room.member", event.sender)
         sender_state_event_id = room_state_ids.get(type_state_key)
         if sender_state_event_id:
-            sender_state_event = await self.store.get_event(
+            sender_state_event: Optional[EventBase] = await self.store.get_event(
                 sender_state_event_id
-            )  # type: Optional[EventBase]
+            )
         else:
             # Attempt to check the historical state for the room.
             historical_state = await self.state_store.get_state_for_event(
@@ -638,6 +600,22 @@ async def _make_summary_text_single_room(
                     "app": self.app_name,
                 }
 
+            # If the room is a space, it gets a slightly different topic.
+            create_event_id = room_state_ids.get(("m.room.create", ""))
+            if create_event_id:
+                create_event = await self.store.get_event(
+                    create_event_id, allow_none=True
+                )
+                if (
+                    create_event
+                    and create_event.content.get("room_type") == RoomTypes.SPACE
+                ):
+                    return self.email_subjects.invite_from_person_to_space % {
+                        "person": inviter_name,
+                        "space": room_name,
+                        "app": self.app_name,
+                    }
+
             return self.email_subjects.invite_from_person_to_room % {
                 "person": inviter_name,
                 "room": room_name,
diff --git a/synapse/push/presentable_names.py b/synapse/push/presentable_names.py
index 04c2c1482c..0510c1cbd5 100644
--- a/synapse/push/presentable_names.py
+++ b/synapse/push/presentable_names.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -200,7 +199,7 @@ def name_from_member_event(member_event: EventBase) -> str:
 
 
 def _state_as_two_level_dict(state: StateMap[str]) -> Dict[str, Dict[str, str]]:
-    ret = {}  # type: Dict[str, Dict[str, str]]
+    ret: Dict[str, Dict[str, str]] = {}
     for k, v in state.items():
         ret.setdefault(k[0], {})[k[1]] = v
     return ret
diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py
index ba1877adcd..7a8dc63976 100644
--- a/synapse/push/push_rule_evaluator.py
+++ b/synapse/push/push_rule_evaluator.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 New Vector Ltd
 #
@@ -20,6 +19,7 @@
 
 from synapse.events import EventBase
 from synapse.types import UserID
+from synapse.util import glob_to_regex, re_word_boundary
 from synapse.util.caches.lrucache import LruCache
 
 logger = logging.getLogger(__name__)
@@ -184,7 +184,7 @@ def _contains_display_name(self, display_name: str) -> bool:
         r = regex_cache.get((display_name, False, True), None)
         if not r:
             r1 = re.escape(display_name)
-            r1 = _re_word_boundary(r1)
+            r1 = re_word_boundary(r1)
             r = re.compile(r1, flags=re.IGNORECASE)
             regex_cache[(display_name, False, True)] = r
 
@@ -195,9 +195,9 @@ def _get_value(self, dotted_key: str) -> Optional[str]:
 
 
 # Caches (string, is_glob, word_boundary) -> regex for push. See _glob_matches
-regex_cache = LruCache(
+regex_cache: LruCache[Tuple[str, bool, bool], Pattern] = LruCache(
     50000, "regex_push_cache"
-)  # type: LruCache[Tuple[str, bool, bool], Pattern]
+)
 
 
 def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool:
@@ -213,7 +213,7 @@ def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool:
     try:
         r = regex_cache.get((glob, True, word_boundary), None)
         if not r:
-            r = _glob_to_re(glob, word_boundary)
+            r = glob_to_regex(glob, word_boundary)
             regex_cache[(glob, True, word_boundary)] = r
         return bool(r.search(value))
     except re.error:
@@ -221,56 +221,6 @@ def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool:
         return False
 
 
-def _glob_to_re(glob: str, word_boundary: bool) -> Pattern:
-    """Generates regex for a given glob.
-
-    Args:
-        glob
-        word_boundary: Whether to match against word boundaries or entire string.
-    """
-    if IS_GLOB.search(glob):
-        r = re.escape(glob)
-
-        r = r.replace(r"\*", ".*?")
-        r = r.replace(r"\?", ".")
-
-        # handle [abc], [a-z] and [!a-z] style ranges.
-        r = GLOB_REGEX.sub(
-            lambda x: (
-                "[%s%s]" % (x.group(1) and "^" or "", x.group(2).replace(r"\\\-", "-"))
-            ),
-            r,
-        )
-        if word_boundary:
-            r = _re_word_boundary(r)
-
-            return re.compile(r, flags=re.IGNORECASE)
-        else:
-            r = "^" + r + "$"
-
-            return re.compile(r, flags=re.IGNORECASE)
-    elif word_boundary:
-        r = re.escape(glob)
-        r = _re_word_boundary(r)
-
-        return re.compile(r, flags=re.IGNORECASE)
-    else:
-        r = "^" + re.escape(glob) + "$"
-        return re.compile(r, flags=re.IGNORECASE)
-
-
-def _re_word_boundary(r: str) -> str:
-    """
-    Adds word boundary characters to the start and end of an
-    expression to require that the match occur as a whole word,
-    but do so respecting the fact that strings starting or ending
-    with non-word characters will change word boundaries.
-    """
-    # we can't use \b as it chokes on unicode. however \W seems to be okay
-    # as shorthand for [^0-9A-Za-z_].
-    return r"(^|\W)%s(\W|$)" % (r,)
-
-
 def _flatten_dict(
     d: Union[EventBase, dict],
     prefix: Optional[List[str]] = None,
diff --git a/synapse/push/push_tools.py b/synapse/push/push_tools.py
index df34103224..9c85200c0f 100644
--- a/synapse/push/push_tools.py
+++ b/synapse/push/push_tools.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py
index cb94127850..021275437c 100644
--- a/synapse/push/pusher.py
+++ b/synapse/push/pusher.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -32,13 +31,13 @@ def __init__(self, hs: "HomeServer"):
         self.hs = hs
         self.config = hs.config
 
-        self.pusher_types = {
+        self.pusher_types: Dict[str, Callable[[HomeServer, PusherConfig], Pusher]] = {
             "http": HttpPusher
-        }  # type: Dict[str, Callable[[HomeServer, PusherConfig], Pusher]]
+        }
 
         logger.info("email enable notifs: %r", hs.config.email_enable_notifs)
         if hs.config.email_enable_notifs:
-            self.mailers = {}  # type: Dict[str, Mailer]
+            self.mailers: Dict[str, Mailer] = {}
 
             self._notif_template_html = hs.config.email_notif_template_html
             self._notif_template_text = hs.config.email_notif_template_text
diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py
index 2534ecb1d4..a1436f3930 100644
--- a/synapse/push/pusherpool.py
+++ b/synapse/push/pusherpool.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -63,8 +61,6 @@ def __init__(self, hs: "HomeServer"):
         self.store = self.hs.get_datastore()
         self.clock = self.hs.get_clock()
 
-        self._account_validity_enabled = hs.config.account_validity_enabled
-
         # We shard the handling of push notifications by user ID.
         self._pusher_shard_config = hs.config.push.pusher_shard_config
         self._instance_name = hs.get_instance_name()
@@ -86,7 +82,9 @@ def __init__(self, hs: "HomeServer"):
         self._last_room_stream_id_seen = self.store.get_room_max_stream_ordering()
 
         # map from user id to app_id:pushkey to pusher
-        self.pushers = {}  # type: Dict[str, Dict[str, Pusher]]
+        self.pushers: Dict[str, Dict[str, Pusher]] = {}
+
+        self._account_validity_handler = hs.get_account_validity_handler()
 
     def start(self) -> None:
         """Starts the pushers off in a background process."""
@@ -237,12 +235,9 @@ async def _on_new_notifications(self, max_token: RoomStreamToken) -> None:
 
             for u in users_affected:
                 # Don't push if the user account has expired
-                if self._account_validity_enabled:
-                    expired = await self.store.is_account_expired(
-                        u, self.clock.time_msec()
-                    )
-                    if expired:
-                        continue
+                expired = await self._account_validity_handler.is_user_expired(u)
+                if expired:
+                    continue
 
                 if u in self.pushers:
                     for p in self.pushers[u].values():
@@ -267,12 +262,9 @@ async def on_new_receipts(
 
             for u in users_affected:
                 # Don't push if the user account has expired
-                if self._account_validity_enabled:
-                    expired = await self.store.is_account_expired(
-                        u, self.clock.time_msec()
-                    )
-                    if expired:
-                        continue
+                expired = await self._account_validity_handler.is_user_expired(u)
+                if expired:
+                    continue
 
                 if u in self.pushers:
                     for p in self.pushers[u].values():
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 2a1c925ee8..cdcbdd772b 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -75,17 +75,17 @@
     "phonenumbers>=8.2.0",
     # we use GaugeHistogramMetric, which was added in prom-client 0.4.0.
     "prometheus_client>=0.4.0",
-    # we use attr.validators.deep_iterable, which arrived in 19.1.0 (Note:
-    # Fedora 31 only has 19.1, so if we want to upgrade we should wait until 33
-    # is out in November.)
-    "attrs>=19.1.0",
+    # we use `order`, which arrived in attrs 19.2.0.
+    # Note: 21.1.0 broke `/sync`, see #9936
+    "attrs>=19.2.0,!=21.1.0",
     "netaddr>=0.7.18",
     "Jinja2>=2.9",
     "bleach>=1.4.3",
     "typing-extensions>=3.7.4",
     # We enforce that we have a `cryptography` version that bundles an `openssl`
     # with the latest security patches.
-    "cryptography>=3.4.7;python_version>='3.6'",
+    "cryptography>=3.4.7",
+    "ijson>=3.0",
 ]
 
 CONDITIONAL_REQUIREMENTS = {
@@ -96,18 +96,8 @@
         "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'",
         "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'",
     ],
-    # ACME support is required to provision TLS certificates from authorities
-    # that use the protocol, such as Let's Encrypt.
-    "acme": [
-        "txacme>=0.9.2",
-        # txacme depends on eliot. Eliot 1.8.0 is incompatible with
-        # python 3.5.2, as per https://github.com/itamarst/eliot/issues/418
-        "eliot<1.8.0;python_version<'3.5.3'",
-    ],
     "saml2": [
-        # pysaml2 6.4.0 is incompatible with Python 3.5 (see https://github.com/IdentityPython/pysaml2/issues/749)
-        "pysaml2>=4.5.0,<6.4.0;python_version<'3.6'",
-        "pysaml2>=4.5.0;python_version>='3.6'",
+        "pysaml2>=4.5.0",
     ],
     "oidc": ["authlib>=0.14.0"],
     # systemd-python is necessary for logging to the systemd journal via
@@ -121,9 +111,11 @@
     # hiredis is not a *strict* dependency, but it makes things much faster.
     # (if it is not installed, we fall back to slow code.)
     "redis": ["txredisapi>=1.4.7", "hiredis"],
+    # Required to use experimental `caches.track_memory_usage` config option.
+    "cache_memory": ["pympler"],
 }
 
-ALL_OPTIONAL_REQUIREMENTS = set()  # type: Set[str]
+ALL_OPTIONAL_REQUIREMENTS: Set[str] = set()
 
 for name, optional_deps in CONDITIONAL_REQUIREMENTS.items():
     # Exclude systemd as it's a system-based requirement.
@@ -201,7 +193,7 @@ def check_requirements(for_feature=None):
     if not for_feature:
         # Check the optional dependencies are up to date. We allow them to not be
         # installed.
-        OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), [])  # type: List[str]
+        OPTS: List[str] = sum(CONDITIONAL_REQUIREMENTS.values(), [])
 
         for dependency in OPTS:
             try:
diff --git a/synapse/replication/__init__.py b/synapse/replication/__init__.py
index b7df13c9ee..f43a360a80 100644
--- a/synapse/replication/__init__.py
+++ b/synapse/replication/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py
index cb4a52dbe9..ba8114ac9e 100644
--- a/synapse/replication/http/__init__.py
+++ b/synapse/replication/http/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py
index b7aa0c280f..25589b0042 100644
--- a/synapse/replication/http/_base.py
+++ b/synapse/replication/http/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,7 +23,8 @@
 
 from synapse.api.errors import HttpResponseException, SynapseError
 from synapse.http import RequestTimedOutError
-from synapse.logging.opentracing import inject_active_span_byte_dict, trace
+from synapse.logging import opentracing
+from synapse.logging.opentracing import trace
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.stringutils import random_string
 
@@ -85,17 +85,17 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta):
             is received.
     """
 
-    NAME = abc.abstractproperty()  # type: str  # type: ignore
-    PATH_ARGS = abc.abstractproperty()  # type: Tuple[str, ...]  # type: ignore
+    NAME: str = abc.abstractproperty()  # type: ignore
+    PATH_ARGS: Tuple[str, ...] = abc.abstractproperty()  # type: ignore
     METHOD = "POST"
     CACHE = True
     RETRY_ON_TIMEOUT = True
 
     def __init__(self, hs: "HomeServer"):
         if self.CACHE:
-            self.response_cache = ResponseCache(
+            self.response_cache: ResponseCache[str] = ResponseCache(
                 hs.get_clock(), "repl." + self.NAME, timeout_ms=30 * 60 * 1000
-            )  # type: ResponseCache[str]
+            )
 
         # We reserve `instance_name` as a parameter to sending requests, so we
         # assert here that sub classes don't try and use the name.
@@ -159,7 +159,10 @@ async def _handle_request(self, request, **kwargs):
     def make_client(cls, hs):
         """Create a client that makes requests.
 
-        Returns a callable that accepts the same parameters as `_serialize_payload`.
+        Returns a callable that accepts the same parameters as
+        `_serialize_payload`, and also accepts an optional `instance_name`
+        parameter to specify which instance to hit (the instance must be in
+        the `instance_map` config).
         """
         clock = hs.get_clock()
         client = hs.get_simple_http_client()
@@ -229,11 +232,11 @@ async def send_request(*, instance_name="master", **kwargs):
                 # have a good idea that the request has either succeeded or failed on
                 # the master, and so whether we should clean up or not.
                 while True:
-                    headers = {}  # type: Dict[bytes, List[bytes]]
+                    headers: Dict[bytes, List[bytes]] = {}
                     # Add an authorization header, if configured.
                     if replication_secret:
                         headers[b"Authorization"] = [b"Bearer " + replication_secret]
-                    inject_active_span_byte_dict(headers, None, check_destination=False)
+                    opentracing.inject_header_dict(headers, check_destination=False)
                     try:
                         result = await request_func(uri, data, headers=headers)
                         break
@@ -282,7 +285,7 @@ def register(self, http_server):
             self.__class__.__name__,
         )
 
-    def _check_auth_and_handle(self, request, **kwargs):
+    async def _check_auth_and_handle(self, request, **kwargs):
         """Called on new incoming requests when caching is enabled. Checks
         if there is a cached response for the request and returns that,
         otherwise calls `_handle_request` and caches its response.
@@ -297,8 +300,8 @@ def _check_auth_and_handle(self, request, **kwargs):
         if self.CACHE:
             txn_id = kwargs.pop("txn_id")
 
-            return self.response_cache.wrap(
+            return await self.response_cache.wrap(
                 txn_id, self._handle_request, request, **kwargs
             )
 
-        return self._handle_request(request, **kwargs)
+        return await self._handle_request(request, **kwargs)
diff --git a/synapse/replication/http/account_data.py b/synapse/replication/http/account_data.py
index 60899b6ad6..70e951af63 100644
--- a/synapse/replication/http/account_data.py
+++ b/synapse/replication/http/account_data.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/http/devices.py b/synapse/replication/http/devices.py
index 807b85d2e1..5a5818ef61 100644
--- a/synapse/replication/http/devices.py
+++ b/synapse/replication/http/devices.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py
index 82ea3b895f..79cadb7b57 100644
--- a/synapse/replication/http/federation.py
+++ b/synapse/replication/http/federation.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/http/login.py b/synapse/replication/http/login.py
index 4ec1bfa6ea..550bd5c95f 100644
--- a/synapse/replication/http/login.py
+++ b/synapse/replication/http/login.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -37,20 +36,29 @@ def __init__(self, hs):
 
     @staticmethod
     async def _serialize_payload(
-        user_id, device_id, initial_display_name, is_guest, is_appservice_ghost
+        user_id,
+        device_id,
+        initial_display_name,
+        is_guest,
+        is_appservice_ghost,
+        should_issue_refresh_token,
     ):
         """
         Args:
+            user_id (int)
             device_id (str|None): Device ID to use, if None a new one is
                 generated.
             initial_display_name (str|None)
             is_guest (bool)
+            is_appservice_ghost (bool)
+            should_issue_refresh_token (bool)
         """
         return {
             "device_id": device_id,
             "initial_display_name": initial_display_name,
             "is_guest": is_guest,
             "is_appservice_ghost": is_appservice_ghost,
+            "should_issue_refresh_token": should_issue_refresh_token,
         }
 
     async def _handle_request(self, request, user_id):
@@ -60,6 +68,7 @@ async def _handle_request(self, request, user_id):
         initial_display_name = content["initial_display_name"]
         is_guest = content["is_guest"]
         is_appservice_ghost = content["is_appservice_ghost"]
+        should_issue_refresh_token = content["should_issue_refresh_token"]
 
         res = await self.registration_handler.register_device_inner(
             user_id,
@@ -67,6 +76,7 @@ async def _handle_request(self, request, user_id):
             initial_display_name,
             is_guest,
             is_appservice_ghost=is_appservice_ghost,
+            should_issue_refresh_token=should_issue_refresh_token,
         )
 
         return 200, res
diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py
index 2812ac12fc..34206c5060 100644
--- a/synapse/replication/http/membership.py
+++ b/synapse/replication/http/membership.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -115,7 +114,7 @@ class ReplicationRemoteKnockRestServlet(ReplicationEndpoint):
     NAME = "remote_knock"
     PATH_ARGS = ("room_id", "user_id")
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.federation_handler = hs.get_federation_handler()
@@ -346,7 +345,7 @@ async def _serialize_payload(  # type: ignore
 
         return {}
 
-    def _handle_request(  # type: ignore
+    async def _handle_request(  # type: ignore
         self, request: Request, room_id: str, user_id: str, change: str
     ) -> Tuple[int, JsonDict]:
         logger.info("user membership change: %s in %s", user_id, room_id)
diff --git a/synapse/replication/http/presence.py b/synapse/replication/http/presence.py
index bc9aa82cb4..bb00247953 100644
--- a/synapse/replication/http/presence.py
+++ b/synapse/replication/http/presence.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -74,6 +73,7 @@ class ReplicationPresenceSetState(ReplicationEndpoint):
         {
             "state": { ... },
             "ignore_status_msg": false,
+            "force_notify": false
         }
 
         200 OK
@@ -92,17 +92,23 @@ def __init__(self, hs: "HomeServer"):
         self._presence_handler = hs.get_presence_handler()
 
     @staticmethod
-    async def _serialize_payload(user_id, state, ignore_status_msg=False):
+    async def _serialize_payload(
+        user_id, state, ignore_status_msg=False, force_notify=False
+    ):
         return {
             "state": state,
             "ignore_status_msg": ignore_status_msg,
+            "force_notify": force_notify,
         }
 
     async def _handle_request(self, request, user_id):
         content = parse_json_object_from_request(request)
 
         await self._presence_handler.set_state(
-            UserID.from_string(user_id), content["state"], content["ignore_status_msg"]
+            UserID.from_string(user_id),
+            content["state"],
+            content["ignore_status_msg"],
+            content["force_notify"],
         )
 
         return (
diff --git a/synapse/replication/http/push.py b/synapse/replication/http/push.py
index 054ed64d34..139427cb1f 100644
--- a/synapse/replication/http/push.py
+++ b/synapse/replication/http/push.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/http/register.py b/synapse/replication/http/register.py
index d005f38767..d6dd7242eb 100644
--- a/synapse/replication/http/register.py
+++ b/synapse/replication/http/register.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -77,7 +76,7 @@ async def _serialize_payload(
     async def _handle_request(self, request, user_id):
         content = parse_json_object_from_request(request)
 
-        self.registration_handler.check_registration_ratelimit(content["address"])
+        await self.registration_handler.check_registration_ratelimit(content["address"])
 
         await self.registration_handler.register_with_store(
             user_id=user_id,
diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py
index a4c5b44292..fae5ffa451 100644
--- a/synapse/replication/http/send_event.py
+++ b/synapse/replication/http/send_event.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/http/streams.py b/synapse/replication/http/streams.py
index 309159e304..9afa147d00 100644
--- a/synapse/replication/http/streams.py
+++ b/synapse/replication/http/streams.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/__init__.py b/synapse/replication/slave/__init__.py
index b7df13c9ee..f43a360a80 100644
--- a/synapse/replication/slave/__init__.py
+++ b/synapse/replication/slave/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/storage/__init__.py b/synapse/replication/slave/storage/__init__.py
index b7df13c9ee..f43a360a80 100644
--- a/synapse/replication/slave/storage/__init__.py
+++ b/synapse/replication/slave/storage/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/storage/_base.py b/synapse/replication/slave/storage/_base.py
index 693c9ab901..e460dd85cd 100644
--- a/synapse/replication/slave/storage/_base.py
+++ b/synapse/replication/slave/storage/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,7 +27,9 @@ class BaseSlavedStore(CacheInvalidationWorkerStore):
     def __init__(self, database: DatabasePool, db_conn, hs):
         super().__init__(database, db_conn, hs)
         if isinstance(self.database_engine, PostgresEngine):
-            self._cache_id_gen = MultiWriterIdGenerator(
+            self._cache_id_gen: Optional[
+                MultiWriterIdGenerator
+            ] = MultiWriterIdGenerator(
                 db_conn,
                 database,
                 stream_name="caches",
@@ -42,7 +43,7 @@ def __init__(self, database: DatabasePool, db_conn, hs):
                 ],
                 sequence_name="cache_invalidation_stream_seq",
                 writers=[],
-            )  # type: Optional[MultiWriterIdGenerator]
+            )
         else:
             self._cache_id_gen = None
 
diff --git a/synapse/replication/slave/storage/_slaved_id_tracker.py b/synapse/replication/slave/storage/_slaved_id_tracker.py
index 0d39a93ed2..2cb7489047 100644
--- a/synapse/replication/slave/storage/_slaved_id_tracker.py
+++ b/synapse/replication/slave/storage/_slaved_id_tracker.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/storage/account_data.py b/synapse/replication/slave/storage/account_data.py
index 21afe5f155..ee74ee7d85 100644
--- a/synapse/replication/slave/storage/account_data.py
+++ b/synapse/replication/slave/storage/account_data.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/replication/slave/storage/appservice.py b/synapse/replication/slave/storage/appservice.py
index 0f8d7037bd..29f50c0add 100644
--- a/synapse/replication/slave/storage/appservice.py
+++ b/synapse/replication/slave/storage/appservice.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/replication/slave/storage/client_ips.py b/synapse/replication/slave/storage/client_ips.py
index 0f5b7adef7..436d39c320 100644
--- a/synapse/replication/slave/storage/client_ips.py
+++ b/synapse/replication/slave/storage/client_ips.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,9 +23,9 @@ class SlavedClientIpStore(BaseSlavedStore):
     def __init__(self, database: DatabasePool, db_conn, hs):
         super().__init__(database, db_conn, hs)
 
-        self.client_ip_last_seen = LruCache(
-            cache_name="client_ip_last_seen", keylen=4, max_size=50000
-        )  # type: LruCache[tuple, int]
+        self.client_ip_last_seen: LruCache[tuple, int] = LruCache(
+            cache_name="client_ip_last_seen", max_size=50000
+        )
 
     async def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id):
         now = int(self._clock.time_msec())
diff --git a/synapse/replication/slave/storage/deviceinbox.py b/synapse/replication/slave/storage/deviceinbox.py
index 1260f6d141..e940751084 100644
--- a/synapse/replication/slave/storage/deviceinbox.py
+++ b/synapse/replication/slave/storage/deviceinbox.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/storage/devices.py b/synapse/replication/slave/storage/devices.py
index e0d86240dd..26bdead565 100644
--- a/synapse/replication/slave/storage/devices.py
+++ b/synapse/replication/slave/storage/devices.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -69,7 +68,7 @@ def _invalidate_caches_for_devices(self, token, rows):
             if row.entity.startswith("@"):
                 self._device_list_stream_cache.entity_has_changed(row.entity, token)
                 self.get_cached_devices_for_user.invalidate((row.entity,))
-                self._get_cached_user_device.invalidate_many((row.entity,))
+                self._get_cached_user_device.invalidate((row.entity,))
                 self.get_device_list_last_stream_id_for_remote.invalidate((row.entity,))
 
             else:
diff --git a/synapse/replication/slave/storage/directory.py b/synapse/replication/slave/storage/directory.py
index 1945bcf9a8..71fde0c96c 100644
--- a/synapse/replication/slave/storage/directory.py
+++ b/synapse/replication/slave/storage/directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py
index fbffe6d85c..d4d3f8c448 100644
--- a/synapse/replication/slave/storage/events.py
+++ b/synapse/replication/slave/storage/events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/replication/slave/storage/filtering.py b/synapse/replication/slave/storage/filtering.py
index 6a23252861..37875bc973 100644
--- a/synapse/replication/slave/storage/filtering.py
+++ b/synapse/replication/slave/storage/filtering.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/storage/groups.py b/synapse/replication/slave/storage/groups.py
index 30955bcbfe..e9bdc38470 100644
--- a/synapse/replication/slave/storage/groups.py
+++ b/synapse/replication/slave/storage/groups.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/storage/keys.py b/synapse/replication/slave/storage/keys.py
index 961579751c..a00b38c512 100644
--- a/synapse/replication/slave/storage/keys.py
+++ b/synapse/replication/slave/storage/keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/storage/presence.py b/synapse/replication/slave/storage/presence.py
deleted file mode 100644
index 55620c03d8..0000000000
--- a/synapse/replication/slave/storage/presence.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.replication.tcp.streams import PresenceStream
-from synapse.storage import DataStore
-from synapse.storage.database import DatabasePool
-from synapse.storage.databases.main.presence import PresenceStore
-from synapse.util.caches.stream_change_cache import StreamChangeCache
-
-from ._base import BaseSlavedStore
-from ._slaved_id_tracker import SlavedIdTracker
-
-
-class SlavedPresenceStore(BaseSlavedStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
-        super().__init__(database, db_conn, hs)
-        self._presence_id_gen = SlavedIdTracker(db_conn, "presence_stream", "stream_id")
-
-        self._presence_on_startup = self._get_active_presence(db_conn)  # type: ignore
-
-        self.presence_stream_cache = StreamChangeCache(
-            "PresenceStreamChangeCache", self._presence_id_gen.get_current_token()
-        )
-
-    _get_active_presence = DataStore._get_active_presence
-    take_presence_startup_info = DataStore.take_presence_startup_info
-    _get_presence_for_user = PresenceStore.__dict__["_get_presence_for_user"]
-    get_presence_for_users = PresenceStore.__dict__["get_presence_for_users"]
-
-    def get_current_presence_token(self):
-        return self._presence_id_gen.get_current_token()
-
-    def process_replication_rows(self, stream_name, instance_name, token, rows):
-        if stream_name == PresenceStream.NAME:
-            self._presence_id_gen.advance(instance_name, token)
-            for row in rows:
-                self.presence_stream_cache.entity_has_changed(row.user_id, token)
-                self._get_presence_for_user.invalidate((row.user_id,))
-        return super().process_replication_rows(stream_name, instance_name, token, rows)
diff --git a/synapse/replication/slave/storage/profile.py b/synapse/replication/slave/storage/profile.py
index f85b20a071..99f4a22642 100644
--- a/synapse/replication/slave/storage/profile.py
+++ b/synapse/replication/slave/storage/profile.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/storage/push_rule.py b/synapse/replication/slave/storage/push_rule.py
index de904c943c..4d5f862862 100644
--- a/synapse/replication/slave/storage/push_rule.py
+++ b/synapse/replication/slave/storage/push_rule.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/replication/slave/storage/pushers.py b/synapse/replication/slave/storage/pushers.py
index 93161c3dfb..2672a2c94b 100644
--- a/synapse/replication/slave/storage/pushers.py
+++ b/synapse/replication/slave/storage/pushers.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/replication/slave/storage/receipts.py b/synapse/replication/slave/storage/receipts.py
index 3dfdd9961d..3826b87dec 100644
--- a/synapse/replication/slave/storage/receipts.py
+++ b/synapse/replication/slave/storage/receipts.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/replication/slave/storage/registration.py b/synapse/replication/slave/storage/registration.py
index a40f064e2b..5dae35a960 100644
--- a/synapse/replication/slave/storage/registration.py
+++ b/synapse/replication/slave/storage/registration.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/slave/storage/room.py b/synapse/replication/slave/storage/room.py
deleted file mode 100644
index 109ac6bea1..0000000000
--- a/synapse/replication/slave/storage/room.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015, 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.replication.tcp.streams import PublicRoomsStream
-from synapse.storage.database import DatabasePool
-from synapse.storage.databases.main.room import RoomWorkerStore
-
-from ._base import BaseSlavedStore
-from ._slaved_id_tracker import SlavedIdTracker
-
-
-class RoomStore(RoomWorkerStore, BaseSlavedStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
-        super().__init__(database, db_conn, hs)
-        self._public_room_id_gen = SlavedIdTracker(
-            db_conn, "public_room_list_stream", "stream_id"
-        )
-
-    def get_current_public_room_stream_id(self):
-        return self._public_room_id_gen.get_current_token()
-
-    def process_replication_rows(self, stream_name, instance_name, token, rows):
-        if stream_name == PublicRoomsStream.NAME:
-            self._public_room_id_gen.advance(instance_name, token)
-
-        return super().process_replication_rows(stream_name, instance_name, token, rows)
diff --git a/synapse/replication/tcp/__init__.py b/synapse/replication/tcp/__init__.py
index 1b8718b11d..1fa60af8e6 100644
--- a/synapse/replication/tcp/__init__.py
+++ b/synapse/replication/tcp/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py
index 3455839d67..3fd2811713 100644
--- a/synapse/replication/tcp/client.py
+++ b/synapse/replication/tcp/client.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,22 +14,35 @@
 """A replication client for use by synapse workers.
 """
 import logging
-from typing import TYPE_CHECKING, Dict, List, Tuple
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
 
 from twisted.internet.defer import Deferred
 from twisted.internet.protocol import ReconnectingClientFactory
 
 from synapse.api.constants import EventTypes
+from synapse.federation import send_queue
+from synapse.federation.sender import FederationSender
 from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.tcp.protocol import ClientReplicationStreamProtocol
-from synapse.replication.tcp.streams import TypingStream
+from synapse.replication.tcp.streams import (
+    AccountDataStream,
+    DeviceListsStream,
+    GroupServerStream,
+    PushersStream,
+    PushRulesStream,
+    ReceiptsStream,
+    TagAccountDataStream,
+    ToDeviceStream,
+    TypingStream,
+)
 from synapse.replication.tcp.streams.events import (
     EventsStream,
     EventsStreamEventRow,
     EventsStreamRow,
 )
-from synapse.types import PersistedEventPosition, UserID
-from synapse.util.async_helpers import timeout_deferred
+from synapse.types import PersistedEventPosition, ReadReceipt, UserID
+from synapse.util.async_helpers import Linearizer, timeout_deferred
 from synapse.util.metrics import Measure
 
 if TYPE_CHECKING:
@@ -39,7 +51,6 @@
 
 logger = logging.getLogger(__name__)
 
-
 # How long we allow callers to wait for replication updates before timing out.
 _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS = 30
 
@@ -106,9 +117,17 @@ def __init__(self, hs: "HomeServer"):
         self._instance_name = hs.get_instance_name()
         self._typing_handler = hs.get_typing_handler()
 
+        self._notify_pushers = hs.config.start_pushers
+        self._pusher_pool = hs.get_pusherpool()
+        self._presence_handler = hs.get_presence_handler()
+
+        self.send_handler: Optional[FederationSenderHandler] = None
+        if hs.should_send_federation():
+            self.send_handler = FederationSenderHandler(hs)
+
         # Map from stream to list of deferreds waiting for the stream to
         # arrive at a particular position. The lists are sorted by stream position.
-        self._streams_to_waiters = {}  # type: Dict[str, List[Tuple[int, Deferred]]]
+        self._streams_to_waiters: Dict[str, List[Tuple[int, Deferred]]] = {}
 
     async def on_rdata(
         self, stream_name: str, instance_name: str, token: int, rows: list
@@ -126,13 +145,51 @@ async def on_rdata(
         """
         self.store.process_replication_rows(stream_name, instance_name, token, rows)
 
+        if self.send_handler:
+            await self.send_handler.process_replication_rows(stream_name, token, rows)
+
         if stream_name == TypingStream.NAME:
             self._typing_handler.process_replication_rows(token, rows)
             self.notifier.on_new_event(
                 "typing_key", token, rooms=[row.room_id for row in rows]
             )
-
-        if stream_name == EventsStream.NAME:
+        elif stream_name == PushRulesStream.NAME:
+            self.notifier.on_new_event(
+                "push_rules_key", token, users=[row.user_id for row in rows]
+            )
+        elif stream_name in (AccountDataStream.NAME, TagAccountDataStream.NAME):
+            self.notifier.on_new_event(
+                "account_data_key", token, users=[row.user_id for row in rows]
+            )
+        elif stream_name == ReceiptsStream.NAME:
+            self.notifier.on_new_event(
+                "receipt_key", token, rooms=[row.room_id for row in rows]
+            )
+            await self._pusher_pool.on_new_receipts(
+                token, token, {row.room_id for row in rows}
+            )
+        elif stream_name == ToDeviceStream.NAME:
+            entities = [row.entity for row in rows if row.entity.startswith("@")]
+            if entities:
+                self.notifier.on_new_event("to_device_key", token, users=entities)
+        elif stream_name == DeviceListsStream.NAME:
+            all_room_ids: Set[str] = set()
+            for row in rows:
+                if row.entity.startswith("@"):
+                    room_ids = await self.store.get_rooms_for_user(row.entity)
+                    all_room_ids.update(room_ids)
+            self.notifier.on_new_event("device_list_key", token, rooms=all_room_ids)
+        elif stream_name == GroupServerStream.NAME:
+            self.notifier.on_new_event(
+                "groups_key", token, users=[row.user_id for row in rows]
+            )
+        elif stream_name == PushersStream.NAME:
+            for row in rows:
+                if row.deleted:
+                    self.stop_pusher(row.user_id, row.app_id, row.pushkey)
+                else:
+                    await self.start_pusher(row.user_id, row.app_id, row.pushkey)
+        elif stream_name == EventsStream.NAME:
             # We shouldn't get multiple rows per token for events stream, so
             # we don't need to optimise this for multiple rows.
             for row in rows:
@@ -144,7 +201,7 @@ async def on_rdata(
                 if row.data.rejected:
                     continue
 
-                extra_users = ()  # type: Tuple[UserID, ...]
+                extra_users: Tuple[UserID, ...] = ()
                 if row.data.type == EventTypes.Member and row.data.state_key:
                     extra_users = (UserID.from_string(row.data.state_key),)
 
@@ -160,6 +217,10 @@ async def on_rdata(
                     membership=row.data.membership,
                 )
 
+        await self._presence_handler.process_replication_rows(
+            stream_name, instance_name, token, rows
+        )
+
         # Notify any waiting deferreds. The list is ordered by position so we
         # just iterate through the list until we reach a position that is
         # greater than the received row position.
@@ -191,7 +252,7 @@ async def on_rdata(
         waiting_list[:] = waiting_list[index_of_first_deferred_not_called:]
 
     async def on_position(self, stream_name: str, instance_name: str, token: int):
-        self.store.process_replication_rows(stream_name, instance_name, token, [])
+        await self.on_rdata(stream_name, instance_name, token, [])
 
         # We poke the generic "replication" notifier to wake anything up that
         # may be streaming.
@@ -200,6 +261,11 @@ async def on_position(self, stream_name: str, instance_name: str, token: int):
     def on_remote_server_up(self, server: str):
         """Called when get a new REMOTE_SERVER_UP command."""
 
+        # Let's wake up the transaction queue for the server in case we have
+        # pending stuff to send to it.
+        if self.send_handler:
+            self.send_handler.wake_destination(server)
+
     async def wait_for_stream_position(
         self, instance_name: str, stream_name: str, position: int
     ):
@@ -219,7 +285,7 @@ async def wait_for_stream_position(
 
         # Create a new deferred that times out after N seconds, as we don't want
         # to wedge here forever.
-        deferred = Deferred()
+        deferred: "Deferred[None]" = Deferred()
         deferred = timeout_deferred(
             deferred, _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, self._reactor
         )
@@ -236,3 +302,158 @@ async def wait_for_stream_position(
             logger.info(
                 "Finished waiting for repl stream %r to reach %s", stream_name, position
             )
+
+    def stop_pusher(self, user_id, app_id, pushkey):
+        if not self._notify_pushers:
+            return
+
+        key = "%s:%s" % (app_id, pushkey)
+        pushers_for_user = self._pusher_pool.pushers.get(user_id, {})
+        pusher = pushers_for_user.pop(key, None)
+        if pusher is None:
+            return
+        logger.info("Stopping pusher %r / %r", user_id, key)
+        pusher.on_stop()
+
+    async def start_pusher(self, user_id, app_id, pushkey):
+        if not self._notify_pushers:
+            return
+
+        key = "%s:%s" % (app_id, pushkey)
+        logger.info("Starting pusher %r / %r", user_id, key)
+        return await self._pusher_pool.start_pusher_by_id(app_id, pushkey, user_id)
+
+
+class FederationSenderHandler:
+    """Processes the fedration replication stream
+
+    This class is only instantiate on the worker responsible for sending outbound
+    federation transactions. It receives rows from the replication stream and forwards
+    the appropriate entries to the FederationSender class.
+    """
+
+    def __init__(self, hs: "HomeServer"):
+        assert hs.should_send_federation()
+
+        self.store = hs.get_datastore()
+        self._is_mine_id = hs.is_mine_id
+        self._hs = hs
+
+        # We need to make a temporary value to ensure that mypy picks up the
+        # right type. We know we should have a federation sender instance since
+        # `should_send_federation` is True.
+        sender = hs.get_federation_sender()
+        assert isinstance(sender, FederationSender)
+        self.federation_sender = sender
+
+        # Stores the latest position in the federation stream we've gotten up
+        # to. This is always set before we use it.
+        self.federation_position: Optional[int] = None
+
+        self._fed_position_linearizer = Linearizer(name="_fed_position_linearizer")
+
+    def wake_destination(self, server: str):
+        self.federation_sender.wake_destination(server)
+
+    async def process_replication_rows(self, stream_name, token, rows):
+        # The federation stream contains things that we want to send out, e.g.
+        # presence, typing, etc.
+        if stream_name == "federation":
+            send_queue.process_rows_for_federation(self.federation_sender, rows)
+            await self.update_token(token)
+
+        # ... and when new receipts happen
+        elif stream_name == ReceiptsStream.NAME:
+            await self._on_new_receipts(rows)
+
+        # ... as well as device updates and messages
+        elif stream_name == DeviceListsStream.NAME:
+            # The entities are either user IDs (starting with '@') whose devices
+            # have changed, or remote servers that we need to tell about
+            # changes.
+            hosts = {row.entity for row in rows if not row.entity.startswith("@")}
+            for host in hosts:
+                self.federation_sender.send_device_messages(host)
+
+        elif stream_name == ToDeviceStream.NAME:
+            # The to_device stream includes stuff to be pushed to both local
+            # clients and remote servers, so we ignore entities that start with
+            # '@' (since they'll be local users rather than destinations).
+            hosts = {row.entity for row in rows if not row.entity.startswith("@")}
+            for host in hosts:
+                self.federation_sender.send_device_messages(host)
+
+    async def _on_new_receipts(self, rows):
+        """
+        Args:
+            rows (Iterable[synapse.replication.tcp.streams.ReceiptsStream.ReceiptsStreamRow]):
+                new receipts to be processed
+        """
+        for receipt in rows:
+            # we only want to send on receipts for our own users
+            if not self._is_mine_id(receipt.user_id):
+                continue
+            if (
+                receipt.data.get("hidden", False)
+                and self._hs.config.experimental.msc2285_enabled
+            ):
+                continue
+            receipt_info = ReadReceipt(
+                receipt.room_id,
+                receipt.receipt_type,
+                receipt.user_id,
+                [receipt.event_id],
+                receipt.data,
+            )
+            await self.federation_sender.send_read_receipt(receipt_info)
+
+    async def update_token(self, token):
+        """Update the record of where we have processed to in the federation stream.
+
+        Called after we have processed a an update received over replication. Sends
+        a FEDERATION_ACK back to the master, and stores the token that we have processed
+         in `federation_stream_position` so that we can restart where we left off.
+        """
+        self.federation_position = token
+
+        # We save and send the ACK to master asynchronously, so we don't block
+        # processing on persistence. We don't need to do this operation for
+        # every single RDATA we receive, we just need to do it periodically.
+
+        if self._fed_position_linearizer.is_queued(None):
+            # There is already a task queued up to save and send the token, so
+            # no need to queue up another task.
+            return
+
+        run_as_background_process("_save_and_send_ack", self._save_and_send_ack)
+
+    async def _save_and_send_ack(self):
+        """Save the current federation position in the database and send an ACK
+        to master with where we're up to.
+        """
+        # We should only be calling this once we've got a token.
+        assert self.federation_position is not None
+
+        try:
+            # We linearize here to ensure we don't have races updating the token
+            #
+            # XXX this appears to be redundant, since the ReplicationCommandHandler
+            # has a linearizer which ensures that we only process one line of
+            # replication data at a time. Should we remove it, or is it doing useful
+            # service for robustness? Or could we replace it with an assertion that
+            # we're not being re-entered?
+
+            with (await self._fed_position_linearizer.queue(None)):
+                # We persist and ack the same position, so we take a copy of it
+                # here as otherwise it can get modified from underneath us.
+                current_position = self.federation_position
+
+                await self.store.update_federation_out_pos(
+                    "federation", current_position
+                )
+
+                # We ACK this token over replication so that the master can drop
+                # its in memory queues
+                self._hs.get_tcp_replication().send_federation_ack(current_position)
+        except Exception:
+            logger.exception("Error updating federation stream position")
diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py
index 8abed1f52d..1311b013da 100644
--- a/synapse/replication/tcp/commands.py
+++ b/synapse/replication/tcp/commands.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -35,7 +34,7 @@ class Command(metaclass=abc.ABCMeta):
     A full command line on the wire is constructed from `NAME + " " + to_line()`
     """
 
-    NAME = None  # type: str
+    NAME: str
 
     @classmethod
     @abc.abstractmethod
@@ -381,7 +380,7 @@ class RemoteServerUpCommand(_SimpleCommand):
     NAME = "REMOTE_SERVER_UP"
 
 
-_COMMANDS = (
+_COMMANDS: Tuple[Type[Command], ...] = (
     ServerCommand,
     RdataCommand,
     PositionCommand,
@@ -394,7 +393,7 @@ class RemoteServerUpCommand(_SimpleCommand):
     UserIpCommand,
     RemoteServerUpCommand,
     ClearUserSyncsCommand,
-)  # type: Tuple[Type[Command], ...]
+)
 
 # Map of command name to command type.
 COMMAND_MAP = {cmd.NAME: cmd for cmd in _COMMANDS}
diff --git a/synapse/replication/tcp/external_cache.py b/synapse/replication/tcp/external_cache.py
index d89a36f25a..b402f82810 100644
--- a/synapse/replication/tcp/external_cache.py
+++ b/synapse/replication/tcp/external_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,7 +15,7 @@
 import logging
 from typing import TYPE_CHECKING, Any, Optional
 
-from prometheus_client import Counter
+from prometheus_client import Counter, Histogram
 
 from synapse.logging.context import make_deferred_yieldable
 from synapse.util import json_decoder, json_encoder
@@ -36,6 +35,20 @@
     labelnames=["cache_name", "hit"],
 )
 
+response_timer = Histogram(
+    "synapse_external_cache_response_time_seconds",
+    "Time taken to get a response from Redis for a cache get/set request",
+    labelnames=["method"],
+    buckets=(
+        0.001,
+        0.002,
+        0.005,
+        0.01,
+        0.02,
+        0.05,
+    ),
+)
+
 
 logger = logging.getLogger(__name__)
 
@@ -73,13 +86,14 @@ async def set(self, cache_name: str, key: str, value: Any, expiry_ms: int) -> No
 
         logger.debug("Caching %s %s: %r", cache_name, key, encoded_value)
 
-        return await make_deferred_yieldable(
-            self._redis_connection.set(
-                self._get_redis_key(cache_name, key),
-                encoded_value,
-                pexpire=expiry_ms,
+        with response_timer.labels("set").time():
+            return await make_deferred_yieldable(
+                self._redis_connection.set(
+                    self._get_redis_key(cache_name, key),
+                    encoded_value,
+                    pexpire=expiry_ms,
+                )
             )
-        )
 
     async def get(self, cache_name: str, key: str) -> Optional[Any]:
         """Look up a key/value in the named cache."""
@@ -87,9 +101,10 @@ async def get(self, cache_name: str, key: str) -> Optional[Any]:
         if self._redis_connection is None:
             return None
 
-        result = await make_deferred_yieldable(
-            self._redis_connection.get(self._get_redis_key(cache_name, key))
-        )
+        with response_timer.labels("get").time():
+            result = await make_deferred_yieldable(
+                self._redis_connection.get(self._get_redis_key(cache_name, key))
+            )
 
         logger.debug("Got cache result %s %s: %r", cache_name, key, result)
 
diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py
index a8894beadf..eae4515363 100644
--- a/synapse/replication/tcp/handler.py
+++ b/synapse/replication/tcp/handler.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -56,6 +55,8 @@
     CachesStream,
     EventsStream,
     FederationStream,
+    PresenceFederationStream,
+    PresenceStream,
     ReceiptsStream,
     Stream,
     TagAccountDataStream,
@@ -100,12 +101,16 @@ def __init__(self, hs: "HomeServer"):
         self._instance_id = hs.get_instance_id()
         self._instance_name = hs.get_instance_name()
 
-        self._streams = {
+        self._is_presence_writer = (
+            hs.get_instance_name() in hs.config.worker.writers.presence
+        )
+
+        self._streams: Dict[str, Stream] = {
             stream.NAME: stream(hs) for stream in STREAMS_MAP.values()
-        }  # type: Dict[str, Stream]
+        }
 
         # List of streams that this instance is the source of
-        self._streams_to_replicate = []  # type: List[Stream]
+        self._streams_to_replicate: List[Stream] = []
 
         for stream in self._streams.values():
             if hs.config.redis.redis_enabled and stream.NAME == CachesStream.NAME:
@@ -154,6 +159,14 @@ def __init__(self, hs: "HomeServer"):
 
                 continue
 
+            if isinstance(stream, (PresenceStream, PresenceFederationStream)):
+                # Only add PresenceStream as a source on the instance in charge
+                # of presence.
+                if self._is_presence_writer:
+                    self._streams_to_replicate.append(stream)
+
+                continue
+
             # Only add any other streams if we're on master.
             if hs.config.worker_app is not None:
                 continue
@@ -167,14 +180,14 @@ def __init__(self, hs: "HomeServer"):
 
         # Map of stream name to batched updates. See RdataCommand for info on
         # how batching works.
-        self._pending_batches = {}  # type: Dict[str, List[Any]]
+        self._pending_batches: Dict[str, List[Any]] = {}
 
         # The factory used to create connections.
-        self._factory = None  # type: Optional[ReconnectingClientFactory]
+        self._factory: Optional[ReconnectingClientFactory] = None
 
         # The currently connected connections. (The list of places we need to send
         # outgoing replication commands to.)
-        self._connections = []  # type: List[IReplicationConnection]
+        self._connections: List[IReplicationConnection] = []
 
         LaterGauge(
             "synapse_replication_tcp_resource_total_connections",
@@ -187,7 +200,7 @@ def __init__(self, hs: "HomeServer"):
         # them in order in a separate background process.
 
         # the streams which are currently being processed by _unsafe_process_queue
-        self._processing_streams = set()  # type: Set[str]
+        self._processing_streams: Set[str] = set()
 
         # for each stream, a queue of commands that are awaiting processing, and the
         # connection that they arrived on.
@@ -197,7 +210,7 @@ def __init__(self, hs: "HomeServer"):
 
         # For each connection, the incoming stream names that have received a POSITION
         # from that connection.
-        self._streams_by_connection = {}  # type: Dict[IReplicationConnection, Set[str]]
+        self._streams_by_connection: Dict[IReplicationConnection, Set[str]] = {}
 
         LaterGauge(
             "synapse_replication_tcp_command_queue",
@@ -351,7 +364,7 @@ def on_USER_SYNC(
     ) -> Optional[Awaitable[None]]:
         user_sync_counter.inc()
 
-        if self._is_master:
+        if self._is_presence_writer:
             return self._presence_handler.update_external_syncs_row(
                 cmd.instance_id, cmd.user_id, cmd.is_syncing, cmd.last_sync_ms
             )
@@ -361,7 +374,7 @@ def on_USER_SYNC(
     def on_CLEAR_USER_SYNC(
         self, conn: IReplicationConnection, cmd: ClearUserSyncsCommand
     ) -> Optional[Awaitable[None]]:
-        if self._is_master:
+        if self._is_presence_writer:
             return self._presence_handler.update_external_syncs_clear(cmd.instance_id)
         else:
             return None
@@ -558,7 +571,7 @@ async def _process_position(
     def on_REMOTE_SERVER_UP(
         self, conn: IReplicationConnection, cmd: RemoteServerUpCommand
     ):
-        """"Called when get a new REMOTE_SERVER_UP command."""
+        """Called when get a new REMOTE_SERVER_UP command."""
         self._replication_data_handler.on_remote_server_up(cmd.data)
 
         self._notifier.notify_remote_server_up(cmd.data)
diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index e829add257..8c80153ab6 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -50,7 +49,7 @@
 import logging
 import struct
 from inspect import isawaitable
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Collection, List, Optional
 
 from prometheus_client import Counter
 from zope.interface import Interface, implementer
@@ -77,7 +76,6 @@
     ServerCommand,
     parse_command_from_line,
 )
-from synapse.types import Collection
 from synapse.util import Clock
 from synapse.util.stringutils import random_string
 
@@ -104,7 +102,7 @@
 
 # A list of all connected protocols. This allows us to send metrics about the
 # connections.
-connected_connections = []  # type: List[BaseReplicationStreamProtocol]
+connected_connections: "List[BaseReplicationStreamProtocol]" = []
 
 
 logger = logging.getLogger(__name__)
@@ -148,15 +146,15 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
 
     # The transport is going to be an ITCPTransport, but that doesn't have the
     # (un)registerProducer methods, those are only on the implementation.
-    transport = None  # type: Connection
+    transport: Connection
 
     delimiter = b"\n"
 
     # Valid commands we expect to receive
-    VALID_INBOUND_COMMANDS = []  # type: Collection[str]
+    VALID_INBOUND_COMMANDS: Collection[str] = []
 
     # Valid commands we can send
-    VALID_OUTBOUND_COMMANDS = []  # type: Collection[str]
+    VALID_OUTBOUND_COMMANDS: Collection[str] = []
 
     max_line_buffer = 10000
 
@@ -167,7 +165,7 @@ def __init__(self, clock: Clock, handler: "ReplicationCommandHandler"):
         self.last_received_command = self.clock.time_msec()
         self.last_sent_command = 0
         # When we requested the connection be closed
-        self.time_we_closed = None  # type: Optional[int]
+        self.time_we_closed: Optional[int] = None
 
         self.received_ping = False  # Have we received a ping from the other side
 
@@ -177,15 +175,16 @@ def __init__(self, clock: Clock, handler: "ReplicationCommandHandler"):
         self.conn_id = random_string(5)  # To dedupe in case of name clashes.
 
         # List of pending commands to send once we've established the connection
-        self.pending_commands = []  # type: List[Command]
+        self.pending_commands: List[Command] = []
 
         # The LoopingCall for sending pings.
-        self._send_ping_loop = None  # type: Optional[task.LoopingCall]
+        self._send_ping_loop: Optional[task.LoopingCall] = None
 
         # a logcontext which we use for processing incoming commands. We declare it as a
         # background process so that the CPU stats get reported to prometheus.
-        ctx_name = "replication-conn-%s" % self.conn_id
-        self._logging_context = BackgroundProcessLoggingContext(ctx_name, ctx_name)
+        self._logging_context = BackgroundProcessLoggingContext(
+            "replication-conn", self.conn_id
+        )
 
     def connectionMade(self):
         logger.info("[%s] Connection established", self.id())
diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py
index 2f4d407f94..8c0df627c8 100644
--- a/synapse/replication/tcp/redis.py
+++ b/synapse/replication/tcp/redis.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -58,9 +57,9 @@ class ConstantProperty(Generic[T, V]):
     it.
     """
 
-    constant = attr.ib()  # type: V
+    constant: V = attr.ib()
 
-    def __get__(self, obj: Optional[T], objtype: Type[T] = None) -> V:
+    def __get__(self, obj: Optional[T], objtype: Optional[Type[T]] = None) -> V:
         return self.constant
 
     def __set__(self, obj: Optional[T], value: V):
@@ -92,9 +91,9 @@ class RedisSubscriber(txredisapi.SubscriberProtocol):
             commands.
     """
 
-    synapse_handler = None  # type: ReplicationCommandHandler
-    synapse_stream_name = None  # type: str
-    synapse_outbound_redis_connection = None  # type: txredisapi.RedisProtocol
+    synapse_handler: "ReplicationCommandHandler"
+    synapse_stream_name: str
+    synapse_outbound_redis_connection: txredisapi.RedisProtocol
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py
index 2018f9f29e..bd47d84258 100644
--- a/synapse/replication/tcp/resource.py
+++ b/synapse/replication/tcp/resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py
index d1a61c3314..f41eabd85e 100644
--- a/synapse/replication/tcp/streams/__init__.py
+++ b/synapse/replication/tcp/streams/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2019 New Vector Ltd
 #
@@ -31,8 +30,8 @@
     CachesStream,
     DeviceListsStream,
     GroupServerStream,
+    PresenceFederationStream,
     PresenceStream,
-    PublicRoomsStream,
     PushersStream,
     PushRulesStream,
     ReceiptsStream,
@@ -51,12 +50,12 @@
         EventsStream,
         BackfillStream,
         PresenceStream,
+        PresenceFederationStream,
         TypingStream,
         ReceiptsStream,
         PushRulesStream,
         PushersStream,
         CachesStream,
-        PublicRoomsStream,
         DeviceListsStream,
         ToDeviceStream,
         FederationStream,
@@ -72,12 +71,12 @@
     "Stream",
     "BackfillStream",
     "PresenceStream",
+    "PresenceFederationStream",
     "TypingStream",
     "ReceiptsStream",
     "PushRulesStream",
     "PushersStream",
     "CachesStream",
-    "PublicRoomsStream",
     "DeviceListsStream",
     "ToDeviceStream",
     "TagAccountDataStream",
diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py
index 3dfee76743..9b905aba9d 100644
--- a/synapse/replication/tcp/streams/_base.py
+++ b/synapse/replication/tcp/streams/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2019 New Vector Ltd
 #
@@ -86,9 +85,9 @@ class Stream:
     time it was called.
     """
 
-    NAME = None  # type: str  # The name of the stream
+    NAME: str  # The name of the stream
     # The type of the row. Used by the default impl of parse_row.
-    ROW_TYPE = None  # type: Any
+    ROW_TYPE: Any = None
 
     @classmethod
     def parse_row(cls, row: StreamRow):
@@ -273,15 +272,20 @@ class PresenceStream(Stream):
     NAME = "presence"
     ROW_TYPE = PresenceStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         store = hs.get_datastore()
 
-        if hs.config.worker_app is None:
-            # on the master, query the presence handler
+        if hs.get_instance_name() in hs.config.worker.writers.presence:
+            # on the presence writer, query the presence handler
             presence_handler = hs.get_presence_handler()
-            update_function = presence_handler.get_all_presence_updates
+
+            from synapse.handlers.presence import PresenceHandler
+
+            assert isinstance(presence_handler, PresenceHandler)
+
+            update_function: UpdateFunction = presence_handler.get_all_presence_updates
         else:
-            # Query master process
+            # Query presence writer process
             update_function = make_http_update_function(hs, self.NAME)
 
         super().__init__(
@@ -291,6 +295,30 @@ def __init__(self, hs):
         )
 
 
+class PresenceFederationStream(Stream):
+    """A stream used to send ad hoc presence updates over federation.
+
+    Streams the remote destination and the user ID of the presence state to
+    send.
+    """
+
+    @attr.s(slots=True, auto_attribs=True)
+    class PresenceFederationStreamRow:
+        destination: str
+        user_id: str
+
+    NAME = "presence_federation"
+    ROW_TYPE = PresenceFederationStreamRow
+
+    def __init__(self, hs: "HomeServer"):
+        federation_queue = hs.get_presence_handler().get_federation_queue()
+        super().__init__(
+            hs.get_instance_name(),
+            federation_queue.get_current_token,
+            federation_queue.get_replication_rows,
+        )
+
+
 class TypingStream(Stream):
     TypingStreamRow = namedtuple(
         "TypingStreamRow", ("room_id", "user_ids")  # str  # list(str)
@@ -304,9 +332,9 @@ def __init__(self, hs: "HomeServer"):
         if writer_instance == hs.get_instance_name():
             # On the writer, query the typing handler
             typing_writer_handler = hs.get_typing_writer_handler()
-            update_function = (
-                typing_writer_handler.get_all_typing_updates
-            )  # type: Callable[[str, int, int, int], Awaitable[Tuple[List[Tuple[int, Any]], int, bool]]]
+            update_function: Callable[
+                [str, int, int, int], Awaitable[Tuple[List[Tuple[int, Any]], int, bool]]
+            ] = typing_writer_handler.get_all_typing_updates
             current_token_function = typing_writer_handler.get_current_token
         else:
             # Query the typing writer process
@@ -419,31 +447,6 @@ def __init__(self, hs):
         )
 
 
-class PublicRoomsStream(Stream):
-    """The public rooms list changed"""
-
-    PublicRoomsStreamRow = namedtuple(
-        "PublicRoomsStreamRow",
-        (
-            "room_id",  # str
-            "visibility",  # str
-            "appservice_id",  # str, optional
-            "network_id",  # str, optional
-        ),
-    )
-
-    NAME = "public_rooms"
-    ROW_TYPE = PublicRoomsStreamRow
-
-    def __init__(self, hs):
-        store = hs.get_datastore()
-        super().__init__(
-            hs.get_instance_name(),
-            current_token_without_instance(store.get_current_public_room_stream_id),
-            store.get_all_new_public_rooms,
-        )
-
-
 class DeviceListsStream(Stream):
     """Either a user has updated their devices or a remote server needs to be
     told about a device update.
diff --git a/synapse/replication/tcp/streams/events.py b/synapse/replication/tcp/streams/events.py
index fa5e37ba7b..a030e9299e 100644
--- a/synapse/replication/tcp/streams/events.py
+++ b/synapse/replication/tcp/streams/events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2019 New Vector Ltd
 #
@@ -66,7 +65,7 @@ class BaseEventsStreamRow:
     """
 
     # Unique string that ids the type. Must be overridden in sub classes.
-    TypeId = None  # type: str
+    TypeId: str
 
     @classmethod
     def from_data(cls, data):
@@ -104,10 +103,10 @@ class EventsStreamCurrentStateRow(BaseEventsStreamRow):
     event_id = attr.ib()  # str, optional
 
 
-_EventRows = (
+_EventRows: Tuple[Type[BaseEventsStreamRow], ...] = (
     EventsStreamEventRow,
     EventsStreamCurrentStateRow,
-)  # type: Tuple[Type[BaseEventsStreamRow], ...]
+)
 
 TypeToRow = {Row.TypeId: Row for Row in _EventRows}
 
@@ -158,9 +157,9 @@ async def _update_function(
 
         # now we fetch up to that many rows from the events table
 
-        event_rows = await self._store.get_all_new_forward_event_rows(
+        event_rows: List[Tuple] = await self._store.get_all_new_forward_event_rows(
             instance_name, from_token, current_token, target_row_count
-        )  # type: List[Tuple]
+        )
 
         # we rely on get_all_new_forward_event_rows strictly honouring the limit, so
         # that we know it is safe to just take upper_limit = event_rows[-1][0].
@@ -173,7 +172,7 @@ async def _update_function(
 
         if len(event_rows) == target_row_count:
             limited = True
-            upper_limit = event_rows[-1][0]  # type: int
+            upper_limit: int = event_rows[-1][0]
         else:
             limited = False
             upper_limit = current_token
@@ -192,30 +191,30 @@ async def _update_function(
         # finally, fetch the ex-outliers rows. We assume there are few enough of these
         # not to bother with the limit.
 
-        ex_outliers_rows = await self._store.get_ex_outlier_stream_rows(
+        ex_outliers_rows: List[Tuple] = await self._store.get_ex_outlier_stream_rows(
             instance_name, from_token, upper_limit
-        )  # type: List[Tuple]
+        )
 
         # we now need to turn the raw database rows returned into tuples suitable
         # for the replication protocol (basically, we add an identifier to
         # distinguish the row type). At the same time, we can limit the event_rows
         # to the max stream_id from state_rows.
 
-        event_updates = (
+        event_updates: Iterable[Tuple[int, Tuple]] = (
             (stream_id, (EventsStreamEventRow.TypeId, rest))
             for (stream_id, *rest) in event_rows
             if stream_id <= upper_limit
-        )  # type: Iterable[Tuple[int, Tuple]]
+        )
 
-        state_updates = (
+        state_updates: Iterable[Tuple[int, Tuple]] = (
             (stream_id, (EventsStreamCurrentStateRow.TypeId, rest))
             for (stream_id, *rest) in state_rows
-        )  # type: Iterable[Tuple[int, Tuple]]
+        )
 
-        ex_outliers_updates = (
+        ex_outliers_updates: Iterable[Tuple[int, Tuple]] = (
             (stream_id, (EventsStreamEventRow.TypeId, rest))
             for (stream_id, *rest) in ex_outliers_rows
-        )  # type: Iterable[Tuple[int, Tuple]]
+        )
 
         # we need to return a sorted list, so merge them together.
         updates = list(heapq.merge(event_updates, state_updates, ex_outliers_updates))
diff --git a/synapse/replication/tcp/streams/federation.py b/synapse/replication/tcp/streams/federation.py
index 9bb8e9e177..c445af9bd9 100644
--- a/synapse/replication/tcp/streams/federation.py
+++ b/synapse/replication/tcp/streams/federation.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2019 New Vector Ltd
 #
@@ -52,9 +51,9 @@ def __init__(self, hs: "HomeServer"):
             current_token = current_token_without_instance(
                 federation_sender.get_current_token
             )
-            update_function = (
-                federation_sender.get_replication_rows
-            )  # type: Callable[[str, int, int, int], Awaitable[Tuple[List[Tuple[int, Any]], int, bool]]]
+            update_function: Callable[
+                [str, int, int, int], Awaitable[Tuple[List[Tuple[int, Any]], int, bool]]
+            ] = federation_sender.get_replication_rows
 
         elif hs.should_send_federation():
             # federation sender: Query master process
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index ee3a9af569..0c756e574e 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -15,39 +14,36 @@
 # limitations under the License.
 from synapse.http.server import JsonResource
 from synapse.rest import admin
-from synapse.rest.client import versions
-from synapse.rest.client.v1 import (
-    directory,
-    events,
-    initial_sync,
-    login as v1_login,
-    logout,
-    presence,
-    profile,
-    push_rule,
-    pusher,
-    room,
-    voip,
-)
-from synapse.rest.client.v2_alpha import (
+from synapse.rest.client import (
     account,
     account_data,
     account_validity,
     auth,
     capabilities,
     devices,
+    directory,
+    events,
     filter,
     groups,
+    initial_sync,
     keys,
     knock,
+    login as v1_login,
+    logout,
     notifications,
     openid,
     password_policy,
+    presence,
+    profile,
+    push_rule,
+    pusher,
     read_marker,
     receipts,
     register,
     relations,
     report_event,
+    room,
+    room_batch,
     room_keys,
     room_upgrade_rest_servlet,
     sendtodevice,
@@ -57,6 +53,8 @@
     thirdparty,
     tokenrefresh,
     user_directory,
+    versions,
+    voip,
 )
 
 
@@ -85,7 +83,6 @@ def register_servlets(client_resource, hs):
         # Partially deprecated in r0
         events.register_servlets(hs, client_resource)
 
-        # "v1" + "r0"
         room.register_servlets(hs, client_resource)
         v1_login.register_servlets(hs, client_resource)
         profile.register_servlets(hs, client_resource)
@@ -95,8 +92,6 @@ def register_servlets(client_resource, hs):
         pusher.register_servlets(hs, client_resource)
         push_rule.register_servlets(hs, client_resource)
         logout.register_servlets(hs, client_resource)
-
-        # "v2"
         sync.register_servlets(hs, client_resource)
         filter.register_servlets(hs, client_resource)
         account.register_servlets(hs, client_resource)
@@ -118,6 +113,7 @@ def register_servlets(client_resource, hs):
         user_directory.register_servlets(hs, client_resource)
         groups.register_servlets(hs, client_resource)
         room_upgrade_rest_servlet.register_servlets(hs, client_resource)
+        room_batch.register_servlets(hs, client_resource)
         capabilities.register_servlets(hs, client_resource)
         account_validity.register_servlets(hs, client_resource)
         password_policy.register_servlets(hs, client_resource)
diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py
index 8457db1e22..d5862a4da4 100644
--- a/synapse/rest/admin/__init__.py
+++ b/synapse/rest/admin/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018-2019 New Vector Ltd
 # Copyright 2020, 2021 The Matrix.org Foundation C.I.C.
@@ -18,11 +17,13 @@
 
 import logging
 import platform
+from typing import TYPE_CHECKING, Optional, Tuple
 
 import synapse
 from synapse.api.errors import Codes, NotFoundError, SynapseError
-from synapse.http.server import JsonResource
+from synapse.http.server import HttpServer, JsonResource
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.http.site import SynapseRequest
 from synapse.rest.admin._base import admin_patterns, assert_requester_is_admin
 from synapse.rest.admin.devices import (
     DeleteDevicesRestServlet,
@@ -50,39 +51,42 @@
 )
 from synapse.rest.admin.server_notice_servlet import SendServerNoticeServlet
 from synapse.rest.admin.statistics import UserMediaStatisticsRestServlet
+from synapse.rest.admin.username_available import UsernameAvailableRestServlet
 from synapse.rest.admin.users import (
     AccountValidityRenewServlet,
     DeactivateAccountRestServlet,
     PushersRestServlet,
+    RateLimitRestServlet,
     ResetPasswordRestServlet,
     SearchUsersRestServlet,
     ShadowBanRestServlet,
     UserAdminServlet,
-    UserMediaRestServlet,
     UserMembershipRestServlet,
     UserRegisterServlet,
     UserRestServletV2,
-    UsersRestServlet,
     UsersRestServletV2,
     UserTokenRestServlet,
     WhoisRestServlet,
 )
-from synapse.types import RoomStreamToken
+from synapse.types import JsonDict, RoomStreamToken
 from synapse.util.versionstring import get_version_string
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
 class VersionServlet(RestServlet):
     PATTERNS = admin_patterns("/server_version$")
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.res = {
             "server_version": get_version_string(synapse),
             "python_version": platform.python_version(),
         }
 
-    def on_GET(self, request):
+    def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
         return 200, self.res
 
 
@@ -91,17 +95,14 @@ class PurgeHistoryRestServlet(RestServlet):
         "/purge_history/(?P<room_id>[^/]*)(/(?P<event_id>[^/]+))?"
     )
 
-    def __init__(self, hs):
-        """
-
-        Args:
-            hs (synapse.server.HomeServer)
-        """
+    def __init__(self, hs: "HomeServer"):
         self.pagination_handler = hs.get_pagination_handler()
         self.store = hs.get_datastore()
         self.auth = hs.get_auth()
 
-    async def on_POST(self, request, room_id, event_id):
+    async def on_POST(
+        self, request: SynapseRequest, room_id: str, event_id: Optional[str]
+    ) -> Tuple[int, JsonDict]:
         await assert_requester_is_admin(self.auth, request)
 
         body = parse_json_object_from_request(request, allow_empty_body=True)
@@ -120,6 +121,8 @@ async def on_POST(self, request, room_id, event_id):
             if event.room_id != room_id:
                 raise SynapseError(400, "Event is for wrong room.")
 
+            # RoomStreamToken expects [int] not Optional[int]
+            assert event.internal_metadata.stream_ordering is not None
             room_token = RoomStreamToken(
                 event.depth, event.internal_metadata.stream_ordering
             )
@@ -174,16 +177,13 @@ async def on_POST(self, request, room_id, event_id):
 class PurgeHistoryStatusRestServlet(RestServlet):
     PATTERNS = admin_patterns("/purge_history_status/(?P<purge_id>[^/]+)")
 
-    def __init__(self, hs):
-        """
-
-        Args:
-            hs (synapse.server.HomeServer)
-        """
+    def __init__(self, hs: "HomeServer"):
         self.pagination_handler = hs.get_pagination_handler()
         self.auth = hs.get_auth()
 
-    async def on_GET(self, request, purge_id):
+    async def on_GET(
+        self, request: SynapseRequest, purge_id: str
+    ) -> Tuple[int, JsonDict]:
         await assert_requester_is_admin(self.auth, request)
 
         purge_status = self.pagination_handler.get_purge_status(purge_id)
@@ -204,12 +204,12 @@ async def on_GET(self, request, purge_id):
 class AdminRestResource(JsonResource):
     """The REST resource which gets mounted at /_synapse/admin"""
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         JsonResource.__init__(self, hs, canonical_json=False)
         register_servlets(hs, self)
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
     """
     Register all the admin servlets.
     """
@@ -224,7 +224,6 @@ def register_servlets(hs, http_server):
     SendServerNoticeServlet(hs).register(http_server)
     VersionServlet(hs).register(http_server)
     UserAdminServlet(hs).register(http_server)
-    UserMediaRestServlet(hs).register(http_server)
     UserMembershipRestServlet(hs).register(http_server)
     UserTokenRestServlet(hs).register(http_server)
     UserRestServletV2(hs).register(http_server)
@@ -240,15 +239,18 @@ def register_servlets(hs, http_server):
     ShadowBanRestServlet(hs).register(http_server)
     ForwardExtremitiesRestServlet(hs).register(http_server)
     RoomEventContextServlet(hs).register(http_server)
+    RateLimitRestServlet(hs).register(http_server)
+    UsernameAvailableRestServlet(hs).register(http_server)
 
 
-def register_servlets_for_client_rest_resource(hs, http_server):
+def register_servlets_for_client_rest_resource(
+    hs: "HomeServer", http_server: HttpServer
+) -> None:
     """Register only the servlets which need to be exposed on /_matrix/client/xxx"""
     WhoisRestServlet(hs).register(http_server)
     PurgeHistoryStatusRestServlet(hs).register(http_server)
     DeactivateAccountRestServlet(hs).register(http_server)
     PurgeHistoryRestServlet(hs).register(http_server)
-    UsersRestServlet(hs).register(http_server)
     ResetPasswordRestServlet(hs).register(http_server)
     SearchUsersRestServlet(hs).register(http_server)
     ShutdownRoomRestServlet(hs).register(http_server)
diff --git a/synapse/rest/admin/_base.py b/synapse/rest/admin/_base.py
index 7681e55b58..d9a2f6ca15 100644
--- a/synapse/rest/admin/_base.py
+++ b/synapse/rest/admin/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +13,7 @@
 # limitations under the License.
 
 import re
+from typing import Iterable, Pattern
 
 from synapse.api.auth import Auth
 from synapse.api.errors import AuthError
@@ -21,7 +21,7 @@
 from synapse.types import UserID
 
 
-def admin_patterns(path_regex: str, version: str = "v1"):
+def admin_patterns(path_regex: str, version: str = "v1") -> Iterable[Pattern]:
     """Returns the list of patterns for an admin endpoint
 
     Args:
diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py
index 5996de11c3..5715190a78 100644
--- a/synapse/rest/admin/devices.py
+++ b/synapse/rest/admin/devices.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Dirk Klimpel
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/admin/event_reports.py b/synapse/rest/admin/event_reports.py
index 381c3fe685..bbfcaf723b 100644
--- a/synapse/rest/admin/event_reports.py
+++ b/synapse/rest/admin/event_reports.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Dirk Klimpel
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/admin/groups.py b/synapse/rest/admin/groups.py
index ebc587aa06..68a3ba3cb7 100644
--- a/synapse/rest/admin/groups.py
+++ b/synapse/rest/admin/groups.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+from typing import TYPE_CHECKING, Tuple
 
 from synapse.api.errors import SynapseError
 from synapse.http.servlet import RestServlet
+from synapse.http.site import SynapseRequest
 from synapse.rest.admin._base import admin_patterns, assert_user_is_admin
+from synapse.types import JsonDict
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 
@@ -26,12 +31,14 @@ class DeleteGroupAdminRestServlet(RestServlet):
 
     PATTERNS = admin_patterns("/delete_group/(?P<group_id>[^/]*)")
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.group_server = hs.get_groups_server_handler()
         self.is_mine_id = hs.is_mine_id
         self.auth = hs.get_auth()
 
-    async def on_POST(self, request, group_id):
+    async def on_POST(
+        self, request: SynapseRequest, group_id: str
+    ) -> Tuple[int, JsonDict]:
         requester = await self.auth.get_user_by_req(request)
         await assert_user_is_admin(self.auth, requester.user)
 
diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py
index 40646ef241..8ce443049e 100644
--- a/synapse/rest/admin/media.py
+++ b/synapse/rest/admin/media.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018-2019 New Vector Ltd
 #
@@ -18,14 +17,16 @@
 from typing import TYPE_CHECKING, Tuple
 
 from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
-from synapse.http.servlet import RestServlet, parse_boolean, parse_integer
+from synapse.http.server import HttpServer
+from synapse.http.servlet import RestServlet, parse_boolean, parse_integer, parse_string
 from synapse.http.site import SynapseRequest
 from synapse.rest.admin._base import (
     admin_patterns,
     assert_requester_is_admin,
     assert_user_is_admin,
 )
-from synapse.types import JsonDict
+from synapse.storage.databases.main.media_repository import MediaSortOrder
+from synapse.types import JsonDict, UserID
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -38,12 +39,11 @@ class QuarantineMediaInRoom(RestServlet):
     this server.
     """
 
-    PATTERNS = (
-        admin_patterns("/room/(?P<room_id>[^/]+)/media/quarantine")
-        +
+    PATTERNS = [
+        *admin_patterns("/room/(?P<room_id>[^/]+)/media/quarantine"),
         # This path kept around for legacy reasons
-        admin_patterns("/quarantine_media/(?P<room_id>[^/]+)")
-    )
+        *admin_patterns("/quarantine_media/(?P<room_id>[^/]+)"),
+    ]
 
     def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastore()
@@ -121,6 +121,35 @@ async def on_POST(
         return 200, {}
 
 
+class UnquarantineMediaByID(RestServlet):
+    """Quarantines local or remote media by a given ID so that no one can download
+    it via this server.
+    """
+
+    PATTERNS = admin_patterns(
+        "/media/unquarantine/(?P<server_name>[^/]+)/(?P<media_id>[^/]+)"
+    )
+
+    def __init__(self, hs: "HomeServer"):
+        self.store = hs.get_datastore()
+        self.auth = hs.get_auth()
+
+    async def on_POST(
+        self, request: SynapseRequest, server_name: str, media_id: str
+    ) -> Tuple[int, JsonDict]:
+        requester = await self.auth.get_user_by_req(request)
+        await assert_user_is_admin(self.auth, requester.user)
+
+        logging.info(
+            "Remove from quarantine local media by ID: %s/%s", server_name, media_id
+        )
+
+        # Remove from quarantine this media id
+        await self.store.quarantine_media_by_id(server_name, media_id, None)
+
+        return 200, {}
+
+
 class ProtectMediaByID(RestServlet):
     """Protect local media from being quarantined."""
 
@@ -138,8 +167,31 @@ async def on_POST(
 
         logging.info("Protecting local media by ID: %s", media_id)
 
-        # Quarantine this media id
-        await self.store.mark_local_media_as_safe(media_id)
+        # Protect this media id
+        await self.store.mark_local_media_as_safe(media_id, safe=True)
+
+        return 200, {}
+
+
+class UnprotectMediaByID(RestServlet):
+    """Unprotect local media from being quarantined."""
+
+    PATTERNS = admin_patterns("/media/unprotect/(?P<media_id>[^/]+)")
+
+    def __init__(self, hs: "HomeServer"):
+        self.store = hs.get_datastore()
+        self.auth = hs.get_auth()
+
+    async def on_POST(
+        self, request: SynapseRequest, media_id: str
+    ) -> Tuple[int, JsonDict]:
+        requester = await self.auth.get_user_by_req(request)
+        await assert_user_is_admin(self.auth, requester.user)
+
+        logging.info("Unprotecting local media by ID: %s", media_id)
+
+        # Unprotect this media id
+        await self.store.mark_local_media_as_safe(media_id, safe=False)
 
         return 200, {}
 
@@ -208,7 +260,9 @@ async def on_DELETE(
 
         logging.info("Deleting local media by ID: %s", media_id)
 
-        deleted_media, total = await self.media_repository.delete_local_media(media_id)
+        deleted_media, total = await self.media_repository.delete_local_media_ids(
+            [media_id]
+        )
         return 200, {"deleted_media": deleted_media, "total": total}
 
 
@@ -261,15 +315,177 @@ async def on_POST(
         return 200, {"deleted_media": deleted_media, "total": total}
 
 
-def register_servlets_for_media_repo(hs: "HomeServer", http_server):
+class UserMediaRestServlet(RestServlet):
+    """
+    Gets information about all uploaded local media for a specific `user_id`.
+    With DELETE request you can delete all this media.
+
+    Example:
+        http://localhost:8008/_synapse/admin/v1/users/@user:server/media
+
+    Args:
+        The parameters `from` and `limit` are required for pagination.
+        By default, a `limit` of 100 is used.
+    Returns:
+        A list of media and an integer representing the total number of
+        media that exist given for this user
+    """
+
+    PATTERNS = admin_patterns("/users/(?P<user_id>[^/]+)/media$")
+
+    def __init__(self, hs: "HomeServer"):
+        self.is_mine = hs.is_mine
+        self.auth = hs.get_auth()
+        self.store = hs.get_datastore()
+        self.media_repository = hs.get_media_repository()
+
+    async def on_GET(
+        self, request: SynapseRequest, user_id: str
+    ) -> Tuple[int, JsonDict]:
+        # This will always be set by the time Twisted calls us.
+        assert request.args is not None
+
+        await assert_requester_is_admin(self.auth, request)
+
+        if not self.is_mine(UserID.from_string(user_id)):
+            raise SynapseError(400, "Can only look up local users")
+
+        user = await self.store.get_user_by_id(user_id)
+        if user is None:
+            raise NotFoundError("Unknown user")
+
+        start = parse_integer(request, "from", default=0)
+        limit = parse_integer(request, "limit", default=100)
+
+        if start < 0:
+            raise SynapseError(
+                400,
+                "Query parameter from must be a string representing a positive integer.",
+                errcode=Codes.INVALID_PARAM,
+            )
+
+        if limit < 0:
+            raise SynapseError(
+                400,
+                "Query parameter limit must be a string representing a positive integer.",
+                errcode=Codes.INVALID_PARAM,
+            )
+
+        # If neither `order_by` nor `dir` is set, set the default order
+        # to newest media is on top for backward compatibility.
+        if b"order_by" not in request.args and b"dir" not in request.args:
+            order_by = MediaSortOrder.CREATED_TS.value
+            direction = "b"
+        else:
+            order_by = parse_string(
+                request,
+                "order_by",
+                default=MediaSortOrder.CREATED_TS.value,
+                allowed_values=(
+                    MediaSortOrder.MEDIA_ID.value,
+                    MediaSortOrder.UPLOAD_NAME.value,
+                    MediaSortOrder.CREATED_TS.value,
+                    MediaSortOrder.LAST_ACCESS_TS.value,
+                    MediaSortOrder.MEDIA_LENGTH.value,
+                    MediaSortOrder.MEDIA_TYPE.value,
+                    MediaSortOrder.QUARANTINED_BY.value,
+                    MediaSortOrder.SAFE_FROM_QUARANTINE.value,
+                ),
+            )
+            direction = parse_string(
+                request, "dir", default="f", allowed_values=("f", "b")
+            )
+
+        media, total = await self.store.get_local_media_by_user_paginate(
+            start, limit, user_id, order_by, direction
+        )
+
+        ret = {"media": media, "total": total}
+        if (start + limit) < total:
+            ret["next_token"] = start + len(media)
+
+        return 200, ret
+
+    async def on_DELETE(
+        self, request: SynapseRequest, user_id: str
+    ) -> Tuple[int, JsonDict]:
+        # This will always be set by the time Twisted calls us.
+        assert request.args is not None
+
+        await assert_requester_is_admin(self.auth, request)
+
+        if not self.is_mine(UserID.from_string(user_id)):
+            raise SynapseError(400, "Can only look up local users")
+
+        user = await self.store.get_user_by_id(user_id)
+        if user is None:
+            raise NotFoundError("Unknown user")
+
+        start = parse_integer(request, "from", default=0)
+        limit = parse_integer(request, "limit", default=100)
+
+        if start < 0:
+            raise SynapseError(
+                400,
+                "Query parameter from must be a string representing a positive integer.",
+                errcode=Codes.INVALID_PARAM,
+            )
+
+        if limit < 0:
+            raise SynapseError(
+                400,
+                "Query parameter limit must be a string representing a positive integer.",
+                errcode=Codes.INVALID_PARAM,
+            )
+
+        # If neither `order_by` nor `dir` is set, set the default order
+        # to newest media is on top for backward compatibility.
+        if b"order_by" not in request.args and b"dir" not in request.args:
+            order_by = MediaSortOrder.CREATED_TS.value
+            direction = "b"
+        else:
+            order_by = parse_string(
+                request,
+                "order_by",
+                default=MediaSortOrder.CREATED_TS.value,
+                allowed_values=(
+                    MediaSortOrder.MEDIA_ID.value,
+                    MediaSortOrder.UPLOAD_NAME.value,
+                    MediaSortOrder.CREATED_TS.value,
+                    MediaSortOrder.LAST_ACCESS_TS.value,
+                    MediaSortOrder.MEDIA_LENGTH.value,
+                    MediaSortOrder.MEDIA_TYPE.value,
+                    MediaSortOrder.QUARANTINED_BY.value,
+                    MediaSortOrder.SAFE_FROM_QUARANTINE.value,
+                ),
+            )
+            direction = parse_string(
+                request, "dir", default="f", allowed_values=("f", "b")
+            )
+
+        media, _ = await self.store.get_local_media_by_user_paginate(
+            start, limit, user_id, order_by, direction
+        )
+
+        deleted_media, total = await self.media_repository.delete_local_media_ids(
+            ([row["media_id"] for row in media])
+        )
+
+        return 200, {"deleted_media": deleted_media, "total": total}
+
+
+def register_servlets_for_media_repo(hs: "HomeServer", http_server: HttpServer) -> None:
     """
     Media repo specific APIs.
     """
     PurgeMediaCacheRestServlet(hs).register(http_server)
     QuarantineMediaInRoom(hs).register(http_server)
     QuarantineMediaByID(hs).register(http_server)
+    UnquarantineMediaByID(hs).register(http_server)
     QuarantineMediaByUser(hs).register(http_server)
     ProtectMediaByID(hs).register(http_server)
+    UnprotectMediaByID(hs).register(http_server)
     ListMediaInRoom(hs).register(http_server)
     DeleteMediaByID(hs).register(http_server)
     DeleteMediaByDateSize(hs).register(http_server)
+    UserMediaRestServlet(hs).register(http_server)
diff --git a/synapse/rest/admin/purge_room_servlet.py b/synapse/rest/admin/purge_room_servlet.py
index 49966ee3e0..2365ff7a0f 100644
--- a/synapse/rest/admin/purge_room_servlet.py
+++ b/synapse/rest/admin/purge_room_servlet.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py
index cfe1bebb91..975c28b225 100644
--- a/synapse/rest/admin/rooms.py
+++ b/synapse/rest/admin/rooms.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,6 +20,7 @@
 from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
 from synapse.api.filtering import Filter
 from synapse.http.servlet import (
+    ResolveRoomIdMixin,
     RestServlet,
     assert_params_in_dict,
     parse_integer,
@@ -34,58 +34,18 @@
     assert_user_is_admin,
 )
 from synapse.storage.databases.main.room import RoomSortOrder
-from synapse.types import JsonDict, RoomAlias, RoomID, UserID, create_requester
+from synapse.types import JsonDict, UserID, create_requester
 from synapse.util import json_decoder
 
 if TYPE_CHECKING:
+    from synapse.api.auth import Auth
+    from synapse.handlers.pagination import PaginationHandler
+    from synapse.handlers.room import RoomShutdownHandler
     from synapse.server import HomeServer
 
-
 logger = logging.getLogger(__name__)
 
 
-class ResolveRoomIdMixin:
-    def __init__(self, hs: "HomeServer"):
-        self.room_member_handler = hs.get_room_member_handler()
-
-    async def resolve_room_id(
-        self, room_identifier: str, remote_room_hosts: Optional[List[str]] = None
-    ) -> Tuple[str, Optional[List[str]]]:
-        """
-        Resolve a room identifier to a room ID, if necessary.
-
-        This also performanes checks to ensure the room ID is of the proper form.
-
-        Args:
-            room_identifier: The room ID or alias.
-            remote_room_hosts: The potential remote room hosts to use.
-
-        Returns:
-            The resolved room ID.
-
-        Raises:
-            SynapseError if the room ID is of the wrong form.
-        """
-        if RoomID.is_valid(room_identifier):
-            resolved_room_id = room_identifier
-        elif RoomAlias.is_valid(room_identifier):
-            room_alias = RoomAlias.from_string(room_identifier)
-            (
-                room_id,
-                remote_room_hosts,
-            ) = await self.room_member_handler.lookup_room_alias(room_alias)
-            resolved_room_id = room_id.to_string()
-        else:
-            raise SynapseError(
-                400, "%s was not legal room ID or room alias" % (room_identifier,)
-            )
-        if not resolved_room_id:
-            raise SynapseError(
-                400, "Unknown room ID or room alias %s" % room_identifier
-            )
-        return resolved_room_id, remote_room_hosts
-
-
 class ShutdownRoomRestServlet(RestServlet):
     """Shuts down a room by removing all local users from the room and blocking
     all future invites and joins to the room. Any local aliases will be repointed
@@ -147,50 +107,14 @@ def __init__(self, hs: "HomeServer"):
     async def on_POST(
         self, request: SynapseRequest, room_id: str
     ) -> Tuple[int, JsonDict]:
-        requester = await self.auth.get_user_by_req(request)
-        await assert_user_is_admin(self.auth, requester.user)
-
-        content = parse_json_object_from_request(request)
-
-        block = content.get("block", False)
-        if not isinstance(block, bool):
-            raise SynapseError(
-                HTTPStatus.BAD_REQUEST,
-                "Param 'block' must be a boolean, if given",
-                Codes.BAD_JSON,
-            )
-
-        purge = content.get("purge", True)
-        if not isinstance(purge, bool):
-            raise SynapseError(
-                HTTPStatus.BAD_REQUEST,
-                "Param 'purge' must be a boolean, if given",
-                Codes.BAD_JSON,
-            )
-
-        force_purge = content.get("force_purge", False)
-        if not isinstance(force_purge, bool):
-            raise SynapseError(
-                HTTPStatus.BAD_REQUEST,
-                "Param 'force_purge' must be a boolean, if given",
-                Codes.BAD_JSON,
-            )
-
-        ret = await self.room_shutdown_handler.shutdown_room(
-            room_id=room_id,
-            new_room_user_id=content.get("new_room_user_id"),
-            new_room_name=content.get("room_name"),
-            message=content.get("message"),
-            requester_user_id=requester.user.to_string(),
-            block=block,
+        return await _delete_room(
+            request,
+            room_id,
+            self.auth,
+            self.room_shutdown_handler,
+            self.pagination_handler,
         )
 
-        # Purge room
-        if purge:
-            await self.pagination_handler.purge_room(room_id, force=force_purge)
-
-        return (200, ret)
-
 
 class ListRoomRestServlet(RestServlet):
     """
@@ -283,7 +207,22 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
 
 
 class RoomRestServlet(RestServlet):
-    """Get room details.
+    """Manage a room.
+
+    On GET : Get details of a room.
+
+    On DELETE : Delete a room from server.
+
+    It is a combination and improvement of shutdown and purge room.
+
+    Shuts down a room by removing all local users from the room.
+    Blocking all future invites and joins to the room is optional.
+
+    If desired any local aliases will be repointed to a new room
+    created by `new_room_user_id` and kicked users will be auto-
+    joined to the new room.
+
+    If 'purge' is true, it will remove all traces of a room from the database.
 
     TODO: Add on_POST to allow room creation without joining the room
     """
@@ -294,6 +233,8 @@ def __init__(self, hs: "HomeServer"):
         self.hs = hs
         self.auth = hs.get_auth()
         self.store = hs.get_datastore()
+        self.room_shutdown_handler = hs.get_room_shutdown_handler()
+        self.pagination_handler = hs.get_pagination_handler()
 
     async def on_GET(
         self, request: SynapseRequest, room_id: str
@@ -309,6 +250,17 @@ async def on_GET(
 
         return (200, ret)
 
+    async def on_DELETE(
+        self, request: SynapseRequest, room_id: str
+    ) -> Tuple[int, JsonDict]:
+        return await _delete_room(
+            request,
+            room_id,
+            self.auth,
+            self.room_shutdown_handler,
+            self.pagination_handler,
+        )
+
 
 class RoomMembersRestServlet(RestServlet):
     """
@@ -409,9 +361,9 @@ async def on_POST(
 
         # Get the room ID from the identifier.
         try:
-            remote_room_hosts = [
+            remote_room_hosts: Optional[List[str]] = [
                 x.decode("ascii") for x in request.args[b"server_name"]
-            ]  # type: Optional[List[str]]
+            ]
         except Exception:
             remote_room_hosts = None
         room_id, remote_room_hosts = await self.resolve_room_id(
@@ -469,6 +421,7 @@ def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
         self.hs = hs
         self.auth = hs.get_auth()
+        self.store = hs.get_datastore()
         self.event_creation_handler = hs.get_event_creation_handler()
         self.state_handler = hs.get_state_handler()
         self.is_mine_id = hs.is_mine_id
@@ -507,7 +460,13 @@ async def on_POST(
             admin_user_id = None
 
             for admin_user in reversed(admin_users):
-                if room_state.get((EventTypes.Member, admin_user)):
+                (
+                    current_membership_type,
+                    _,
+                ) = await self.store.get_local_current_membership_for_user_in_room(
+                    admin_user, room_id
+                )
+                if current_membership_type == "join":
                     admin_user_id = admin_user
                     break
 
@@ -656,12 +615,10 @@ async def on_GET(
         limit = parse_integer(request, "limit", default=10)
 
         # picking the API shape for symmetry with /messages
-        filter_str = parse_string(request, b"filter", encoding="utf-8")
+        filter_str = parse_string(request, "filter", encoding="utf-8")
         if filter_str:
             filter_json = urlparse.unquote(filter_str)
-            event_filter = Filter(
-                json_decoder.decode(filter_json)
-            )  # type: Optional[Filter]
+            event_filter: Optional[Filter] = Filter(json_decoder.decode(filter_json))
         else:
             event_filter = None
 
@@ -695,3 +652,55 @@ async def on_GET(
         )
 
         return 200, results
+
+
+async def _delete_room(
+    request: SynapseRequest,
+    room_id: str,
+    auth: "Auth",
+    room_shutdown_handler: "RoomShutdownHandler",
+    pagination_handler: "PaginationHandler",
+) -> Tuple[int, JsonDict]:
+    requester = await auth.get_user_by_req(request)
+    await assert_user_is_admin(auth, requester.user)
+
+    content = parse_json_object_from_request(request)
+
+    block = content.get("block", False)
+    if not isinstance(block, bool):
+        raise SynapseError(
+            HTTPStatus.BAD_REQUEST,
+            "Param 'block' must be a boolean, if given",
+            Codes.BAD_JSON,
+        )
+
+    purge = content.get("purge", True)
+    if not isinstance(purge, bool):
+        raise SynapseError(
+            HTTPStatus.BAD_REQUEST,
+            "Param 'purge' must be a boolean, if given",
+            Codes.BAD_JSON,
+        )
+
+    force_purge = content.get("force_purge", False)
+    if not isinstance(force_purge, bool):
+        raise SynapseError(
+            HTTPStatus.BAD_REQUEST,
+            "Param 'force_purge' must be a boolean, if given",
+            Codes.BAD_JSON,
+        )
+
+    ret = await room_shutdown_handler.shutdown_room(
+        room_id=room_id,
+        new_room_user_id=content.get("new_room_user_id"),
+        new_room_name=content.get("room_name"),
+        message=content.get("message"),
+        requester_user_id=requester.user.to_string(),
+        block=block,
+    )
+
+    # Purge room
+    if purge:
+        await pagination_handler.purge_room(room_id, force=force_purge)
+
+    return (200, ret)
diff --git a/synapse/rest/admin/server_notice_servlet.py b/synapse/rest/admin/server_notice_servlet.py
index f495666f4a..b5e4c474ef 100644
--- a/synapse/rest/admin/server_notice_servlet.py
+++ b/synapse/rest/admin/server_notice_servlet.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -55,7 +54,6 @@ def __init__(self, hs: "HomeServer"):
         self.hs = hs
         self.auth = hs.get_auth()
         self.txns = HttpTransactionCache(hs)
-        self.snm = hs.get_server_notices_manager()
 
     def register(self, json_resource: HttpServer):
         PATTERN = "/send_server_notice"
@@ -78,7 +76,10 @@ async def on_POST(
         event_type = body.get("type", EventTypes.Message)
         state_key = body.get("state_key")
 
-        if not self.snm.is_enabled():
+        # We grab the server notices manager here as its initialisation has a check for worker processes,
+        # but worker processes still need to initialise SendServerNoticeServlet (as it is part of the
+        # admin api).
+        if not self.hs.get_server_notices_manager().is_enabled():
             raise SynapseError(400, "Server notices are not enabled on this server")
 
         user_id = body["user_id"]
@@ -86,7 +87,7 @@ async def on_POST(
         if not self.hs.is_mine_id(user_id):
             raise SynapseError(400, "Server notices can only be sent to local users")
 
-        event = await self.snm.send_notice(
+        event = await self.hs.get_server_notices_manager().send_notice(
             user_id=body["user_id"],
             type=event_type,
             state_key=state_key,
diff --git a/synapse/rest/admin/statistics.py b/synapse/rest/admin/statistics.py
index f2490e382d..948de94ccd 100644
--- a/synapse/rest/admin/statistics.py
+++ b/synapse/rest/admin/statistics.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Dirk Klimpel
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/admin/username_available.py b/synapse/rest/admin/username_available.py
new file mode 100644
index 0000000000..2bf1472967
--- /dev/null
+++ b/synapse/rest/admin/username_available.py
@@ -0,0 +1,51 @@
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Tuple
+
+from synapse.http.servlet import RestServlet, parse_string
+from synapse.http.site import SynapseRequest
+from synapse.rest.admin._base import admin_patterns, assert_requester_is_admin
+from synapse.types import JsonDict
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class UsernameAvailableRestServlet(RestServlet):
+    """An admin API to check if a given username is available, regardless of whether registration is enabled.
+
+    Example:
+        GET /_synapse/admin/v1/username_available?username=foo
+        200 OK
+        {
+            "available": true
+        }
+    """
+
+    PATTERNS = admin_patterns("/username_available")
+
+    def __init__(self, hs: "HomeServer"):
+        self.auth = hs.get_auth()
+        self.registration_handler = hs.get_registration_handler()
+
+    async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
+        await assert_requester_is_admin(self.auth, request)
+
+        username = parse_string(request, "username", required=True)
+        await self.registration_handler.check_username(username)
+        return HTTPStatus.OK, {"available": True}
diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py
index 309bd2771b..3c8a0c6883 100644
--- a/synapse/rest/admin/users.py
+++ b/synapse/rest/admin/users.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,6 +14,7 @@
 import hashlib
 import hmac
 import logging
+import secrets
 from http import HTTPStatus
 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
 
@@ -34,8 +34,8 @@
     assert_requester_is_admin,
     assert_user_is_admin,
 )
-from synapse.rest.client.v2_alpha._base import client_patterns
-from synapse.storage.databases.main.media_repository import MediaSortOrder
+from synapse.rest.client._base import client_patterns
+from synapse.storage.databases.main.stats import UserSortOrder
 from synapse.types import JsonDict, UserID
 
 if TYPE_CHECKING:
@@ -44,29 +44,6 @@
 logger = logging.getLogger(__name__)
 
 
-class UsersRestServlet(RestServlet):
-    PATTERNS = admin_patterns("/users/(?P<user_id>[^/]*)$")
-
-    def __init__(self, hs: "HomeServer"):
-        self.hs = hs
-        self.store = hs.get_datastore()
-        self.auth = hs.get_auth()
-        self.admin_handler = hs.get_admin_handler()
-
-    async def on_GET(
-        self, request: SynapseRequest, user_id: str
-    ) -> Tuple[int, List[JsonDict]]:
-        target_user = UserID.from_string(user_id)
-        await assert_requester_is_admin(self.auth, request)
-
-        if not self.hs.is_mine(target_user):
-            raise SynapseError(400, "Can only users a local user")
-
-        ret = await self.store.get_users()
-
-        return 200, ret
-
-
 class UsersRestServletV2(RestServlet):
     PATTERNS = admin_patterns("/users$", "v2")
 
@@ -84,6 +61,7 @@ class UsersRestServletV2(RestServlet):
     The parameter `name` can be used to filter by user id or display name.
     The parameter `guests` can be used to exclude guest users.
     The parameter `deactivated` can be used to include deactivated users.
+    The parameter `order_by` can be used to order the result.
     """
 
     def __init__(self, hs: "HomeServer"):
@@ -112,13 +90,32 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
                 errcode=Codes.INVALID_PARAM,
             )
 
-        user_id = parse_string(request, "user_id", default=None)
-        name = parse_string(request, "name", default=None)
+        user_id = parse_string(request, "user_id")
+        name = parse_string(request, "name")
         guests = parse_boolean(request, "guests", default=True)
         deactivated = parse_boolean(request, "deactivated", default=False)
 
+        order_by = parse_string(
+            request,
+            "order_by",
+            default=UserSortOrder.NAME.value,
+            allowed_values=(
+                UserSortOrder.NAME.value,
+                UserSortOrder.DISPLAYNAME.value,
+                UserSortOrder.GUEST.value,
+                UserSortOrder.ADMIN.value,
+                UserSortOrder.DEACTIVATED.value,
+                UserSortOrder.USER_TYPE.value,
+                UserSortOrder.AVATAR_URL.value,
+                UserSortOrder.SHADOW_BANNED.value,
+                UserSortOrder.CREATION_TS.value,
+            ),
+        )
+
+        direction = parse_string(request, "dir", default="f", allowed_values=("f", "b"))
+
         users, total = await self.store.get_users_paginate(
-            start, limit, user_id, name, guests, deactivated
+            start, limit, user_id, name, guests, deactivated, order_by, direction
         )
         ret = {"users": users, "total": total}
         if (start + limit) < total:
@@ -174,7 +171,7 @@ async def on_GET(
 
         target_user = UserID.from_string(user_id)
         if not self.hs.is_mine(target_user):
-            raise SynapseError(400, "Can only lookup local users")
+            raise SynapseError(400, "Can only look up local users")
 
         ret = await self.admin_handler.get_user(target_user)
 
@@ -198,20 +195,57 @@ async def on_PUT(
         user = await self.admin_handler.get_user(target_user)
         user_id = target_user.to_string()
 
+        # check for required parameters for each threepid
+        threepids = body.get("threepids")
+        if threepids is not None:
+            for threepid in threepids:
+                assert_params_in_dict(threepid, ["medium", "address"])
+
+        # check for required parameters for each external_id
+        external_ids = body.get("external_ids")
+        if external_ids is not None:
+            for external_id in external_ids:
+                assert_params_in_dict(external_id, ["auth_provider", "external_id"])
+
+        user_type = body.get("user_type", None)
+        if user_type is not None and user_type not in UserTypes.ALL_USER_TYPES:
+            raise SynapseError(400, "Invalid user type")
+
+        set_admin_to = body.get("admin", False)
+        if not isinstance(set_admin_to, bool):
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST,
+                "Param 'admin' must be a boolean, if given",
+                Codes.BAD_JSON,
+            )
+
+        password = body.get("password", None)
+        if password is not None:
+            if not isinstance(password, str) or len(password) > 512:
+                raise SynapseError(400, "Invalid password")
+
+        deactivate = body.get("deactivated", False)
+        if not isinstance(deactivate, bool):
+            raise SynapseError(400, "'deactivated' parameter is not of type boolean")
+
+        # convert into List[Tuple[str, str]]
+        if external_ids is not None:
+            new_external_ids = []
+            for external_id in external_ids:
+                new_external_ids.append(
+                    (external_id["auth_provider"], external_id["external_id"])
+                )
+
         if user:  # modify user
             if "displayname" in body:
                 await self.profile_handler.set_displayname(
                     target_user, requester, body["displayname"], True
                 )
 
-            if "threepids" in body:
-                # check for required parameters for each threepid
-                for threepid in body["threepids"]:
-                    assert_params_in_dict(threepid, ["medium", "address"])
-
+            if threepids is not None:
                 # remove old threepids from user
-                threepids = await self.store.user_get_threepids(user_id)
-                for threepid in threepids:
+                old_threepids = await self.store.user_get_threepids(user_id)
+                for threepid in old_threepids:
                     try:
                         await self.auth_handler.delete_threepid(
                             user_id, threepid["medium"], threepid["address"], None
@@ -222,18 +256,39 @@ async def on_PUT(
 
                 # add new threepids to user
                 current_time = self.hs.get_clock().time_msec()
-                for threepid in body["threepids"]:
+                for threepid in threepids:
                     await self.auth_handler.add_threepid(
                         user_id, threepid["medium"], threepid["address"], current_time
                     )
 
-            if "avatar_url" in body and type(body["avatar_url"]) == str:
+            if external_ids is not None:
+                # get changed external_ids (added and removed)
+                cur_external_ids = await self.store.get_external_ids_by_user(user_id)
+                add_external_ids = set(new_external_ids) - set(cur_external_ids)
+                del_external_ids = set(cur_external_ids) - set(new_external_ids)
+
+                # remove old external_ids
+                for auth_provider, external_id in del_external_ids:
+                    await self.store.remove_user_external_id(
+                        auth_provider,
+                        external_id,
+                        user_id,
+                    )
+
+                # add new external_ids
+                for auth_provider, external_id in add_external_ids:
+                    await self.store.record_user_external_id(
+                        auth_provider,
+                        external_id,
+                        user_id,
+                    )
+
+            if "avatar_url" in body and isinstance(body["avatar_url"], str):
                 await self.profile_handler.set_avatar_url(
                     target_user, requester, body["avatar_url"], True
                 )
 
             if "admin" in body:
-                set_admin_to = bool(body["admin"])
                 if set_admin_to != user["admin"]:
                     auth_user = requester.user
                     if target_user == auth_user and not set_admin_to:
@@ -241,29 +296,18 @@ async def on_PUT(
 
                     await self.store.set_server_admin(target_user, set_admin_to)
 
-            if "password" in body:
-                if not isinstance(body["password"], str) or len(body["password"]) > 512:
-                    raise SynapseError(400, "Invalid password")
-                else:
-                    new_password = body["password"]
-                    logout_devices = True
-
-                    new_password_hash = await self.auth_handler.hash(new_password)
-
-                    await self.set_password_handler.set_password(
-                        target_user.to_string(),
-                        new_password_hash,
-                        logout_devices,
-                        requester,
-                    )
+            if password is not None:
+                logout_devices = True
+                new_password_hash = await self.auth_handler.hash(password)
+
+                await self.set_password_handler.set_password(
+                    target_user.to_string(),
+                    new_password_hash,
+                    logout_devices,
+                    requester,
+                )
 
             if "deactivated" in body:
-                deactivate = body["deactivated"]
-                if not isinstance(deactivate, bool):
-                    raise SynapseError(
-                        400, "'deactivated' parameter is not of type boolean"
-                    )
-
                 if deactivate and not user["deactivated"]:
                     await self.deactivate_account_handler.deactivate_account(
                         target_user.to_string(), False, requester, by_admin=True
@@ -287,36 +331,24 @@ async def on_PUT(
             return 200, user
 
         else:  # create user
-            password = body.get("password")
+            displayname = body.get("displayname", None)
+
             password_hash = None
             if password is not None:
-                if not isinstance(password, str) or len(password) > 512:
-                    raise SynapseError(400, "Invalid password")
                 password_hash = await self.auth_handler.hash(password)
 
-            admin = body.get("admin", None)
-            user_type = body.get("user_type", None)
-            displayname = body.get("displayname", None)
-
-            if user_type is not None and user_type not in UserTypes.ALL_USER_TYPES:
-                raise SynapseError(400, "Invalid user type")
-
             user_id = await self.registration_handler.register_user(
                 localpart=target_user.localpart,
                 password_hash=password_hash,
-                admin=bool(admin),
+                admin=set_admin_to,
                 default_display_name=displayname,
                 user_type=user_type,
                 by_admin=True,
             )
 
-            if "threepids" in body:
-                # check for required parameters for each threepid
-                for threepid in body["threepids"]:
-                    assert_params_in_dict(threepid, ["medium", "address"])
-
+            if threepids is not None:
                 current_time = self.hs.get_clock().time_msec()
-                for threepid in body["threepids"]:
+                for threepid in threepids:
                     await self.auth_handler.add_threepid(
                         user_id, threepid["medium"], threepid["address"], current_time
                     )
@@ -336,6 +368,14 @@ async def on_PUT(
                             data={},
                         )
 
+            if external_ids is not None:
+                for auth_provider, external_id in new_external_ids:
+                    await self.store.record_user_external_id(
+                        auth_provider,
+                        external_id,
+                        user_id,
+                    )
+
             if "avatar_url" in body and isinstance(body["avatar_url"], str):
                 await self.profile_handler.set_avatar_url(
                     target_user, requester, body["avatar_url"], True
@@ -361,7 +401,7 @@ class UserRegisterServlet(RestServlet):
     def __init__(self, hs: "HomeServer"):
         self.auth_handler = hs.get_auth_handler()
         self.reactor = hs.get_reactor()
-        self.nonces = {}  # type: Dict[str, int]
+        self.nonces: Dict[str, int] = {}
         self.hs = hs
 
     def _clear_old_nonces(self):
@@ -380,7 +420,7 @@ def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
         """
         self._clear_old_nonces()
 
-        nonce = self.hs.get_secrets().token_hex(64)
+        nonce = secrets.token_hex(64)
         self.nonces[nonce] = int(self.reactor.seconds())
         return 200, {"nonce": nonce}
 
@@ -463,7 +503,7 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
             raise SynapseError(403, "HMAC incorrect")
 
         # Reuse the parts of RegisterRestServlet to reduce code duplication
-        from synapse.rest.client.v2_alpha.register import RegisterRestServlet
+        from synapse.rest.client.register import RegisterRestServlet
 
         register = RegisterRestServlet(self.hs)
 
@@ -482,13 +522,12 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
 
 class WhoisRestServlet(RestServlet):
     path_regex = "/whois/(?P<user_id>[^/]*)$"
-    PATTERNS = (
-        admin_patterns(path_regex)
-        +
+    PATTERNS = [
+        *admin_patterns(path_regex),
         # URL for spec reason
         # https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid
-        client_patterns("/admin" + path_regex, v1=True)
-    )
+        *client_patterns("/admin" + path_regex, v1=True),
+    ]
 
     def __init__(self, hs: "HomeServer"):
         self.hs = hs
@@ -557,11 +596,7 @@ async def on_POST(
 class AccountValidityRenewServlet(RestServlet):
     PATTERNS = admin_patterns("/account_validity/validity$")
 
-    def __init__(self, hs):
-        """
-        Args:
-            hs (synapse.server.HomeServer): server
-        """
+    def __init__(self, hs: "HomeServer"):
         self.hs = hs
         self.account_activity_handler = hs.get_account_validity_handler()
         self.auth = hs.get_auth()
@@ -569,16 +604,24 @@ def __init__(self, hs):
     async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
         await assert_requester_is_admin(self.auth, request)
 
-        body = parse_json_object_from_request(request)
+        if self.account_activity_handler.on_legacy_admin_request_callback:
+            expiration_ts = await (
+                self.account_activity_handler.on_legacy_admin_request_callback(request)
+            )
+        else:
+            body = parse_json_object_from_request(request)
 
-        if "user_id" not in body:
-            raise SynapseError(400, "Missing property 'user_id' in the request body")
+            if "user_id" not in body:
+                raise SynapseError(
+                    400,
+                    "Missing property 'user_id' in the request body",
+                )
 
-        expiration_ts = await self.account_activity_handler.renew_account_for_user(
-            body["user_id"],
-            body.get("expiration_ts"),
-            not body.get("enable_renewal_emails", True),
-        )
+            expiration_ts = await self.account_activity_handler.renew_account_for_user(
+                body["user_id"],
+                body.get("expiration_ts"),
+                not body.get("enable_renewal_emails", True),
+            )
 
         res = {"expiration_ts": expiration_ts}
         return 200, res
@@ -795,7 +838,7 @@ async def on_GET(
         await assert_requester_is_admin(self.auth, request)
 
         if not self.is_mine(UserID.from_string(user_id)):
-            raise SynapseError(400, "Can only lookup local users")
+            raise SynapseError(400, "Can only look up local users")
 
         if not await self.store.get_user_by_id(user_id):
             raise NotFoundError("User not found")
@@ -807,97 +850,6 @@ async def on_GET(
         return 200, {"pushers": filtered_pushers, "total": len(filtered_pushers)}
 
 
-class UserMediaRestServlet(RestServlet):
-    """
-    Gets information about all uploaded local media for a specific `user_id`.
-
-    Example:
-        http://localhost:8008/_synapse/admin/v1/users/
-        @user:server/media
-
-    Args:
-        The parameters `from` and `limit` are required for pagination.
-        By default, a `limit` of 100 is used.
-    Returns:
-        A list of media and an integer representing the total number of
-        media that exist given for this user
-    """
-
-    PATTERNS = admin_patterns("/users/(?P<user_id>[^/]+)/media$")
-
-    def __init__(self, hs: "HomeServer"):
-        self.is_mine = hs.is_mine
-        self.auth = hs.get_auth()
-        self.store = hs.get_datastore()
-
-    async def on_GET(
-        self, request: SynapseRequest, user_id: str
-    ) -> Tuple[int, JsonDict]:
-        # This will always be set by the time Twisted calls us.
-        assert request.args is not None
-
-        await assert_requester_is_admin(self.auth, request)
-
-        if not self.is_mine(UserID.from_string(user_id)):
-            raise SynapseError(400, "Can only lookup local users")
-
-        user = await self.store.get_user_by_id(user_id)
-        if user is None:
-            raise NotFoundError("Unknown user")
-
-        start = parse_integer(request, "from", default=0)
-        limit = parse_integer(request, "limit", default=100)
-
-        if start < 0:
-            raise SynapseError(
-                400,
-                "Query parameter from must be a string representing a positive integer.",
-                errcode=Codes.INVALID_PARAM,
-            )
-
-        if limit < 0:
-            raise SynapseError(
-                400,
-                "Query parameter limit must be a string representing a positive integer.",
-                errcode=Codes.INVALID_PARAM,
-            )
-
-        # If neither `order_by` nor `dir` is set, set the default order
-        # to newest media is on top for backward compatibility.
-        if b"order_by" not in request.args and b"dir" not in request.args:
-            order_by = MediaSortOrder.CREATED_TS.value
-            direction = "b"
-        else:
-            order_by = parse_string(
-                request,
-                "order_by",
-                default=MediaSortOrder.CREATED_TS.value,
-                allowed_values=(
-                    MediaSortOrder.MEDIA_ID.value,
-                    MediaSortOrder.UPLOAD_NAME.value,
-                    MediaSortOrder.CREATED_TS.value,
-                    MediaSortOrder.LAST_ACCESS_TS.value,
-                    MediaSortOrder.MEDIA_LENGTH.value,
-                    MediaSortOrder.MEDIA_TYPE.value,
-                    MediaSortOrder.QUARANTINED_BY.value,
-                    MediaSortOrder.SAFE_FROM_QUARANTINE.value,
-                ),
-            )
-            direction = parse_string(
-                request, "dir", default="f", allowed_values=("f", "b")
-            )
-
-        media, total = await self.store.get_local_media_by_user_paginate(
-            start, limit, user_id, order_by, direction
-        )
-
-        ret = {"media": media, "total": total}
-        if (start + limit) < total:
-            ret["next_token"] = start + len(media)
-
-        return 200, ret
-
-
 class UserTokenRestServlet(RestServlet):
     """An admin API for logging in as a user.
 
@@ -985,3 +937,114 @@ async def on_POST(
         await self.store.set_shadow_banned(UserID.from_string(user_id), True)
 
         return 200, {}
+
+
+class RateLimitRestServlet(RestServlet):
+    """An admin API to override ratelimiting for an user.
+
+    Example:
+        POST /_synapse/admin/v1/users/@test:example.com/override_ratelimit
+        {
+          "messages_per_second": 0,
+          "burst_count": 0
+        }
+        200 OK
+        {
+          "messages_per_second": 0,
+          "burst_count": 0
+        }
+    """
+
+    PATTERNS = admin_patterns("/users/(?P<user_id>[^/]*)/override_ratelimit")
+
+    def __init__(self, hs: "HomeServer"):
+        self.hs = hs
+        self.store = hs.get_datastore()
+        self.auth = hs.get_auth()
+
+    async def on_GET(
+        self, request: SynapseRequest, user_id: str
+    ) -> Tuple[int, JsonDict]:
+        await assert_requester_is_admin(self.auth, request)
+
+        if not self.hs.is_mine_id(user_id):
+            raise SynapseError(400, "Can only look up local users")
+
+        if not await self.store.get_user_by_id(user_id):
+            raise NotFoundError("User not found")
+
+        ratelimit = await self.store.get_ratelimit_for_user(user_id)
+
+        if ratelimit:
+            # convert `null` to `0` for consistency
+            # both values do the same in retelimit handler
+            ret = {
+                "messages_per_second": 0
+                if ratelimit.messages_per_second is None
+                else ratelimit.messages_per_second,
+                "burst_count": 0
+                if ratelimit.burst_count is None
+                else ratelimit.burst_count,
+            }
+        else:
+            ret = {}
+
+        return 200, ret
+
+    async def on_POST(
+        self, request: SynapseRequest, user_id: str
+    ) -> Tuple[int, JsonDict]:
+        await assert_requester_is_admin(self.auth, request)
+
+        if not self.hs.is_mine_id(user_id):
+            raise SynapseError(400, "Only local users can be ratelimited")
+
+        if not await self.store.get_user_by_id(user_id):
+            raise NotFoundError("User not found")
+
+        body = parse_json_object_from_request(request, allow_empty_body=True)
+
+        messages_per_second = body.get("messages_per_second", 0)
+        burst_count = body.get("burst_count", 0)
+
+        if not isinstance(messages_per_second, int) or messages_per_second < 0:
+            raise SynapseError(
+                400,
+                "%r parameter must be a positive int" % (messages_per_second,),
+                errcode=Codes.INVALID_PARAM,
+            )
+
+        if not isinstance(burst_count, int) or burst_count < 0:
+            raise SynapseError(
+                400,
+                "%r parameter must be a positive int" % (burst_count,),
+                errcode=Codes.INVALID_PARAM,
+            )
+
+        await self.store.set_ratelimit_for_user(
+            user_id, messages_per_second, burst_count
+        )
+        ratelimit = await self.store.get_ratelimit_for_user(user_id)
+        assert ratelimit is not None
+
+        ret = {
+            "messages_per_second": ratelimit.messages_per_second,
+            "burst_count": ratelimit.burst_count,
+        }
+
+        return 200, ret
+
+    async def on_DELETE(
+        self, request: SynapseRequest, user_id: str
+    ) -> Tuple[int, JsonDict]:
+        await assert_requester_is_admin(self.auth, request)
+
+        if not self.hs.is_mine_id(user_id):
+            raise SynapseError(400, "Only local users can be ratelimited")
+
+        if not await self.store.get_user_by_id(user_id):
+            raise NotFoundError("User not found")
+
+        await self.store.delete_ratelimit_for_user(user_id)
+
+        return 200, {}
diff --git a/synapse/rest/client/__init__.py b/synapse/rest/client/__init__.py
index fe0ac3f8e9..f9830cc51f 100644
--- a/synapse/rest/client/__init__.py
+++ b/synapse/rest/client/__init__.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2014-2016 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/_base.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/_base.py
rename to synapse/rest/client/_base.py
index f016b4f1bd..0443f4571c 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/account.py
similarity index 97%
rename from synapse/rest/client/v2_alpha/account.py
rename to synapse/rest/client/account.py
index 80ee0d2d8e..69e3083fea 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/account.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018, 2019 New Vector Ltd
@@ -18,7 +17,7 @@
 import random
 import re
 from http import HTTPStatus
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 from urllib.parse import urlparse
 
 from synapse.api.constants import LoginType
@@ -42,7 +41,7 @@
 from synapse.types import UserID
 from synapse.util.msisdn import phone_number_to_msisdn
 from synapse.util.stringutils import assert_valid_client_secret, random_string
-from synapse.util.threepids import canonicalise_email, check_3pid_allowed
+from synapse.util.threepids import check_3pid_allowed, validate_email
 
 from ._base import client_patterns, interactive_auth_handler
 
@@ -95,7 +94,7 @@ async def on_POST(self, request):
         # Stored in the database "foo@bar.com"
         # User requests with "FOO@bar.com" would raise a Not Found error
         try:
-            email = canonicalise_email(body["email"])
+            email = validate_email(body["email"])
         except ValueError as e:
             raise SynapseError(400, str(e))
         send_attempt = body["send_attempt"]
@@ -105,7 +104,9 @@ async def on_POST(self, request):
             # Raise if the provided next_link value isn't valid
             assert_valid_next_link(self.hs, next_link)
 
-        self.identity_handler.ratelimit_request_token_requests(request, "email", email)
+        await self.identity_handler.ratelimit_request_token_requests(
+            request, "email", email
+        )
 
         # The email will be sent to the stored address.
         # This avoids a potential account hijack by requesting a password reset to
@@ -254,7 +255,7 @@ async def on_POST(self, request):
                     # We store all email addresses canonicalised in the DB.
                     # (See add_threepid in synapse/handlers/auth.py)
                     try:
-                        threepid["address"] = canonicalise_email(threepid["address"])
+                        threepid["address"] = validate_email(threepid["address"])
                     except ValueError as e:
                         raise SynapseError(400, str(e))
                 # if using email, we must know about the email they're authing with!
@@ -402,7 +403,7 @@ async def on_POST(self, request):
         # Otherwise the email will be sent to "FOO@bar.com" and stored as
         # "foo@bar.com" in database.
         try:
-            email = canonicalise_email(body["email"])
+            email = validate_email(body["email"])
         except ValueError as e:
             raise SynapseError(400, str(e))
         send_attempt = body["send_attempt"]
@@ -415,7 +416,9 @@ async def on_POST(self, request):
                 Codes.THREEPID_DENIED,
             )
 
-        self.identity_handler.ratelimit_request_token_requests(request, "email", email)
+        await self.identity_handler.ratelimit_request_token_requests(
+            request, "email", email
+        )
 
         if next_link:
             # Raise if the provided next_link value isn't valid
@@ -496,7 +499,7 @@ async def on_POST(self, request):
                 Codes.THREEPID_DENIED,
             )
 
-        self.identity_handler.ratelimit_request_token_requests(
+        await self.identity_handler.ratelimit_request_token_requests(
             request, "msisdn", msisdn
         )
 
@@ -689,7 +692,10 @@ async def on_POST(self, request):
             # This makes the API entirely change shape when we have an AS token;
             # it really should be an entirely separate API - perhaps
             # /account/3pid/replicate or something.
-            threepid = body.get("threepid")
+            threepid: Optional[dict] = body.get("threepid")
+
+            if not threepid:
+                raise SynapseError(400, "Missing param 'threepid'")
 
             await self.auth_handler.add_threepid(
                 user_id,
@@ -1053,7 +1059,14 @@ def __init__(self, hs):
     async def on_GET(self, request):
         requester = await self.auth.get_user_by_req(request)
 
-        return 200, {"user_id": requester.user.to_string()}
+        response = {"user_id": requester.user.to_string()}
+
+        # Appservices and similar accounts do not have device IDs
+        # that we can report on, so exclude them for compliance.
+        if requester.device_id is not None:
+            response["device_id"] = requester.device_id
+
+        return 200, response
 
 
 def register_servlets(hs, http_server):
diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/account_data.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/account_data.py
rename to synapse/rest/client/account_data.py
index c9f13e4ac5..6b038f5cc0 100644
--- a/synapse/rest/client/v2_alpha/account_data.py
+++ b/synapse/rest/client/account_data.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/account_validity.py b/synapse/rest/client/account_validity.py
similarity index 84%
rename from synapse/rest/client/v2_alpha/account_validity.py
rename to synapse/rest/client/account_validity.py
index 40c5bd4d8c..3ebe401861 100644
--- a/synapse/rest/client/v2_alpha/account_validity.py
+++ b/synapse/rest/client/account_validity.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +14,7 @@
 
 import logging
 
-from synapse.api.errors import AuthError, SynapseError
+from synapse.api.errors import SynapseError
 from synapse.http.server import respond_with_html
 from synapse.http.servlet import RestServlet
 
@@ -38,12 +37,14 @@ def __init__(self, hs):
         self.account_activity_handler = hs.get_account_validity_handler()
         self.auth = hs.get_auth()
         self.account_renewed_template = (
-            hs.config.account_validity_account_renewed_template
+            hs.config.account_validity.account_validity_account_renewed_template
         )
         self.account_previously_renewed_template = (
-            hs.config.account_validity_account_previously_renewed_template
+            hs.config.account_validity.account_validity_account_previously_renewed_template
+        )
+        self.invalid_token_template = (
+            hs.config.account_validity.account_validity_invalid_token_template
         )
-        self.invalid_token_template = hs.config.account_validity_invalid_token_template
 
     async def on_GET(self, request):
         if b"token" not in request.args:
@@ -87,15 +88,10 @@ def __init__(self, hs):
         self.account_activity_handler = hs.get_account_validity_handler()
         self.auth = hs.get_auth()
         self.account_validity_renew_by_email_enabled = (
-            self.hs.config.account_validity_renew_by_email_enabled
+            hs.config.account_validity.account_validity_renew_by_email_enabled
         )
 
     async def on_POST(self, request):
-        if not self.account_validity_renew_by_email_enabled:
-            raise AuthError(
-                403, "Account renewal via email is disabled on this server."
-            )
-
         requester = await self.auth.get_user_by_req(request, allow_expired=True)
         user_id = requester.user.to_string()
         await self.account_activity_handler.send_renewal_email_to_user(user_id)
diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/auth.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/auth.py
rename to synapse/rest/client/auth.py
index 75ece1c911..6ea1b50a62 100644
--- a/synapse/rest/client/v2_alpha/auth.py
+++ b/synapse/rest/client/auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/capabilities.py b/synapse/rest/client/capabilities.py
similarity index 87%
rename from synapse/rest/client/v2_alpha/capabilities.py
rename to synapse/rest/client/capabilities.py
index 44ccf10ed4..88e3aac797 100644
--- a/synapse/rest/client/v2_alpha/capabilities.py
+++ b/synapse/rest/client/capabilities.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +14,7 @@
 import logging
 from typing import TYPE_CHECKING, Tuple
 
-from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
+from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, MSC3244_CAPABILITIES
 from synapse.http.servlet import RestServlet
 from synapse.http.site import SynapseRequest
 from synapse.types import JsonDict
@@ -56,6 +55,12 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
                 "m.change_password": {"enabled": change_password},
             }
         }
+
+        if self.config.experimental.msc3244_enabled:
+            response["capabilities"]["m.room_versions"][
+                "org.matrix.msc3244.room_capabilities"
+            ] = MSC3244_CAPABILITIES
+
         return 200, response
 
 
diff --git a/synapse/rest/client/v2_alpha/devices.py b/synapse/rest/client/devices.py
similarity index 96%
rename from synapse/rest/client/v2_alpha/devices.py
rename to synapse/rest/client/devices.py
index 3d07aadd39..8b9674db06 100644
--- a/synapse/rest/client/v2_alpha/devices.py
+++ b/synapse/rest/client/devices.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -87,6 +86,9 @@ async def on_POST(self, request):
             request,
             body,
             "remove device(s) from your account",
+            # Users might call this multiple times in a row while cleaning up
+            # devices, allow a single UI auth session to be re-used.
+            can_skip_ui_auth=True,
         )
 
         await self.device_handler.delete_devices(
@@ -136,6 +138,9 @@ async def on_DELETE(self, request, device_id):
             request,
             body,
             "remove a device from your account",
+            # Users might call this multiple times in a row while cleaning up
+            # devices, allow a single UI auth session to be re-used.
+            can_skip_ui_auth=True,
         )
 
         await self.device_handler.delete_device(requester.user.to_string(), device_id)
diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/directory.py
similarity index 98%
rename from synapse/rest/client/v1/directory.py
rename to synapse/rest/client/directory.py
index e5af26b176..ffa075c8e5 100644
--- a/synapse/rest/client/v1/directory.py
+++ b/synapse/rest/client/directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,7 +23,7 @@
     SynapseError,
 )
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client._base import client_patterns
 from synapse.types import RoomAlias
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/events.py
similarity index 97%
rename from synapse/rest/client/v1/events.py
rename to synapse/rest/client/events.py
index 6de4078290..52bb579cfd 100644
--- a/synapse/rest/client/v1/events.py
+++ b/synapse/rest/client/events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +17,7 @@
 
 from synapse.api.errors import SynapseError
 from synapse.http.servlet import RestServlet
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client._base import client_patterns
 from synapse.streams.config import PaginationConfig
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/filter.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/filter.py
rename to synapse/rest/client/filter.py
index 7cc692643b..411667a9c8 100644
--- a/synapse/rest/client/v2_alpha/filter.py
+++ b/synapse/rest/client/filter.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/groups.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/groups.py
rename to synapse/rest/client/groups.py
index 08fb6b2b06..6285680c00 100644
--- a/synapse/rest/client/v2_alpha/groups.py
+++ b/synapse/rest/client/groups.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/initial_sync.py
similarity index 95%
rename from synapse/rest/client/v1/initial_sync.py
rename to synapse/rest/client/initial_sync.py
index 91da0ee573..12ba0e91db 100644
--- a/synapse/rest/client/v1/initial_sync.py
+++ b/synapse/rest/client/initial_sync.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +14,7 @@
 
 
 from synapse.http.servlet import RestServlet, parse_boolean
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client._base import client_patterns
 from synapse.streams.config import PaginationConfig
 
 
diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/keys.py
similarity index 96%
rename from synapse/rest/client/v2_alpha/keys.py
rename to synapse/rest/client/keys.py
index f092e5b3a2..d0d9d30d40 100644
--- a/synapse/rest/client/v2_alpha/keys.py
+++ b/synapse/rest/client/keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
@@ -161,9 +160,12 @@ def __init__(self, hs):
     async def on_POST(self, request):
         requester = await self.auth.get_user_by_req(request, allow_guest=True)
         user_id = requester.user.to_string()
+        device_id = requester.device_id
         timeout = parse_integer(request, "timeout", 10 * 1000)
         body = parse_json_object_from_request(request)
-        result = await self.e2e_keys_handler.query_devices(body, timeout, user_id)
+        result = await self.e2e_keys_handler.query_devices(
+            body, timeout, user_id, device_id
+        )
         return 200, result
 
 
@@ -192,7 +194,7 @@ def __init__(self, hs):
     async def on_GET(self, request):
         requester = await self.auth.get_user_by_req(request, allow_guest=True)
 
-        from_token_string = parse_string(request, "from")
+        from_token_string = parse_string(request, "from", required=True)
         set_tag("from", from_token_string)
 
         # We want to enforce they do pass us one, but we ignore it and return
@@ -275,6 +277,9 @@ async def on_POST(self, request):
             request,
             body,
             "add a device signing key to your account",
+            # Allow skipping of UI auth since this is frequently called directly
+            # after login and it is silly to ask users to re-auth immediately.
+            can_skip_ui_auth=True,
         )
 
         result = await self.e2e_keys_handler.upload_signing_keys_for_user(user_id, body)
diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/knock.py
similarity index 86%
rename from synapse/rest/client/v2_alpha/knock.py
rename to synapse/rest/client/knock.py
index 51dac88fc7..7d1bc40658 100644
--- a/synapse/rest/client/v2_alpha/knock.py
+++ b/synapse/rest/client/knock.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Sorunome
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -14,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import TYPE_CHECKING, Optional, Tuple
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
 
 from twisted.web.server import Request
 
@@ -23,7 +22,7 @@
 from synapse.http.servlet import (
     RestServlet,
     parse_json_object_from_request,
-    parse_list_from_args,
+    parse_strings_from_args,
 )
 from synapse.http.site import SynapseRequest
 from synapse.logging.opentracing import set_tag
@@ -40,12 +39,10 @@
 
 class KnockRoomAliasServlet(RestServlet):
     """
-    POST /xyz.amorgan.knock/{roomIdOrAlias}
+    POST /knock/{roomIdOrAlias}
     """
 
-    PATTERNS = client_patterns(
-        "/xyz.amorgan.knock/(?P<room_identifier>[^/]*)", releases=()
-    )
+    PATTERNS = client_patterns("/knock/(?P<room_identifier>[^/]*)")
 
     def __init__(self, hs: "HomeServer"):
         super().__init__()
@@ -68,10 +65,13 @@ async def on_POST(
 
         if RoomID.is_valid(room_identifier):
             room_id = room_identifier
-            try:
-                remote_room_hosts = parse_list_from_args(request.args, "server_name")  # type: ignore
-            except KeyError:
-                remote_room_hosts = None
+
+            # twisted.web.server.Request.args is incorrectly defined as Optional[Any]
+            args: Dict[bytes, List[bytes]] = request.args  # type: ignore
+
+            remote_room_hosts = parse_strings_from_args(
+                args, "server_name", required=False
+            )
         elif RoomAlias.is_valid(room_identifier):
             handler = self.room_member_handler
             room_alias = RoomAlias.from_string(room_identifier)
diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/login.py
similarity index 71%
rename from synapse/rest/client/v1/login.py
rename to synapse/rest/client/login.py
index e4c352f572..0c8d8967b7 100644
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/login.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +14,9 @@
 
 import logging
 import re
-from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Optional
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional
+
+from typing_extensions import TypedDict
 
 from synapse.api.errors import Codes, LoginError, SynapseError
 from synapse.api.ratelimiting import Ratelimiter
@@ -26,11 +27,14 @@
 from synapse.http.server import HttpServer, finish_request
 from synapse.http.servlet import (
     RestServlet,
+    assert_params_in_dict,
+    parse_boolean,
+    parse_bytes_from_args,
     parse_json_object_from_request,
     parse_string,
 )
 from synapse.http.site import SynapseRequest
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client._base import client_patterns
 from synapse.rest.well_known import WellKnownBuilder
 from synapse.types import JsonDict, UserID
 
@@ -40,6 +44,16 @@
 logger = logging.getLogger(__name__)
 
 
+class LoginResponse(TypedDict, total=False):
+    user_id: str
+    access_token: str
+    home_server: str
+    expires_in_ms: Optional[int]
+    refresh_token: Optional[str]
+    device_id: str
+    well_known: Optional[Dict[str, Any]]
+
+
 class LoginRestServlet(RestServlet):
     PATTERNS = client_patterns("/login$", v1=True)
     CAS_TYPE = "m.login.cas"
@@ -48,6 +62,7 @@ class LoginRestServlet(RestServlet):
     JWT_TYPE = "org.matrix.login.jwt"
     JWT_TYPE_DEPRECATED = "m.login.jwt"
     APPSERVICE_TYPE = "uk.half-shot.msc2778.login.application_service"
+    REFRESH_TOKEN_PARAM = "org.matrix.msc2918.refresh_token"
 
     def __init__(self, hs: "HomeServer"):
         super().__init__()
@@ -65,20 +80,25 @@ def __init__(self, hs: "HomeServer"):
         self.cas_enabled = hs.config.cas_enabled
         self.oidc_enabled = hs.config.oidc_enabled
         self._msc2858_enabled = hs.config.experimental.msc2858_enabled
+        self._msc2918_enabled = hs.config.access_token_lifetime is not None
 
         self.auth = hs.get_auth()
 
+        self.clock = hs.get_clock()
+
         self.auth_handler = self.hs.get_auth_handler()
         self.registration_handler = hs.get_registration_handler()
         self._sso_handler = hs.get_sso_handler()
 
         self._well_known_builder = WellKnownBuilder(hs)
         self._address_ratelimiter = Ratelimiter(
+            store=hs.get_datastore(),
             clock=hs.get_clock(),
             rate_hz=self.hs.config.rc_login_address.per_second,
             burst_count=self.hs.config.rc_login_address.burst_count,
         )
         self._account_ratelimiter = Ratelimiter(
+            store=hs.get_datastore(),
             clock=hs.get_clock(),
             rate_hz=self.hs.config.rc_login_account.per_second,
             burst_count=self.hs.config.rc_login_account.burst_count,
@@ -96,7 +116,7 @@ def on_GET(self, request: SynapseRequest):
             flows.append({"type": LoginRestServlet.CAS_TYPE})
 
         if self.cas_enabled or self.saml2_enabled or self.oidc_enabled:
-            sso_flow = {
+            sso_flow: JsonDict = {
                 "type": LoginRestServlet.SSO_TYPE,
                 "identity_providers": [
                     _get_auth_flow_dict_for_idp(
@@ -104,7 +124,7 @@ def on_GET(self, request: SynapseRequest):
                     )
                     for idp in self._sso_handler.get_identity_providers().values()
                 ],
-            }  # type: JsonDict
+            }
 
             if self._msc2858_enabled:
                 # backwards-compatibility support for clients which don't
@@ -125,9 +145,7 @@ def on_GET(self, request: SynapseRequest):
             # login flow types returned.
             flows.append({"type": LoginRestServlet.TOKEN_TYPE})
 
-        flows.extend(
-            ({"type": t} for t in self.auth_handler.get_supported_login_types())
-        )
+        flows.extend({"type": t} for t in self.auth_handler.get_supported_login_types())
 
         flows.append({"type": LoginRestServlet.APPSERVICE_TYPE})
 
@@ -136,26 +154,50 @@ def on_GET(self, request: SynapseRequest):
     async def on_POST(self, request: SynapseRequest):
         login_submission = parse_json_object_from_request(request)
 
+        if self._msc2918_enabled:
+            # Check if this login should also issue a refresh token, as per
+            # MSC2918
+            should_issue_refresh_token = parse_boolean(
+                request, name=LoginRestServlet.REFRESH_TOKEN_PARAM, default=False
+            )
+        else:
+            should_issue_refresh_token = False
+
         try:
             if login_submission["type"] == LoginRestServlet.APPSERVICE_TYPE:
                 appservice = self.auth.get_appservice_by_req(request)
 
                 if appservice.is_rate_limited():
-                    self._address_ratelimiter.ratelimit(request.getClientIP())
+                    await self._address_ratelimiter.ratelimit(
+                        None, request.getClientIP()
+                    )
 
-                result = await self._do_appservice_login(login_submission, appservice)
+                result = await self._do_appservice_login(
+                    login_submission,
+                    appservice,
+                    should_issue_refresh_token=should_issue_refresh_token,
+                )
             elif self.jwt_enabled and (
                 login_submission["type"] == LoginRestServlet.JWT_TYPE
                 or login_submission["type"] == LoginRestServlet.JWT_TYPE_DEPRECATED
             ):
-                self._address_ratelimiter.ratelimit(request.getClientIP())
-                result = await self._do_jwt_login(login_submission)
+                await self._address_ratelimiter.ratelimit(None, request.getClientIP())
+                result = await self._do_jwt_login(
+                    login_submission,
+                    should_issue_refresh_token=should_issue_refresh_token,
+                )
             elif login_submission["type"] == LoginRestServlet.TOKEN_TYPE:
-                self._address_ratelimiter.ratelimit(request.getClientIP())
-                result = await self._do_token_login(login_submission)
+                await self._address_ratelimiter.ratelimit(None, request.getClientIP())
+                result = await self._do_token_login(
+                    login_submission,
+                    should_issue_refresh_token=should_issue_refresh_token,
+                )
             else:
-                self._address_ratelimiter.ratelimit(request.getClientIP())
-                result = await self._do_other_login(login_submission)
+                await self._address_ratelimiter.ratelimit(None, request.getClientIP())
+                result = await self._do_other_login(
+                    login_submission,
+                    should_issue_refresh_token=should_issue_refresh_token,
+                )
         except KeyError:
             raise SynapseError(400, "Missing JSON keys.")
 
@@ -165,7 +207,10 @@ async def on_POST(self, request: SynapseRequest):
         return 200, result
 
     async def _do_appservice_login(
-        self, login_submission: JsonDict, appservice: ApplicationService
+        self,
+        login_submission: JsonDict,
+        appservice: ApplicationService,
+        should_issue_refresh_token: bool = False,
     ):
         identifier = login_submission.get("identifier")
         logger.info("Got appservice login request with identifier: %r", identifier)
@@ -194,14 +239,21 @@ async def _do_appservice_login(
             raise LoginError(403, "Invalid access_token", errcode=Codes.FORBIDDEN)
 
         return await self._complete_login(
-            qualified_user_id, login_submission, ratelimit=appservice.is_rate_limited()
+            qualified_user_id,
+            login_submission,
+            ratelimit=appservice.is_rate_limited(),
+            should_issue_refresh_token=should_issue_refresh_token,
         )
 
-    async def _do_other_login(self, login_submission: JsonDict) -> Dict[str, str]:
+    async def _do_other_login(
+        self, login_submission: JsonDict, should_issue_refresh_token: bool = False
+    ) -> LoginResponse:
         """Handle non-token/saml/jwt logins
 
         Args:
             login_submission:
+            should_issue_refresh_token: True if this login should issue
+                a refresh token alongside the access token.
 
         Returns:
             HTTP response
@@ -220,7 +272,10 @@ async def _do_other_login(self, login_submission: JsonDict) -> Dict[str, str]:
             login_submission, ratelimit=True
         )
         result = await self._complete_login(
-            canonical_user_id, login_submission, callback
+            canonical_user_id,
+            login_submission,
+            callback,
+            should_issue_refresh_token=should_issue_refresh_token,
         )
         return result
 
@@ -228,11 +283,12 @@ async def _complete_login(
         self,
         user_id: str,
         login_submission: JsonDict,
-        callback: Optional[Callable[[Dict[str, str]], Awaitable[None]]] = None,
+        callback: Optional[Callable[[LoginResponse], Awaitable[None]]] = None,
         create_non_existent_users: bool = False,
         ratelimit: bool = True,
         auth_provider_id: Optional[str] = None,
-    ) -> Dict[str, str]:
+        should_issue_refresh_token: bool = False,
+    ) -> LoginResponse:
         """Called when we've successfully authed the user and now need to
         actually login them in (e.g. create devices). This gets called on
         all successful logins.
@@ -249,6 +305,8 @@ async def _complete_login(
             ratelimit: Whether to ratelimit the login request.
             auth_provider_id: The SSO IdP the user used, if any (just used for the
                 prometheus metrics).
+            should_issue_refresh_token: True if this login should issue
+                a refresh token alongside the access token.
 
         Returns:
             result: Dictionary of account information after successful login.
@@ -258,7 +316,7 @@ async def _complete_login(
         # too often. This happens here rather than before as we don't
         # necessarily know the user before now.
         if ratelimit:
-            self._account_ratelimiter.ratelimit(user_id.lower())
+            await self._account_ratelimiter.ratelimit(None, user_id.lower())
 
         if create_non_existent_users:
             canonical_uid = await self.auth_handler.check_user_exists(user_id)
@@ -270,28 +328,48 @@ async def _complete_login(
 
         device_id = login_submission.get("device_id")
         initial_display_name = login_submission.get("initial_device_display_name")
-        device_id, access_token = await self.registration_handler.register_device(
-            user_id, device_id, initial_display_name, auth_provider_id=auth_provider_id
+        (
+            device_id,
+            access_token,
+            valid_until_ms,
+            refresh_token,
+        ) = await self.registration_handler.register_device(
+            user_id,
+            device_id,
+            initial_display_name,
+            auth_provider_id=auth_provider_id,
+            should_issue_refresh_token=should_issue_refresh_token,
         )
 
-        result = {
-            "user_id": user_id,
-            "access_token": access_token,
-            "home_server": self.hs.hostname,
-            "device_id": device_id,
-        }
+        result = LoginResponse(
+            user_id=user_id,
+            access_token=access_token,
+            home_server=self.hs.hostname,
+            device_id=device_id,
+        )
+
+        if valid_until_ms is not None:
+            expires_in_ms = valid_until_ms - self.clock.time_msec()
+            result["expires_in_ms"] = expires_in_ms
+
+        if refresh_token is not None:
+            result["refresh_token"] = refresh_token
 
         if callback is not None:
             await callback(result)
 
         return result
 
-    async def _do_token_login(self, login_submission: JsonDict) -> Dict[str, str]:
+    async def _do_token_login(
+        self, login_submission: JsonDict, should_issue_refresh_token: bool = False
+    ) -> LoginResponse:
         """
         Handle the final stage of SSO login.
 
         Args:
-             login_submission: The JSON request body.
+            login_submission: The JSON request body.
+            should_issue_refresh_token: True if this login should issue
+                a refresh token alongside the access token.
 
         Returns:
             The body of the JSON response.
@@ -305,9 +383,12 @@ async def _do_token_login(self, login_submission: JsonDict) -> Dict[str, str]:
             login_submission,
             self.auth_handler._sso_login_callback,
             auth_provider_id=res.auth_provider_id,
+            should_issue_refresh_token=should_issue_refresh_token,
         )
 
-    async def _do_jwt_login(self, login_submission: JsonDict) -> Dict[str, str]:
+    async def _do_jwt_login(
+        self, login_submission: JsonDict, should_issue_refresh_token: bool = False
+    ) -> LoginResponse:
         token = login_submission.get("token", None)
         if token is None:
             raise LoginError(
@@ -338,7 +419,10 @@ async def _do_jwt_login(self, login_submission: JsonDict) -> Dict[str, str]:
 
         user_id = UserID(user, self.hs.hostname).to_string()
         result = await self._complete_login(
-            user_id, login_submission, create_non_existent_users=True
+            user_id,
+            login_submission,
+            create_non_existent_users=True,
+            should_issue_refresh_token=should_issue_refresh_token,
         )
         return result
 
@@ -356,7 +440,7 @@ def _get_auth_flow_dict_for_idp(
         use_unstable_brands: whether we should use brand identifiers suitable
            for the unstable API
     """
-    e = {"id": idp.idp_id, "name": idp.idp_name}  # type: JsonDict
+    e: JsonDict = {"id": idp.idp_id, "name": idp.idp_name}
     if idp.idp_icon:
         e["icon"] = idp.idp_icon
     if idp.idp_brand:
@@ -367,6 +451,42 @@ def _get_auth_flow_dict_for_idp(
     return e
 
 
+class RefreshTokenServlet(RestServlet):
+    PATTERNS = client_patterns(
+        "/org.matrix.msc2918.refresh_token/refresh$", releases=(), unstable=True
+    )
+
+    def __init__(self, hs: "HomeServer"):
+        self._auth_handler = hs.get_auth_handler()
+        self._clock = hs.get_clock()
+        self.access_token_lifetime = hs.config.access_token_lifetime
+
+    async def on_POST(
+        self,
+        request: SynapseRequest,
+    ):
+        refresh_submission = parse_json_object_from_request(request)
+
+        assert_params_in_dict(refresh_submission, ["refresh_token"])
+        token = refresh_submission["refresh_token"]
+        if not isinstance(token, str):
+            raise SynapseError(400, "Invalid param: refresh_token", Codes.INVALID_PARAM)
+
+        valid_until_ms = self._clock.time_msec() + self.access_token_lifetime
+        access_token, refresh_token = await self._auth_handler.refresh_token(
+            token, valid_until_ms
+        )
+        expires_in_ms = valid_until_ms - self._clock.time_msec()
+        return (
+            200,
+            {
+                "access_token": access_token,
+                "refresh_token": refresh_token,
+                "expires_in_ms": expires_in_ms,
+            },
+        )
+
+
 class SsoRedirectServlet(RestServlet):
     PATTERNS = list(client_patterns("/login/(cas|sso)/redirect$", v1=True)) + [
         re.compile(
@@ -434,9 +554,8 @@ async def on_GET(
             finish_request(request)
             return
 
-        client_redirect_url = parse_string(
-            request, "redirectUrl", required=True, encoding=None
-        )
+        args: Dict[bytes, List[bytes]] = request.args  # type: ignore
+        client_redirect_url = parse_bytes_from_args(args, "redirectUrl", required=True)
         sso_url = await self._sso_handler.handle_redirect_request(
             request,
             client_redirect_url,
@@ -474,6 +593,8 @@ async def on_GET(self, request: SynapseRequest) -> None:
 
 def register_servlets(hs, http_server):
     LoginRestServlet(hs).register(http_server)
+    if hs.config.access_token_lifetime is not None:
+        RefreshTokenServlet(hs).register(http_server)
     SsoRedirectServlet(hs).register(http_server)
     if hs.config.cas_enabled:
         CasTicketServlet(hs).register(http_server)
diff --git a/synapse/rest/client/v1/logout.py b/synapse/rest/client/logout.py
similarity index 96%
rename from synapse/rest/client/v1/logout.py
rename to synapse/rest/client/logout.py
index ad8cea49c6..6055cac2bd 100644
--- a/synapse/rest/client/v1/logout.py
+++ b/synapse/rest/client/logout.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,7 +15,7 @@
 import logging
 
 from synapse.http.servlet import RestServlet
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client._base import client_patterns
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/rest/client/v2_alpha/notifications.py b/synapse/rest/client/notifications.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/notifications.py
rename to synapse/rest/client/notifications.py
index 87063ec8b1..0ede643c2d 100644
--- a/synapse/rest/client/v2_alpha/notifications.py
+++ b/synapse/rest/client/notifications.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/openid.py b/synapse/rest/client/openid.py
similarity index 97%
rename from synapse/rest/client/v2_alpha/openid.py
rename to synapse/rest/client/openid.py
index 5b996e2d63..e8d2673819 100644
--- a/synapse/rest/client/v2_alpha/openid.py
+++ b/synapse/rest/client/openid.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -86,7 +85,7 @@ async def on_POST(self, request, user_id):
                 "access_token": token,
                 "token_type": "Bearer",
                 "matrix_server_name": self.server_name,
-                "expires_in": self.EXPIRES_MS / 1000,
+                "expires_in": self.EXPIRES_MS // 1000,
             },
         )
 
diff --git a/synapse/rest/client/v2_alpha/password_policy.py b/synapse/rest/client/password_policy.py
similarity index 98%
rename from synapse/rest/client/v2_alpha/password_policy.py
rename to synapse/rest/client/password_policy.py
index 68b27ff23a..a83927aee6 100644
--- a/synapse/rest/client/v2_alpha/password_policy.py
+++ b/synapse/rest/client/password_policy.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/presence.py
similarity index 92%
rename from synapse/rest/client/v1/presence.py
rename to synapse/rest/client/presence.py
index 94bfe2d1b0..50f2b78aab 100644
--- a/synapse/rest/client/v1/presence.py
+++ b/synapse/rest/client/presence.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,7 +19,7 @@
 from synapse.api.errors import AuthError, SynapseError
 from synapse.handlers.presence import format_user_presence_state
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client._base import client_patterns
 from synapse.types import UserID
 
 logger = logging.getLogger(__name__)
@@ -36,10 +35,15 @@ def __init__(self, hs):
         self.clock = hs.get_clock()
         self.auth = hs.get_auth()
 
+        self._use_presence = hs.config.server.use_presence
+
     async def on_GET(self, request, user_id):
         requester = await self.auth.get_user_by_req(request)
         user = UserID.from_string(user_id)
 
+        if not self._use_presence:
+            return 200, {"presence": "offline"}
+
         if requester.user != user:
             allowed = await self.presence_handler.is_visible(
                 observed_user=user, observer_user=requester.user
@@ -79,7 +83,7 @@ async def on_PUT(self, request, user_id):
         except Exception:
             raise SynapseError(400, "Unable to parse state")
 
-        if self.hs.config.use_presence:
+        if self._use_presence:
             await self.presence_handler.set_state(user, state)
 
         return 200, {}
diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/profile.py
similarity index 98%
rename from synapse/rest/client/v1/profile.py
rename to synapse/rest/client/profile.py
index d77e20e135..24c0b28b43 100644
--- a/synapse/rest/client/v1/profile.py
+++ b/synapse/rest/client/profile.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +17,7 @@
 
 from synapse.api.errors import Codes, SynapseError
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client._base import client_patterns
 from synapse.types import UserID
 
 
diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/push_rule.py
similarity index 99%
rename from synapse/rest/client/v1/push_rule.py
rename to synapse/rest/client/push_rule.py
index 241e535917..702b351d18 100644
--- a/synapse/rest/client/v1/push_rule.py
+++ b/synapse/rest/client/push_rule.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,7 +26,7 @@
 from synapse.push.baserules import BASE_RULE_IDS, NEW_RULE_IDS
 from synapse.push.clientformat import format_push_rules_for_user
 from synapse.push.rulekinds import PRIORITY_CLASS_MAP
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client._base import client_patterns
 from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException
 
 
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/pusher.py
similarity index 98%
rename from synapse/rest/client/v1/pusher.py
rename to synapse/rest/client/pusher.py
index 0c148a213d..84619c5e41 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/pusher.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,7 +23,7 @@
     parse_string,
 )
 from synapse.push import PusherConfigException
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client._base import client_patterns
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/rest/client/v2_alpha/read_marker.py b/synapse/rest/client/read_marker.py
similarity index 81%
rename from synapse/rest/client/v2_alpha/read_marker.py
rename to synapse/rest/client/read_marker.py
index 55c6688f52..027f8b81fa 100644
--- a/synapse/rest/client/v2_alpha/read_marker.py
+++ b/synapse/rest/client/read_marker.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,6 +14,8 @@
 
 import logging
 
+from synapse.api.constants import ReadReceiptEventFields
+from synapse.api.errors import Codes, SynapseError
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 
 from ._base import client_patterns
@@ -38,14 +39,24 @@ async def on_POST(self, request, room_id):
         await self.presence_handler.bump_presence_active_time(requester.user)
 
         body = parse_json_object_from_request(request)
-
         read_event_id = body.get("m.read", None)
+        hidden = body.get(ReadReceiptEventFields.MSC2285_HIDDEN, False)
+
+        if not isinstance(hidden, bool):
+            raise SynapseError(
+                400,
+                "Param %s must be a boolean, if given"
+                % ReadReceiptEventFields.MSC2285_HIDDEN,
+                Codes.BAD_JSON,
+            )
+
         if read_event_id:
             await self.receipts_handler.received_client_receipt(
                 room_id,
                 "m.read",
                 user_id=requester.user.to_string(),
                 event_id=read_event_id,
+                hidden=hidden,
             )
 
         read_marker_event_id = body.get("m.fully_read", None)
diff --git a/synapse/rest/client/v2_alpha/receipts.py b/synapse/rest/client/receipts.py
similarity index 68%
rename from synapse/rest/client/v2_alpha/receipts.py
rename to synapse/rest/client/receipts.py
index 6f7246a394..d9ab836cd8 100644
--- a/synapse/rest/client/v2_alpha/receipts.py
+++ b/synapse/rest/client/receipts.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,8 +14,9 @@
 
 import logging
 
-from synapse.api.errors import SynapseError
-from synapse.http.servlet import RestServlet
+from synapse.api.constants import ReadReceiptEventFields
+from synapse.api.errors import Codes, SynapseError
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
 
 from ._base import client_patterns
 
@@ -43,10 +43,25 @@ async def on_POST(self, request, room_id, receipt_type, event_id):
         if receipt_type != "m.read":
             raise SynapseError(400, "Receipt type must be 'm.read'")
 
+        body = parse_json_object_from_request(request, allow_empty_body=True)
+        hidden = body.get(ReadReceiptEventFields.MSC2285_HIDDEN, False)
+
+        if not isinstance(hidden, bool):
+            raise SynapseError(
+                400,
+                "Param %s must be a boolean, if given"
+                % ReadReceiptEventFields.MSC2285_HIDDEN,
+                Codes.BAD_JSON,
+            )
+
         await self.presence_handler.bump_presence_active_time(requester.user)
 
         await self.receipts_handler.received_client_receipt(
-            room_id, receipt_type, user_id=requester.user.to_string(), event_id=event_id
+            room_id,
+            receipt_type,
+            user_id=requester.user.to_string(),
+            event_id=event_id,
+            hidden=hidden,
         )
 
         return 200, {}
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/register.py
similarity index 90%
rename from synapse/rest/client/v2_alpha/register.py
rename to synapse/rest/client/register.py
index a7aea914e9..9562d6bc3a 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/register.py
@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2015-2016 OpenMarket Ltd
-# Copyright 2017-2018 New Vector Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import hmac
 import logging
 import random
@@ -24,7 +21,7 @@
 import synapse
 import synapse.api.auth
 import synapse.types
-from synapse.api.constants import LoginType
+from synapse.api.constants import APP_SERVICE_REGISTRATION_TYPE, LoginType
 from synapse.api.errors import (
     Codes,
     InteractiveAuthIncompleteError,
@@ -34,7 +31,7 @@
 )
 from synapse.config import ConfigError
 from synapse.config.captcha import CaptchaConfig
-from synapse.config.consent_config import ConsentConfig
+from synapse.config.consent import ConsentConfig
 from synapse.config.emailconfig import ThreepidBehaviour
 from synapse.config.ratelimiting import FederationRateLimitConfig
 from synapse.config.registration import RegistrationConfig
@@ -45,15 +42,21 @@
 from synapse.http.servlet import (
     RestServlet,
     assert_params_in_dict,
+    parse_boolean,
     parse_json_object_from_request,
     parse_string,
 )
 from synapse.metrics import threepid_send_requests
 from synapse.push.mailer import Mailer
+from synapse.types import JsonDict
 from synapse.util.msisdn import phone_number_to_msisdn
 from synapse.util.ratelimitutils import FederationRateLimiter
 from synapse.util.stringutils import assert_valid_client_secret, random_string
-from synapse.util.threepids import canonicalise_email, check_3pid_allowed
+from synapse.util.threepids import (
+    canonicalise_email,
+    check_3pid_allowed,
+    validate_email,
+)
 
 from ._base import client_patterns, interactive_auth_handler
 
@@ -113,9 +116,9 @@ async def on_POST(self, request):
         # For emails, canonicalise the address.
         # We store all email addresses canonicalised in the DB.
         # (See on_POST in EmailThreepidRequestTokenRestServlet
-        # in synapse/rest/client/v2_alpha/account.py)
+        # in synapse/rest/client/account.py)
         try:
-            email = canonicalise_email(body["email"])
+            email = validate_email(body["email"])
         except ValueError as e:
             raise SynapseError(400, str(e))
         send_attempt = body["send_attempt"]
@@ -128,7 +131,9 @@ async def on_POST(self, request):
                 Codes.THREEPID_DENIED,
             )
 
-        self.identity_handler.ratelimit_request_token_requests(request, "email", email)
+        await self.identity_handler.ratelimit_request_token_requests(
+            request, "email", email
+        )
 
         existing_user_id = await self.hs.get_datastore().get_user_id_by_threepid(
             "email", email
@@ -212,7 +217,7 @@ async def on_POST(self, request):
                 Codes.THREEPID_DENIED,
             )
 
-        self.identity_handler.ratelimit_request_token_requests(
+        await self.identity_handler.ratelimit_request_token_requests(
             request, "msisdn", msisdn
         )
 
@@ -393,6 +398,7 @@ def __init__(self, hs):
         self.password_policy_handler = hs.get_password_policy_handler()
         self.clock = hs.get_clock()
         self._registration_enabled = self.hs.config.enable_registration
+        self._msc2918_enabled = hs.config.access_token_lifetime is not None
 
         self._registration_flows = _calculate_registration_flows(
             hs.config, self.auth_handler
@@ -404,7 +410,7 @@ async def on_POST(self, request):
 
         client_addr = request.getClientIP()
 
-        self.ratelimiter.ratelimit(client_addr, update=False)
+        await self.ratelimiter.ratelimit(None, client_addr, update=False)
 
         kind = b"user"
         if b"kind" in request.args:
@@ -418,6 +424,15 @@ async def on_POST(self, request):
                 "Do not understand membership kind: %s" % (kind.decode("utf8"),)
             )
 
+        if self._msc2918_enabled:
+            # Check if this registration should also issue a refresh token, as
+            # per MSC2918
+            should_issue_refresh_token = parse_boolean(
+                request, name="org.matrix.msc2918.refresh_token", default=False
+            )
+        else:
+            should_issue_refresh_token = False
+
         # We don't care about usernames for this deployment. In fact, the act
         # of checking whether they exist already can leak metadata about
         # which users are already registered.
@@ -430,10 +445,6 @@ async def on_POST(self, request):
 
         desired_display_name = body.get("display_name")
 
-        appservice = None
-        if self.auth.has_access_token(request):
-            appservice = self.auth.get_appservice_by_req(request)
-
         # We need to retrieve the password early in order to pass it to
         # application service registration
         # This is specific to shadow server registration of users via an AS
@@ -443,7 +454,16 @@ async def on_POST(self, request):
         # different registration flows to normal users
 
         # == Application Service Registration ==
-        if appservice:
+        if body.get("type") == APP_SERVICE_REGISTRATION_TYPE:
+            if not self.auth.has_access_token(request):
+                raise SynapseError(
+                    400,
+                    "Appservice token must be provided when using a type of m.login.application_service",
+                )
+
+            # Verify the AS
+            self.auth.get_appservice_by_req(request)
+
             # Set the desired user according to the AS API (which uses the
             # 'user' key not 'username'). Since this is a new addition, we'll
             # fallback to 'username' if they gave one.
@@ -460,10 +480,20 @@ async def on_POST(self, request):
                 raise SynapseError(400, "Desired Username is missing or not a string")
 
             result = await self._do_appservice_registration(
-                desired_username, password, desired_display_name, access_token, body
+                desired_username,
+                password,
+                desired_display_name,
+                access_token,
+                body,
+                should_issue_refresh_token=should_issue_refresh_token,
             )
 
             return 200, result
+        elif self.auth.has_access_token(request):
+            raise SynapseError(
+                400,
+                "An access token should not be provided on requests to /register (except if type is m.login.application_service)",
+            )
 
         # == Normal User Registration == (everyone else)
         if not self._registration_enabled:
@@ -669,7 +699,7 @@ async def on_POST(self, request):
                         # For emails, canonicalise the address.
                         # We store all email addresses canonicalised in the DB.
                         # (See on_POST in EmailThreepidRequestTokenRestServlet
-                        # in synapse/rest/client/v2_alpha/account.py)
+                        # in synapse/rest/client/account.py)
                         if medium == "email":
                             try:
                                 address = canonicalise_email(address)
@@ -727,7 +757,9 @@ async def on_POST(self, request):
             registered = True
 
         return_dict = await self._create_registration_details(
-            registered_user_id, params
+            registered_user_id,
+            params,
+            should_issue_refresh_token=should_issue_refresh_token,
         )
 
         if registered:
@@ -740,17 +772,21 @@ async def on_POST(self, request):
         return 200, return_dict
 
     async def _do_appservice_registration(
-        self, username, password, display_name, as_token, body
+        self,
+        username,
+        password,
+        display_name,
+        as_token,
+        body,
+        should_issue_refresh_token: bool = False,
     ):
-        # FIXME: appservice_register() is horribly duplicated with register()
-        # and they should probably just be combined together with a config flag.
-
         if password:
             # Hash the password
             #
             # In mainline hashing of the password was moved further on in the registration
             # flow, but we need it here for the AS use-case of shadow servers
             password = await self.auth_handler.hash(password)
+
         user_id = await self.registration_handler.appservice_register(
             username, as_token, password, display_name
         )
@@ -758,6 +794,7 @@ async def _do_appservice_registration(
             user_id,
             body,
             is_appservice_ghost=True,
+            should_issue_refresh_token=should_issue_refresh_token,
         )
 
         auth_result = body.get("auth_result")
@@ -776,16 +813,23 @@ async def _do_appservice_registration(
         return result
 
     async def _create_registration_details(
-        self, user_id, params, is_appservice_ghost=False
+        self,
+        user_id: str,
+        params: JsonDict,
+        is_appservice_ghost: bool = False,
+        should_issue_refresh_token: bool = False,
     ):
         """Complete registration of newly-registered user
 
         Allocates device_id if one was not given; also creates access_token.
 
         Args:
-            (str) user_id: full canonical @user:id
-            (object) params: registration parameters, from which we pull
-                device_id, initial_device_name and inhibit_login
+            user_id: full canonical @user:id
+            params: registration parameters, from which we pull device_id,
+                initial_device_name and inhibit_login
+            is_appservice_ghost
+            should_issue_refresh_token: True if this registration should issue
+                a refresh token alongside the access token.
         Returns:
              dictionary for response from /register
         """
@@ -793,15 +837,29 @@ async def _create_registration_details(
         if not params.get("inhibit_login", False):
             device_id = params.get("device_id")
             initial_display_name = params.get("initial_device_display_name")
-            device_id, access_token = await self.registration_handler.register_device(
+            (
+                device_id,
+                access_token,
+                valid_until_ms,
+                refresh_token,
+            ) = await self.registration_handler.register_device(
                 user_id,
                 device_id,
                 initial_display_name,
                 is_guest=False,
                 is_appservice_ghost=is_appservice_ghost,
+                should_issue_refresh_token=should_issue_refresh_token,
             )
 
             result.update({"access_token": access_token, "device_id": device_id})
+
+            if valid_until_ms is not None:
+                expires_in_ms = valid_until_ms - self.clock.time_msec()
+                result["expires_in_ms"] = expires_in_ms
+
+            if refresh_token is not None:
+                result["refresh_token"] = refresh_token
+
         return result
 
     async def _do_guest_registration(self, params, address=None):
@@ -815,19 +873,30 @@ async def _do_guest_registration(self, params, address=None):
         # we have nowhere to store it.
         device_id = synapse.api.auth.GUEST_DEVICE_ID
         initial_display_name = params.get("initial_device_display_name")
-        device_id, access_token = await self.registration_handler.register_device(
+        (
+            device_id,
+            access_token,
+            valid_until_ms,
+            refresh_token,
+        ) = await self.registration_handler.register_device(
             user_id, device_id, initial_display_name, is_guest=True
         )
 
-        return (
-            200,
-            {
-                "user_id": user_id,
-                "device_id": device_id,
-                "access_token": access_token,
-                "home_server": self.hs.hostname,
-            },
-        )
+        result = {
+            "user_id": user_id,
+            "device_id": device_id,
+            "access_token": access_token,
+            "home_server": self.hs.hostname,
+        }
+
+        if valid_until_ms is not None:
+            expires_in_ms = valid_until_ms - self.clock.time_msec()
+            result["expires_in_ms"] = expires_in_ms
+
+        if refresh_token is not None:
+            result["refresh_token"] = refresh_token
+
+        return 200, result
 
 
 def cap(name):
diff --git a/synapse/rest/client/v2_alpha/relations.py b/synapse/rest/client/relations.py
similarity index 94%
rename from synapse/rest/client/v2_alpha/relations.py
rename to synapse/rest/client/relations.py
index fe765da23c..0821cd285f 100644
--- a/synapse/rest/client/v2_alpha/relations.py
+++ b/synapse/rest/client/relations.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -159,19 +158,21 @@ async def on_GET(
         event = await self.event_handler.get_event(requester.user, room_id, parent_id)
 
         limit = parse_integer(request, "limit", default=5)
-        from_token = parse_string(request, "from")
-        to_token = parse_string(request, "to")
+        from_token_str = parse_string(request, "from")
+        to_token_str = parse_string(request, "to")
 
         if event.internal_metadata.is_redacted():
             # If the event is redacted, return an empty list of relations
             pagination_chunk = PaginationChunk(chunk=[])
         else:
             # Return the relations
-            if from_token:
-                from_token = RelationPaginationToken.from_string(from_token)
+            from_token = None
+            if from_token_str:
+                from_token = RelationPaginationToken.from_string(from_token_str)
 
-            if to_token:
-                to_token = RelationPaginationToken.from_string(to_token)
+            to_token = None
+            if to_token_str:
+                to_token = RelationPaginationToken.from_string(to_token_str)
 
             pagination_chunk = await self.store.get_relations_for_event(
                 event_id=parent_id,
@@ -257,19 +258,21 @@ async def on_GET(
             raise SynapseError(400, "Relation type must be 'annotation'")
 
         limit = parse_integer(request, "limit", default=5)
-        from_token = parse_string(request, "from")
-        to_token = parse_string(request, "to")
+        from_token_str = parse_string(request, "from")
+        to_token_str = parse_string(request, "to")
 
         if event.internal_metadata.is_redacted():
             # If the event is redacted, return an empty list of relations
             pagination_chunk = PaginationChunk(chunk=[])
         else:
             # Return the relations
-            if from_token:
-                from_token = AggregationPaginationToken.from_string(from_token)
+            from_token = None
+            if from_token_str:
+                from_token = AggregationPaginationToken.from_string(from_token_str)
 
-            if to_token:
-                to_token = AggregationPaginationToken.from_string(to_token)
+            to_token = None
+            if to_token_str:
+                to_token = AggregationPaginationToken.from_string(to_token_str)
 
             pagination_chunk = await self.store.get_aggregation_groups_for_event(
                 event_id=parent_id,
@@ -337,14 +340,16 @@ async def on_GET(self, request, room_id, parent_id, relation_type, event_type, k
             raise SynapseError(400, "Relation type must be 'annotation'")
 
         limit = parse_integer(request, "limit", default=5)
-        from_token = parse_string(request, "from")
-        to_token = parse_string(request, "to")
+        from_token_str = parse_string(request, "from")
+        to_token_str = parse_string(request, "to")
 
-        if from_token:
-            from_token = RelationPaginationToken.from_string(from_token)
+        from_token = None
+        if from_token_str:
+            from_token = RelationPaginationToken.from_string(from_token_str)
 
-        if to_token:
-            to_token = RelationPaginationToken.from_string(to_token)
+        to_token = None
+        if to_token_str:
+            to_token = RelationPaginationToken.from_string(to_token_str)
 
         result = await self.store.get_relations_for_event(
             event_id=parent_id,
diff --git a/synapse/rest/client/v2_alpha/report_event.py b/synapse/rest/client/report_event.py
similarity index 85%
rename from synapse/rest/client/v2_alpha/report_event.py
rename to synapse/rest/client/report_event.py
index 215d619ca1..07ea39a8a3 100644
--- a/synapse/rest/client/v2_alpha/report_event.py
+++ b/synapse/rest/client/report_event.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,11 +16,7 @@
 from http import HTTPStatus
 
 from synapse.api.errors import Codes, SynapseError
-from synapse.http.servlet import (
-    RestServlet,
-    assert_params_in_dict,
-    parse_json_object_from_request,
-)
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
 
 from ._base import client_patterns
 
@@ -43,15 +38,14 @@ async def on_POST(self, request, room_id, event_id):
         user_id = requester.user.to_string()
 
         body = parse_json_object_from_request(request)
-        assert_params_in_dict(body, ("reason", "score"))
 
-        if not isinstance(body["reason"], str):
+        if not isinstance(body.get("reason", ""), str):
             raise SynapseError(
                 HTTPStatus.BAD_REQUEST,
                 "Param 'reason' must be a string",
                 Codes.BAD_JSON,
             )
-        if not isinstance(body["score"], int):
+        if not isinstance(body.get("score", 0), int):
             raise SynapseError(
                 HTTPStatus.BAD_REQUEST,
                 "Param 'score' must be an integer",
@@ -62,7 +56,7 @@ async def on_POST(self, request, room_id, event_id):
             room_id=room_id,
             event_id=event_id,
             user_id=user_id,
-            reason=body["reason"],
+            reason=body.get("reason"),
             content=body,
             received_ts=self.clock.time_msec(),
         )
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/room.py
similarity index 84%
rename from synapse/rest/client/v1/room.py
rename to synapse/rest/client/room.py
index d01b738e32..98c3a5e334 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/room.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -17,43 +16,37 @@
 """ This module contains REST servlets to do with rooms: /rooms/<paths> """
 import logging
 import re
-from typing import TYPE_CHECKING, Optional, Tuple
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
 from urllib import parse as urlparse
 
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import (
     AuthError,
     Codes,
-    HttpResponseException,
     InvalidClientCredentialsError,
+    MissingClientTokenError,
     ShadowBanError,
     SynapseError,
 )
 from synapse.api.filtering import Filter
 from synapse.events.utils import format_event_for_client_v2
 from synapse.http.servlet import (
+    ResolveRoomIdMixin,
     RestServlet,
     assert_params_in_dict,
     parse_boolean,
     parse_integer,
     parse_json_object_from_request,
-    parse_list_from_args,
     parse_string,
+    parse_strings_from_args,
 )
 from synapse.http.site import SynapseRequest
 from synapse.logging.opentracing import set_tag
+from synapse.rest.client._base import client_patterns
 from synapse.rest.client.transactions import HttpTransactionCache
-from synapse.rest.client.v2_alpha._base import client_patterns
 from synapse.storage.state import StateFilter
 from synapse.streams.config import PaginationConfig
-from synapse.types import (
-    JsonDict,
-    RoomAlias,
-    RoomID,
-    StreamToken,
-    ThirdPartyInstanceID,
-    UserID,
-)
+from synapse.types import JsonDict, StreamToken, ThirdPartyInstanceID, UserID
 from synapse.util import json_decoder
 from synapse.util.stringutils import parse_and_validate_server_name, random_string
 
@@ -268,10 +261,10 @@ def on_PUT(self, request, room_id, event_type, txn_id):
 
 
 # TODO: Needs unit testing for room ID + alias joins
-class JoinRoomAliasServlet(TransactionRestServlet):
+class JoinRoomAliasServlet(ResolveRoomIdMixin, TransactionRestServlet):
     def __init__(self, hs):
         super().__init__(hs)
-        self.room_member_handler = hs.get_room_member_handler()
+        super(ResolveRoomIdMixin, self).__init__(hs)  # ensure the Mixin is set up
         self.auth = hs.get_auth()
 
     def register(self, http_server):
@@ -279,7 +272,12 @@ def register(self, http_server):
         PATTERNS = "/join/(?P<room_identifier>[^/]*)"
         register_txn_path(self, PATTERNS, http_server)
 
-    async def on_POST(self, request, room_identifier, txn_id=None):
+    async def on_POST(
+        self,
+        request: SynapseRequest,
+        room_identifier: str,
+        txn_id: Optional[str] = None,
+    ):
         requester = await self.auth.get_user_by_req(request, allow_guest=True)
 
         try:
@@ -289,21 +287,13 @@ async def on_POST(self, request, room_identifier, txn_id=None):
             # cheekily send invalid bodies.
             content = {}
 
-        if RoomID.is_valid(room_identifier):
-            room_id = room_identifier
-            try:
-                remote_room_hosts = parse_list_from_args(request.args, "server_name")
-            except KeyError:
-                remote_room_hosts = None
-        elif RoomAlias.is_valid(room_identifier):
-            handler = self.room_member_handler
-            room_alias = RoomAlias.from_string(room_identifier)
-            room_id, remote_room_hosts = await handler.lookup_room_alias(room_alias)
-            room_id = room_id.to_string()
-        else:
-            raise SynapseError(
-                400, "%s was not legal room ID or room alias" % (room_identifier,)
-            )
+        # twisted.web.server.Request.args is incorrectly defined as Optional[Any]
+        args: Dict[bytes, List[bytes]] = request.args  # type: ignore
+        remote_room_hosts = parse_strings_from_args(args, "server_name", required=False)
+        room_id, remote_room_hosts = await self.resolve_room_id(
+            room_identifier,
+            remote_room_hosts,
+        )
 
         await self.room_member_handler.update_membership(
             requester=requester,
@@ -336,7 +326,7 @@ def __init__(self, hs):
         self.auth = hs.get_auth()
 
     async def on_GET(self, request):
-        server = parse_string(request, "server", default=None)
+        server = parse_string(request, "server")
 
         try:
             await self.auth.get_user_by_req(request, allow_guest=True)
@@ -355,8 +345,8 @@ async def on_GET(self, request):
             if server:
                 raise e
 
-        limit = parse_integer(request, "limit", 0)
-        since_token = parse_string(request, "since", None)
+        limit: Optional[int] = parse_integer(request, "limit", 0)
+        since_token = parse_string(request, "since")
 
         if limit == 0:
             # zero is a special value which corresponds to no limit.
@@ -374,12 +364,9 @@ async def on_GET(self, request):
                     Codes.INVALID_PARAM,
                 )
 
-            try:
-                data = await handler.get_remote_public_room_list(
-                    server, limit=limit, since_token=since_token
-                )
-            except HttpResponseException as e:
-                raise e.to_synapse_error()
+            data = await handler.get_remote_public_room_list(
+                server, limit=limit, since_token=since_token
+            )
         else:
             data = await handler.get_local_public_room_list(
                 limit=limit, since_token=since_token
@@ -390,10 +377,10 @@ async def on_GET(self, request):
     async def on_POST(self, request):
         await self.auth.get_user_by_req(request, allow_guest=True)
 
-        server = parse_string(request, "server", default=None)
+        server = parse_string(request, "server")
         content = parse_json_object_from_request(request)
 
-        limit = int(content.get("limit", 100))  # type: Optional[int]
+        limit: Optional[int] = int(content.get("limit", 100))
         since_token = content.get("since", None)
         search_filter = content.get("filter", None)
 
@@ -427,17 +414,15 @@ async def on_POST(self, request):
                     Codes.INVALID_PARAM,
                 )
 
-            try:
-                data = await handler.get_remote_public_room_list(
-                    server,
-                    limit=limit,
-                    since_token=since_token,
-                    search_filter=search_filter,
-                    include_all_networks=include_all_networks,
-                    third_party_instance_id=third_party_instance_id,
-                )
-            except HttpResponseException as e:
-                raise e.to_synapse_error()
+            data = await handler.get_remote_public_room_list(
+                server,
+                limit=limit,
+                since_token=since_token,
+                search_filter=search_filter,
+                include_all_networks=include_all_networks,
+                third_party_instance_id=third_party_instance_id,
+            )
+
         else:
             data = await handler.get_local_public_room_list(
                 limit=limit,
@@ -536,12 +521,10 @@ async def on_GET(self, request, room_id):
             self.store, request, default_limit=10
         )
         as_client_event = b"raw" not in request.args
-        filter_str = parse_string(request, b"filter", encoding="utf-8")
+        filter_str = parse_string(request, "filter", encoding="utf-8")
         if filter_str:
             filter_json = urlparse.unquote(filter_str)
-            event_filter = Filter(
-                json_decoder.decode(filter_json)
-            )  # type: Optional[Filter]
+            event_filter: Optional[Filter] = Filter(json_decoder.decode(filter_json))
             if (
                 event_filter
                 and event_filter.filter_json.get("event_format", "client")
@@ -651,12 +634,10 @@ async def on_GET(self, request, room_id, event_id):
         limit = parse_integer(request, "limit", default=10)
 
         # picking the API shape for symmetry with /messages
-        filter_str = parse_string(request, b"filter", encoding="utf-8")
+        filter_str = parse_string(request, "filter", encoding="utf-8")
         if filter_str:
             filter_json = urlparse.unquote(filter_str)
-            event_filter = Filter(
-                json_decoder.decode(filter_json)
-            )  # type: Optional[Filter]
+            event_filter: Optional[Filter] = Filter(json_decoder.decode(filter_json))
         else:
             event_filter = None
 
@@ -910,7 +891,7 @@ class RoomAliasListServlet(RestServlet):
             r"^/_matrix/client/unstable/org\.matrix\.msc2432"
             r"/rooms/(?P<room_id>[^/]*)/aliases"
         ),
-    ]
+    ] + list(client_patterns("/rooms/(?P<room_id>[^/]*)/aliases$", unstable=False))
 
     def __init__(self, hs: "HomeServer"):
         super().__init__()
@@ -1006,20 +987,29 @@ class RoomSpaceSummaryRestServlet(RestServlet):
     def __init__(self, hs: "HomeServer"):
         super().__init__()
         self._auth = hs.get_auth()
-        self._space_summary_handler = hs.get_space_summary_handler()
+        self._room_summary_handler = hs.get_room_summary_handler()
 
     async def on_GET(
         self, request: SynapseRequest, room_id: str
     ) -> Tuple[int, JsonDict]:
         requester = await self._auth.get_user_by_req(request, allow_guest=True)
 
-        return 200, await self._space_summary_handler.get_space_summary(
+        max_rooms_per_space = parse_integer(request, "max_rooms_per_space")
+        if max_rooms_per_space is not None and max_rooms_per_space < 0:
+            raise SynapseError(
+                400,
+                "Value for 'max_rooms_per_space' must be a non-negative integer",
+                Codes.BAD_JSON,
+            )
+
+        return 200, await self._room_summary_handler.get_space_summary(
             requester.user.to_string(),
             room_id,
             suggested_only=parse_boolean(request, "suggested_only", default=False),
-            max_rooms_per_space=parse_integer(request, "max_rooms_per_space"),
+            max_rooms_per_space=max_rooms_per_space,
         )
 
+    # TODO When switching to the stable endpoint, remove the POST handler.
     async def on_POST(
         self, request: SynapseRequest, room_id: str
     ) -> Tuple[int, JsonDict]:
@@ -1033,12 +1023,19 @@ async def on_POST(
             )
 
         max_rooms_per_space = content.get("max_rooms_per_space")
-        if max_rooms_per_space is not None and not isinstance(max_rooms_per_space, int):
-            raise SynapseError(
-                400, "'max_rooms_per_space' must be an integer", Codes.BAD_JSON
-            )
+        if max_rooms_per_space is not None:
+            if not isinstance(max_rooms_per_space, int):
+                raise SynapseError(
+                    400, "'max_rooms_per_space' must be an integer", Codes.BAD_JSON
+                )
+            if max_rooms_per_space < 0:
+                raise SynapseError(
+                    400,
+                    "Value for 'max_rooms_per_space' must be a non-negative integer",
+                    Codes.BAD_JSON,
+                )
 
-        return 200, await self._space_summary_handler.get_space_summary(
+        return 200, await self._room_summary_handler.get_space_summary(
             requester.user.to_string(),
             room_id,
             suggested_only=suggested_only,
@@ -1046,6 +1043,84 @@ async def on_POST(
         )
 
 
+class RoomHierarchyRestServlet(RestServlet):
+    PATTERNS = (
+        re.compile(
+            "^/_matrix/client/unstable/org.matrix.msc2946"
+            "/rooms/(?P<room_id>[^/]*)/hierarchy$"
+        ),
+    )
+
+    def __init__(self, hs: "HomeServer"):
+        super().__init__()
+        self._auth = hs.get_auth()
+        self._room_summary_handler = hs.get_room_summary_handler()
+
+    async def on_GET(
+        self, request: SynapseRequest, room_id: str
+    ) -> Tuple[int, JsonDict]:
+        requester = await self._auth.get_user_by_req(request, allow_guest=True)
+
+        max_depth = parse_integer(request, "max_depth")
+        if max_depth is not None and max_depth < 0:
+            raise SynapseError(
+                400, "'max_depth' must be a non-negative integer", Codes.BAD_JSON
+            )
+
+        limit = parse_integer(request, "limit")
+        if limit is not None and limit <= 0:
+            raise SynapseError(
+                400, "'limit' must be a positive integer", Codes.BAD_JSON
+            )
+
+        return 200, await self._room_summary_handler.get_room_hierarchy(
+            requester.user.to_string(),
+            room_id,
+            suggested_only=parse_boolean(request, "suggested_only", default=False),
+            max_depth=max_depth,
+            limit=limit,
+            from_token=parse_string(request, "from"),
+        )
+
+
+class RoomSummaryRestServlet(ResolveRoomIdMixin, RestServlet):
+    PATTERNS = (
+        re.compile(
+            "^/_matrix/client/unstable/im.nheko.summary"
+            "/rooms/(?P<room_identifier>[^/]*)/summary$"
+        ),
+    )
+
+    def __init__(self, hs: "HomeServer"):
+        super().__init__(hs)
+        self._auth = hs.get_auth()
+        self._room_summary_handler = hs.get_room_summary_handler()
+
+    async def on_GET(
+        self, request: SynapseRequest, room_identifier: str
+    ) -> Tuple[int, JsonDict]:
+        try:
+            requester = await self._auth.get_user_by_req(request, allow_guest=True)
+            requester_user_id: Optional[str] = requester.user.to_string()
+        except MissingClientTokenError:
+            # auth is optional
+            requester_user_id = None
+
+        # twisted.web.server.Request.args is incorrectly defined as Optional[Any]
+        args: Dict[bytes, List[bytes]] = request.args  # type: ignore
+        remote_room_hosts = parse_strings_from_args(args, "via", required=False)
+        room_id, remote_room_hosts = await self.resolve_room_id(
+            room_identifier,
+            remote_room_hosts,
+        )
+
+        return 200, await self._room_summary_handler.get_room_summary(
+            requester_user_id,
+            room_id,
+            remote_room_hosts,
+        )
+
+
 def register_servlets(hs: "HomeServer", http_server, is_worker=False):
     RoomStateEventRestServlet(hs).register(http_server)
     RoomMemberListRestServlet(hs).register(http_server)
@@ -1059,18 +1134,19 @@ def register_servlets(hs: "HomeServer", http_server, is_worker=False):
     RoomRedactEventRestServlet(hs).register(http_server)
     RoomTypingRestServlet(hs).register(http_server)
     RoomEventContextServlet(hs).register(http_server)
-
-    if hs.config.experimental.spaces_enabled:
-        RoomSpaceSummaryRestServlet(hs).register(http_server)
+    RoomSpaceSummaryRestServlet(hs).register(http_server)
+    RoomHierarchyRestServlet(hs).register(http_server)
+    if hs.config.experimental.msc3266_enabled:
+        RoomSummaryRestServlet(hs).register(http_server)
+    RoomEventServlet(hs).register(http_server)
+    JoinedRoomsRestServlet(hs).register(http_server)
+    RoomAliasListServlet(hs).register(http_server)
+    SearchRestServlet(hs).register(http_server)
+    RoomCreateRestServlet(hs).register(http_server)
 
     # Some servlets only get registered for the main process.
     if not is_worker:
-        RoomCreateRestServlet(hs).register(http_server)
         RoomForgetRestServlet(hs).register(http_server)
-        SearchRestServlet(hs).register(http_server)
-        JoinedRoomsRestServlet(hs).register(http_server)
-        RoomEventServlet(hs).register(http_server)
-        RoomAliasListServlet(hs).register(http_server)
 
 
 def register_deprecated_servlets(hs, http_server):
diff --git a/synapse/rest/client/room_batch.py b/synapse/rest/client/room_batch.py
new file mode 100644
index 0000000000..3172aba605
--- /dev/null
+++ b/synapse/rest/client/room_batch.py
@@ -0,0 +1,441 @@
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+
+from synapse.api.constants import EventContentFields, EventTypes
+from synapse.api.errors import AuthError, Codes, SynapseError
+from synapse.appservice import ApplicationService
+from synapse.http.servlet import (
+    RestServlet,
+    assert_params_in_dict,
+    parse_json_object_from_request,
+    parse_string,
+    parse_strings_from_args,
+)
+from synapse.rest.client.transactions import HttpTransactionCache
+from synapse.types import Requester, UserID, create_requester
+from synapse.util.stringutils import random_string
+
+logger = logging.getLogger(__name__)
+
+
+class RoomBatchSendEventRestServlet(RestServlet):
+    """
+    API endpoint which can insert a chunk of events historically back in time
+    next to the given `prev_event`.
+
+    `chunk_id` comes from `next_chunk_id `in the response of the batch send
+    endpoint and is derived from the "insertion" events added to each chunk.
+    It's not required for the first batch send.
+
+    `state_events_at_start` is used to define the historical state events
+    needed to auth the events like join events. These events will float
+    outside of the normal DAG as outlier's and won't be visible in the chat
+    history which also allows us to insert multiple chunks without having a bunch
+    of `@mxid joined the room` noise between each chunk.
+
+    `events` is chronological chunk/list of events you want to insert.
+    There is a reverse-chronological constraint on chunks so once you insert
+    some messages, you can only insert older ones after that.
+    tldr; Insert chunks from your most recent history -> oldest history.
+
+    POST /_matrix/client/unstable/org.matrix.msc2716/rooms/<roomID>/batch_send?prev_event=<eventID>&chunk_id=<chunkID>
+    {
+        "events": [ ... ],
+        "state_events_at_start": [ ... ]
+    }
+    """
+
+    PATTERNS = (
+        re.compile(
+            "^/_matrix/client/unstable/org.matrix.msc2716"
+            "/rooms/(?P<room_id>[^/]*)/batch_send$"
+        ),
+    )
+
+    def __init__(self, hs):
+        super().__init__()
+        self.hs = hs
+        self.store = hs.get_datastore()
+        self.state_store = hs.get_storage().state
+        self.event_creation_handler = hs.get_event_creation_handler()
+        self.room_member_handler = hs.get_room_member_handler()
+        self.auth = hs.get_auth()
+        self.txns = HttpTransactionCache(hs)
+
+    async def _inherit_depth_from_prev_ids(self, prev_event_ids) -> int:
+        (
+            most_recent_prev_event_id,
+            most_recent_prev_event_depth,
+        ) = await self.store.get_max_depth_of(prev_event_ids)
+
+        # We want to insert the historical event after the `prev_event` but before the successor event
+        #
+        # We inherit depth from the successor event instead of the `prev_event`
+        # because events returned from `/messages` are first sorted by `topological_ordering`
+        # which is just the `depth` and then tie-break with `stream_ordering`.
+        #
+        # We mark these inserted historical events as "backfilled" which gives them a
+        # negative `stream_ordering`. If we use the same depth as the `prev_event`,
+        # then our historical event will tie-break and be sorted before the `prev_event`
+        # when it should come after.
+        #
+        # We want to use the successor event depth so they appear after `prev_event` because
+        # it has a larger `depth` but before the successor event because the `stream_ordering`
+        # is negative before the successor event.
+        successor_event_ids = await self.store.get_successor_events(
+            [most_recent_prev_event_id]
+        )
+
+        # If we can't find any successor events, then it's a forward extremity of
+        # historical messages and we can just inherit from the previous historical
+        # event which we can already assume has the correct depth where we want
+        # to insert into.
+        if not successor_event_ids:
+            depth = most_recent_prev_event_depth
+        else:
+            (
+                _,
+                oldest_successor_depth,
+            ) = await self.store.get_min_depth_of(successor_event_ids)
+
+            depth = oldest_successor_depth
+
+        return depth
+
+    def _create_insertion_event_dict(
+        self, sender: str, room_id: str, origin_server_ts: int
+    ):
+        """Creates an event dict for an "insertion" event with the proper fields
+        and a random chunk ID.
+
+        Args:
+            sender: The event author MXID
+            room_id: The room ID that the event belongs to
+            origin_server_ts: Timestamp when the event was sent
+
+        Returns:
+            Tuple of event ID and stream ordering position
+        """
+
+        next_chunk_id = random_string(8)
+        insertion_event = {
+            "type": EventTypes.MSC2716_INSERTION,
+            "sender": sender,
+            "room_id": room_id,
+            "content": {
+                EventContentFields.MSC2716_NEXT_CHUNK_ID: next_chunk_id,
+                EventContentFields.MSC2716_HISTORICAL: True,
+            },
+            "origin_server_ts": origin_server_ts,
+        }
+
+        return insertion_event
+
+    async def _create_requester_for_user_id_from_app_service(
+        self, user_id: str, app_service: ApplicationService
+    ) -> Requester:
+        """Creates a new requester for the given user_id
+        and validates that the app service is allowed to control
+        the given user.
+
+        Args:
+            user_id: The author MXID that the app service is controlling
+            app_service: The app service that controls the user
+
+        Returns:
+            Requester object
+        """
+
+        await self.auth.validate_appservice_can_control_user_id(app_service, user_id)
+
+        return create_requester(user_id, app_service=app_service)
+
+    async def on_POST(self, request, room_id):
+        requester = await self.auth.get_user_by_req(request, allow_guest=False)
+
+        if not requester.app_service:
+            raise AuthError(
+                403,
+                "Only application services can use the /batchsend endpoint",
+            )
+
+        body = parse_json_object_from_request(request)
+        assert_params_in_dict(body, ["state_events_at_start", "events"])
+
+        prev_events_from_query = parse_strings_from_args(request.args, "prev_event")
+        chunk_id_from_query = parse_string(request, "chunk_id")
+
+        if prev_events_from_query is None:
+            raise SynapseError(
+                400,
+                "prev_event query parameter is required when inserting historical messages back in time",
+                errcode=Codes.MISSING_PARAM,
+            )
+
+        # For the event we are inserting next to (`prev_events_from_query`),
+        # find the most recent auth events (derived from state events) that
+        # allowed that message to be sent. We will use that as a base
+        # to auth our historical messages against.
+        (
+            most_recent_prev_event_id,
+            _,
+        ) = await self.store.get_max_depth_of(prev_events_from_query)
+        # mapping from (type, state_key) -> state_event_id
+        prev_state_map = await self.state_store.get_state_ids_for_event(
+            most_recent_prev_event_id
+        )
+        # List of state event ID's
+        prev_state_ids = list(prev_state_map.values())
+        auth_event_ids = prev_state_ids
+
+        state_events_at_start = []
+        for state_event in body["state_events_at_start"]:
+            assert_params_in_dict(
+                state_event, ["type", "origin_server_ts", "content", "sender"]
+            )
+
+            logger.debug(
+                "RoomBatchSendEventRestServlet inserting state_event=%s, auth_event_ids=%s",
+                state_event,
+                auth_event_ids,
+            )
+
+            event_dict = {
+                "type": state_event["type"],
+                "origin_server_ts": state_event["origin_server_ts"],
+                "content": state_event["content"],
+                "room_id": room_id,
+                "sender": state_event["sender"],
+                "state_key": state_event["state_key"],
+            }
+
+            # Mark all events as historical
+            event_dict["content"][EventContentFields.MSC2716_HISTORICAL] = True
+
+            # Make the state events float off on their own
+            fake_prev_event_id = "$" + random_string(43)
+
+            # TODO: This is pretty much the same as some other code to handle inserting state in this file
+            if event_dict["type"] == EventTypes.Member:
+                membership = event_dict["content"].get("membership", None)
+                event_id, _ = await self.room_member_handler.update_membership(
+                    await self._create_requester_for_user_id_from_app_service(
+                        state_event["sender"], requester.app_service
+                    ),
+                    target=UserID.from_string(event_dict["state_key"]),
+                    room_id=room_id,
+                    action=membership,
+                    content=event_dict["content"],
+                    outlier=True,
+                    prev_event_ids=[fake_prev_event_id],
+                    # Make sure to use a copy of this list because we modify it
+                    # later in the loop here. Otherwise it will be the same
+                    # reference and also update in the event when we append later.
+                    auth_event_ids=auth_event_ids.copy(),
+                )
+            else:
+                # TODO: Add some complement tests that adds state that is not member joins
+                # and will use this code path. Maybe we only want to support join state events
+                # and can get rid of this `else`?
+                (
+                    event,
+                    _,
+                ) = await self.event_creation_handler.create_and_send_nonmember_event(
+                    await self._create_requester_for_user_id_from_app_service(
+                        state_event["sender"], requester.app_service
+                    ),
+                    event_dict,
+                    outlier=True,
+                    prev_event_ids=[fake_prev_event_id],
+                    # Make sure to use a copy of this list because we modify it
+                    # later in the loop here. Otherwise it will be the same
+                    # reference and also update in the event when we append later.
+                    auth_event_ids=auth_event_ids.copy(),
+                )
+                event_id = event.event_id
+
+            state_events_at_start.append(event_id)
+            auth_event_ids.append(event_id)
+
+        events_to_create = body["events"]
+
+        inherited_depth = await self._inherit_depth_from_prev_ids(
+            prev_events_from_query
+        )
+
+        # Figure out which chunk to connect to. If they passed in
+        # chunk_id_from_query let's use it. The chunk ID passed in comes
+        # from the chunk_id in the "insertion" event from the previous chunk.
+        last_event_in_chunk = events_to_create[-1]
+        chunk_id_to_connect_to = chunk_id_from_query
+        base_insertion_event = None
+        if chunk_id_from_query:
+            #  All but the first base insertion event should point at a fake
+            #  event, which causes the HS to ask for the state at the start of
+            #  the chunk later.
+            prev_event_ids = [fake_prev_event_id]
+            # TODO: Verify the chunk_id_from_query corresponds to an insertion event
+            pass
+        # Otherwise, create an insertion event to act as a starting point.
+        #
+        # We don't always have an insertion event to start hanging more history
+        # off of (ideally there would be one in the main DAG, but that's not the
+        # case if we're wanting to add history to e.g. existing rooms without
+        # an insertion event), in which case we just create a new insertion event
+        # that can then get pointed to by a "marker" event later.
+        else:
+            prev_event_ids = prev_events_from_query
+
+            base_insertion_event_dict = self._create_insertion_event_dict(
+                sender=requester.user.to_string(),
+                room_id=room_id,
+                origin_server_ts=last_event_in_chunk["origin_server_ts"],
+            )
+            base_insertion_event_dict["prev_events"] = prev_event_ids.copy()
+
+            (
+                base_insertion_event,
+                _,
+            ) = await self.event_creation_handler.create_and_send_nonmember_event(
+                await self._create_requester_for_user_id_from_app_service(
+                    base_insertion_event_dict["sender"],
+                    requester.app_service,
+                ),
+                base_insertion_event_dict,
+                prev_event_ids=base_insertion_event_dict.get("prev_events"),
+                auth_event_ids=auth_event_ids,
+                historical=True,
+                depth=inherited_depth,
+            )
+
+            chunk_id_to_connect_to = base_insertion_event["content"][
+                EventContentFields.MSC2716_NEXT_CHUNK_ID
+            ]
+
+        # Connect this current chunk to the insertion event from the previous chunk
+        chunk_event = {
+            "type": EventTypes.MSC2716_CHUNK,
+            "sender": requester.user.to_string(),
+            "room_id": room_id,
+            "content": {
+                EventContentFields.MSC2716_CHUNK_ID: chunk_id_to_connect_to,
+                EventContentFields.MSC2716_HISTORICAL: True,
+            },
+            # Since the chunk event is put at the end of the chunk,
+            # where the newest-in-time event is, copy the origin_server_ts from
+            # the last event we're inserting
+            "origin_server_ts": last_event_in_chunk["origin_server_ts"],
+        }
+        # Add the chunk event to the end of the chunk (newest-in-time)
+        events_to_create.append(chunk_event)
+
+        # Add an "insertion" event to the start of each chunk (next to the oldest-in-time
+        # event in the chunk) so the next chunk can be connected to this one.
+        insertion_event = self._create_insertion_event_dict(
+            sender=requester.user.to_string(),
+            room_id=room_id,
+            # Since the insertion event is put at the start of the chunk,
+            # where the oldest-in-time event is, copy the origin_server_ts from
+            # the first event we're inserting
+            origin_server_ts=events_to_create[0]["origin_server_ts"],
+        )
+        # Prepend the insertion event to the start of the chunk (oldest-in-time)
+        events_to_create = [insertion_event] + events_to_create
+
+        event_ids = []
+        events_to_persist = []
+        for ev in events_to_create:
+            assert_params_in_dict(ev, ["type", "origin_server_ts", "content", "sender"])
+
+            event_dict = {
+                "type": ev["type"],
+                "origin_server_ts": ev["origin_server_ts"],
+                "content": ev["content"],
+                "room_id": room_id,
+                "sender": ev["sender"],  # requester.user.to_string(),
+                "prev_events": prev_event_ids.copy(),
+            }
+
+            # Mark all events as historical
+            event_dict["content"][EventContentFields.MSC2716_HISTORICAL] = True
+
+            event, context = await self.event_creation_handler.create_event(
+                await self._create_requester_for_user_id_from_app_service(
+                    ev["sender"], requester.app_service
+                ),
+                event_dict,
+                prev_event_ids=event_dict.get("prev_events"),
+                auth_event_ids=auth_event_ids,
+                historical=True,
+                depth=inherited_depth,
+            )
+            logger.debug(
+                "RoomBatchSendEventRestServlet inserting event=%s, prev_event_ids=%s, auth_event_ids=%s",
+                event,
+                prev_event_ids,
+                auth_event_ids,
+            )
+
+            assert self.hs.is_mine_id(event.sender), "User must be our own: %s" % (
+                event.sender,
+            )
+
+            events_to_persist.append((event, context))
+            event_id = event.event_id
+
+            event_ids.append(event_id)
+            prev_event_ids = [event_id]
+
+        # Persist events in reverse-chronological order so they have the
+        # correct stream_ordering as they are backfilled (which decrements).
+        # Events are sorted by (topological_ordering, stream_ordering)
+        # where topological_ordering is just depth.
+        for (event, context) in reversed(events_to_persist):
+            ev = await self.event_creation_handler.handle_new_client_event(
+                await self._create_requester_for_user_id_from_app_service(
+                    event["sender"], requester.app_service
+                ),
+                event=event,
+                context=context,
+            )
+
+        # Add the base_insertion_event to the bottom of the list we return
+        if base_insertion_event is not None:
+            event_ids.append(base_insertion_event.event_id)
+
+        return 200, {
+            "state_events": state_events_at_start,
+            "events": event_ids,
+            "next_chunk_id": insertion_event["content"][
+                EventContentFields.MSC2716_NEXT_CHUNK_ID
+            ],
+        }
+
+    def on_GET(self, request, room_id):
+        return 501, "Not implemented"
+
+    def on_PUT(self, request, room_id):
+        return self.txns.fetch_or_execute_request(
+            request, self.on_POST, request, room_id
+        )
+
+
+def register_servlets(hs, http_server):
+    msc2716_enabled = hs.config.experimental.msc2716_enabled
+
+    if msc2716_enabled:
+        RoomBatchSendEventRestServlet(hs).register(http_server)
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/room_keys.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/room_keys.py
rename to synapse/rest/client/room_keys.py
index 53de97923f..263596be86 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/room_keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017, 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py b/synapse/rest/client/room_upgrade_rest_servlet.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py
rename to synapse/rest/client/room_upgrade_rest_servlet.py
index 147920767f..6d1b083acb 100644
--- a/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py
+++ b/synapse/rest/client/room_upgrade_rest_servlet.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/sendtodevice.py b/synapse/rest/client/sendtodevice.py
similarity index 96%
rename from synapse/rest/client/v2_alpha/sendtodevice.py
rename to synapse/rest/client/sendtodevice.py
index 79c1b526ee..d537d811d8 100644
--- a/synapse/rest/client/v2_alpha/sendtodevice.py
+++ b/synapse/rest/client/sendtodevice.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -60,7 +59,7 @@ async def _put(self, request, message_type, txn_id):
             requester, message_type, content["messages"]
         )
 
-        response = (200, {})  # type: Tuple[int, dict]
+        response: Tuple[int, dict] = (200, {})
         return response
 
 
diff --git a/synapse/rest/client/v2_alpha/shared_rooms.py b/synapse/rest/client/shared_rooms.py
similarity index 98%
rename from synapse/rest/client/v2_alpha/shared_rooms.py
rename to synapse/rest/client/shared_rooms.py
index c866d5151c..d2e7f04b40 100644
--- a/synapse/rest/client/v2_alpha/shared_rooms.py
+++ b/synapse/rest/client/shared_rooms.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Half-Shot
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/sync.py
similarity index 88%
rename from synapse/rest/client/v2_alpha/sync.py
rename to synapse/rest/client/sync.py
index c01ba14cd2..e18f4d01b3 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/sync.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +13,7 @@
 # limitations under the License.
 import itertools
 import logging
+from collections import defaultdict
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple
 
 from synapse.api.constants import Membership, PresenceState
@@ -112,7 +112,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
             default="online",
             allowed_values=self.ALLOWED_PRESENCE,
         )
-        filter_id = parse_string(request, "filter", default=None)
+        filter_id = parse_string(request, "filter")
         full_state = parse_boolean(request, "full_state", default=False)
 
         logger.debug(
@@ -233,29 +233,55 @@ async def encode_response(self, time_now, sync_result, access_token_id, filter):
         )
 
         logger.debug("building sync response dict")
-        return {
-            "account_data": {"events": sync_result.account_data},
-            "to_device": {"events": sync_result.to_device},
-            "device_lists": {
-                "changed": list(sync_result.device_lists.changed),
-                "left": list(sync_result.device_lists.left),
-            },
-            "presence": SyncRestServlet.encode_presence(sync_result.presence, time_now),
-            "rooms": {
-                Membership.JOIN: joined,
-                Membership.INVITE: invited,
-                Membership.KNOCK: knocked,
-                Membership.LEAVE: archived,
-            },
-            "groups": {
-                Membership.JOIN: sync_result.groups.join,
-                Membership.INVITE: sync_result.groups.invite,
-                Membership.LEAVE: sync_result.groups.leave,
-            },
-            "device_one_time_keys_count": sync_result.device_one_time_keys_count,
-            "org.matrix.msc2732.device_unused_fallback_key_types": sync_result.device_unused_fallback_key_types,
-            "next_batch": await sync_result.next_batch.to_string(self.store),
-        }
+
+        response: dict = defaultdict(dict)
+        response["next_batch"] = await sync_result.next_batch.to_string(self.store)
+
+        if sync_result.account_data:
+            response["account_data"] = {"events": sync_result.account_data}
+        if sync_result.presence:
+            response["presence"] = SyncRestServlet.encode_presence(
+                sync_result.presence, time_now
+            )
+
+        if sync_result.to_device:
+            response["to_device"] = {"events": sync_result.to_device}
+
+        if sync_result.device_lists.changed:
+            response["device_lists"]["changed"] = list(sync_result.device_lists.changed)
+        if sync_result.device_lists.left:
+            response["device_lists"]["left"] = list(sync_result.device_lists.left)
+
+        # We always include this because https://github.com/vector-im/element-android/issues/3725
+        # The spec isn't terribly clear on when this can be omitted and how a client would tell
+        # the difference between "no keys present" and "nothing changed" in terms of whole field
+        # absent / individual key type entry absent
+        # Corresponding synapse issue: https://github.com/matrix-org/synapse/issues/10456
+        response["device_one_time_keys_count"] = sync_result.device_one_time_keys_count
+
+        # https://github.com/matrix-org/matrix-doc/blob/54255851f642f84a4f1aaf7bc063eebe3d76752b/proposals/2732-olm-fallback-keys.md
+        # states that this field should always be included, as long as the server supports the feature.
+        response[
+            "org.matrix.msc2732.device_unused_fallback_key_types"
+        ] = sync_result.device_unused_fallback_key_types
+
+        if joined:
+            response["rooms"][Membership.JOIN] = joined
+        if invited:
+            response["rooms"][Membership.INVITE] = invited
+        if knocked:
+            response["rooms"][Membership.KNOCK] = knocked
+        if archived:
+            response["rooms"][Membership.LEAVE] = archived
+
+        if sync_result.groups.join:
+            response["groups"][Membership.JOIN] = sync_result.groups.join
+        if sync_result.groups.invite:
+            response["groups"][Membership.INVITE] = sync_result.groups.invite
+        if sync_result.groups.leave:
+            response["groups"][Membership.LEAVE] = sync_result.groups.leave
+
+        return response
 
     @staticmethod
     def encode_presence(events, time_now):
diff --git a/synapse/rest/client/v2_alpha/tags.py b/synapse/rest/client/tags.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/tags.py
rename to synapse/rest/client/tags.py
index a97cd66c52..c14f83be18 100644
--- a/synapse/rest/client/v2_alpha/tags.py
+++ b/synapse/rest/client/tags.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/thirdparty.py b/synapse/rest/client/thirdparty.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/thirdparty.py
rename to synapse/rest/client/thirdparty.py
index 0c127a1b5f..b5c67c9bb6 100644
--- a/synapse/rest/client/v2_alpha/thirdparty.py
+++ b/synapse/rest/client/thirdparty.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/tokenrefresh.py b/synapse/rest/client/tokenrefresh.py
similarity index 97%
rename from synapse/rest/client/v2_alpha/tokenrefresh.py
rename to synapse/rest/client/tokenrefresh.py
index 79317c74ba..b2f858545c 100644
--- a/synapse/rest/client/v2_alpha/tokenrefresh.py
+++ b/synapse/rest/client/tokenrefresh.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py
index 7be5c0fb88..94ff3719ce 100644
--- a/synapse/rest/client/transactions.py
+++ b/synapse/rest/client/transactions.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/v2_alpha/user_directory.py b/synapse/rest/client/user_directory.py
similarity index 99%
rename from synapse/rest/client/v2_alpha/user_directory.py
rename to synapse/rest/client/user_directory.py
index eeddfa31f8..9b091b0913 100644
--- a/synapse/rest/client/v2_alpha/user_directory.py
+++ b/synapse/rest/client/user_directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py
index 54eed3251b..72d2cb98de 100644
--- a/synapse/rest/client/versions.py
+++ b/synapse/rest/client/versions.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018-2019 New Vector Ltd
@@ -86,6 +85,8 @@ def on_GET(self, request):
                     "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private,
                     # Supports the busy presence state described in MSC3026.
                     "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled,
+                    # Supports receiving hidden read receipts as per MSC2285
+                    "org.matrix.msc2285": self.config.experimental.msc2285_enabled,
                 },
             },
         )
diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/voip.py
similarity index 96%
rename from synapse/rest/client/v1/voip.py
rename to synapse/rest/client/voip.py
index d07ca2c47c..f53020520d 100644
--- a/synapse/rest/client/v1/voip.py
+++ b/synapse/rest/client/voip.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +17,7 @@
 import hmac
 
 from synapse.http.servlet import RestServlet
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client._base import client_patterns
 
 
 class VoipRestServlet(RestServlet):
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index 8b9ef26cf2..11f7320832 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,6 +17,7 @@
 from hashlib import sha256
 from http import HTTPStatus
 from os import path
+from typing import Dict, List
 
 import jinja2
 from jinja2 import TemplateNotFound
@@ -25,7 +25,7 @@
 from synapse.api.errors import NotFoundError, StoreError, SynapseError
 from synapse.config import ConfigError
 from synapse.http.server import DirectServeHtmlResource, respond_with_html
-from synapse.http.servlet import parse_string
+from synapse.http.servlet import parse_bytes_from_args, parse_string
 from synapse.types import UserID
 
 # language to use for the templates. TODO: figure this out from Accept-Language
@@ -33,14 +33,6 @@
 
 logger = logging.getLogger(__name__)
 
-# use hmac.compare_digest if we have it (python 2.7.7), else just use equality
-if hasattr(hmac, "compare_digest"):
-    compare_digest = hmac.compare_digest
-else:
-
-    def compare_digest(a, b):
-        return a == b
-
 
 class ConsentResource(DirectServeHtmlResource):
     """A twisted Resource to display a privacy policy and gather consent to it
@@ -120,12 +112,13 @@ async def _async_render_GET(self, request):
             request (twisted.web.http.Request):
         """
         version = parse_string(request, "v", default=self._default_consent_version)
-        username = parse_string(request, "u", required=False, default="")
+        username = parse_string(request, "u", default="")
         userhmac = None
         has_consented = False
         public_version = username == ""
         if not public_version:
-            userhmac_bytes = parse_string(request, "h", required=True, encoding=None)
+            args: Dict[bytes, List[bytes]] = request.args
+            userhmac_bytes = parse_bytes_from_args(args, "h", required=True)
 
             self._check_hash(username, userhmac_bytes)
 
@@ -161,7 +154,8 @@ async def _async_render_POST(self, request):
         """
         version = parse_string(request, "v", required=True)
         username = parse_string(request, "u", required=True)
-        userhmac = parse_string(request, "h", required=True, encoding=None)
+        args: Dict[bytes, List[bytes]] = request.args
+        userhmac = parse_bytes_from_args(args, "h", required=True)
 
         self._check_hash(username, userhmac)
 
@@ -210,5 +204,5 @@ def _check_hash(self, userid, userhmac):
             .encode("ascii")
         )
 
-        if not compare_digest(want_mac, userhmac):
+        if not hmac.compare_digest(want_mac, userhmac):
             raise SynapseError(HTTPStatus.FORBIDDEN, "HMAC incorrect")
diff --git a/synapse/rest/health.py b/synapse/rest/health.py
index 0170950bf3..4487b54abf 100644
--- a/synapse/rest/health.py
+++ b/synapse/rest/health.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/key/__init__.py b/synapse/rest/key/__init__.py
index fe0ac3f8e9..629e2df74a 100644
--- a/synapse/rest/key/__init__.py
+++ b/synapse/rest/key/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/key/v2/__init__.py b/synapse/rest/key/v2/__init__.py
index cb5abcf826..c6c63073ea 100644
--- a/synapse/rest/key/v2/__init__.py
+++ b/synapse/rest/key/v2/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py
index d8e8e48c1c..a5fcd15e3a 100644
--- a/synapse/rest/key/v2/local_key_resource.py
+++ b/synapse/rest/key/v2/local_key_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -49,11 +48,6 @@ class LocalKey(Resource):
                     "key": # base64 encoded NACL verification key.
                 }
             },
-            "tls_fingerprints": [ # Fingerprints of the TLS certs this server uses.
-                {
-                    "sha256": # base64 encoded sha256 fingerprint of the X509 cert
-                },
-            ],
             "signatures": {
                 "this.server.example.com": {
                    "algorithm:version": # NACL signature for this server
@@ -90,14 +84,11 @@ def response_json_object(self):
                 "expired_ts": key.expired_ts,
             }
 
-        tls_fingerprints = self.config.tls_fingerprints
-
         json_object = {
             "valid_until_ts": self.valid_until_ts,
             "server_name": self.config.server_name,
             "verify_keys": verify_keys,
             "old_verify_keys": old_verify_keys,
-            "tls_fingerprints": tls_fingerprints,
         }
         for key in self.config.signing_key:
             json_object = sign_json(json_object, self.config.server_name, key)
diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py
index c57ac22e58..63a40b1852 100644
--- a/synapse/rest/key/v2/remote_key_resource.py
+++ b/synapse/rest/key/v2/remote_key_resource.py
@@ -22,6 +22,7 @@
 from synapse.http.server import DirectServeJsonResource, respond_with_json
 from synapse.http.servlet import parse_integer, parse_json_object_from_request
 from synapse.util import json_decoder
+from synapse.util.async_helpers import yieldable_gather_results
 
 logger = logging.getLogger(__name__)
 
@@ -73,9 +74,6 @@ class RemoteKey(DirectServeJsonResource):
                         "expired_ts": 0, # when the key stop being used.
                     }
                 }
-                "tls_fingerprints": [
-                    { "sha256": # fingerprint }
-                ]
                 "signatures": {
                     "remote.server.example.com": {...}
                     "this.server.example.com": {...}
@@ -99,7 +97,7 @@ def __init__(self, hs):
     async def _async_render_GET(self, request):
         if len(request.postpath) == 1:
             (server,) = request.postpath
-            query = {server.decode("ascii"): {}}  # type: dict
+            query: dict = {server.decode("ascii"): {}}
         elif len(request.postpath) == 2:
             server, key_id = request.postpath
             minimum_valid_until_ts = parse_integer(request, "minimum_valid_until_ts")
@@ -143,8 +141,8 @@ async def query_keys(self, request, query, query_remote_on_cache_miss=False):
         time_now_ms = self.clock.time_msec()
 
         # Note that the value is unused.
-        cache_misses = {}  # type: Dict[str, Dict[str, int]]
-        for (server_name, key_id, from_server), results in cached.items():
+        cache_misses: Dict[str, Dict[str, int]] = {}
+        for (server_name, key_id, _), results in cached.items():
             results = [(result["ts_added_ms"], result) for result in results]
 
             if not results and key_id is not None:
@@ -206,14 +204,20 @@ async def query_keys(self, request, query, query_remote_on_cache_miss=False):
                 # Cast to bytes since postgresql returns a memoryview.
                 json_results.add(bytes(most_recent_result["key_json"]))
             else:
-                for ts_added, result in results:
+                for _, result in results:
                     # Cast to bytes since postgresql returns a memoryview.
                     json_results.add(bytes(result["key_json"]))
 
         # If there is a cache miss, request the missing keys, then recurse (and
         # ensure the result is sent).
         if cache_misses and query_remote_on_cache_miss:
-            await self.fetcher.get_keys(cache_misses)
+            await yieldable_gather_results(
+                lambda t: self.fetcher.get_keys(*t),
+                (
+                    (server_name, list(keys), 0)
+                    for server_name, keys in cache_misses.items()
+                ),
+            )
             await self.query_keys(request, query, query_remote_on_cache_miss=False)
         else:
             signed_keys = []
diff --git a/synapse/rest/media/v1/__init__.py b/synapse/rest/media/v1/__init__.py
index 3b8c96e267..3dd16d4bb5 100644
--- a/synapse/rest/media/v1/__init__.py
+++ b/synapse/rest/media/v1/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +17,7 @@
 # check for JPEG support.
 try:
     PIL.Image._getdecoder("rgb", "jpeg", None)
-except IOError as e:
+except OSError as e:
     if str(e).startswith("decoder jpeg not available"):
         raise Exception(
             "FATAL: jpeg codec not supported. Install pillow correctly! "
@@ -33,7 +32,7 @@
 # check for PNG support.
 try:
     PIL.Image._getdecoder("rgb", "zip", None)
-except IOError as e:
+except OSError as e:
     if str(e).startswith("decoder zip not available"):
         raise Exception(
             "FATAL: zip codec not supported. Install pillow correctly! "
diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index 6366947071..90364ebcf7 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019-2021 The Matrix.org Foundation C.I.C.
 #
@@ -50,7 +49,7 @@
 def parse_media_id(request: Request) -> Tuple[str, str, Optional[str]]:
     try:
         # The type on postpath seems incorrect in Twisted 21.2.0.
-        postpath = request.postpath  # type: List[bytes]  # type: ignore
+        postpath: List[bytes] = request.postpath  # type: ignore
         assert postpath
 
         # This allows users to append e.g. /test.png to the URL. Useful for
diff --git a/synapse/rest/media/v1/config_resource.py b/synapse/rest/media/v1/config_resource.py
index c41a7ab412..a1d36e5cf1 100644
--- a/synapse/rest/media/v1/config_resource.py
+++ b/synapse/rest/media/v1/config_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 Will Hunt <will@half-shot.uk>
 # Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 #
@@ -31,7 +30,7 @@ class MediaConfigResource(DirectServeJsonResource):
 
     def __init__(self, hs: "HomeServer"):
         super().__init__()
-        config = hs.get_config()
+        config = hs.config
         self.clock = hs.get_clock()
         self.auth = hs.get_auth()
         self.limits_dict = {"m.upload.size": config.max_upload_size}
diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py
index 5dadaeaf57..d6d938953e 100644
--- a/synapse/rest/media/v1/download_resource.py
+++ b/synapse/rest/media/v1/download_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 #
@@ -50,6 +49,8 @@ async def _async_render_GET(self, request: Request) -> None:
             b" media-src 'self';"
             b" object-src 'self';",
         )
+        # Limited non-standard form of CSP for IE11
+        request.setHeader(b"X-Content-Security-Policy", b"sandbox;")
         request.setHeader(
             b"Referrer-Policy",
             b"no-referrer",
diff --git a/synapse/rest/media/v1/filepath.py b/synapse/rest/media/v1/filepath.py
index 7792f26e78..09531ebf54 100644
--- a/synapse/rest/media/v1/filepath.py
+++ b/synapse/rest/media/v1/filepath.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 #
@@ -22,7 +21,7 @@
 NEW_FORMAT_ID_RE = re.compile(r"^\d\d\d\d-\d\d-\d\d")
 
 
-def _wrap_in_base_path(func: "Callable[..., str]") -> "Callable[..., str]":
+def _wrap_in_base_path(func: Callable[..., str]) -> Callable[..., str]:
     """Takes a function that returns a relative path and turns it into an
     absolute path based on the location of the primary media store
     """
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 0c041b542d..0f5ce41ff8 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018-2021 The Matrix.org Foundation C.I.C.
 #
@@ -77,16 +76,18 @@ def __init__(self, hs: "HomeServer"):
         self.max_upload_size = hs.config.max_upload_size
         self.max_image_pixels = hs.config.max_image_pixels
 
-        self.primary_base_path = hs.config.media_store_path  # type: str
-        self.filepaths = MediaFilePaths(self.primary_base_path)  # type: MediaFilePaths
+        Thumbnailer.set_limits(self.max_image_pixels)
+
+        self.primary_base_path: str = hs.config.media_store_path
+        self.filepaths: MediaFilePaths = MediaFilePaths(self.primary_base_path)
 
         self.dynamic_thumbnails = hs.config.dynamic_thumbnails
         self.thumbnail_requirements = hs.config.thumbnail_requirements
 
         self.remote_media_linearizer = Linearizer(name="media_remote")
 
-        self.recently_accessed_remotes = set()  # type: Set[Tuple[str, str]]
-        self.recently_accessed_locals = set()  # type: Set[str]
+        self.recently_accessed_remotes: Set[Tuple[str, str]] = set()
+        self.recently_accessed_locals: Set[str] = set()
 
         self.federation_domain_whitelist = hs.config.federation_domain_whitelist
 
@@ -468,6 +469,9 @@ async def _download_remote_file(
         return media_info
 
     def _get_thumbnail_requirements(self, media_type):
+        scpos = media_type.find(";")
+        if scpos > 0:
+            media_type = media_type[:scpos]
         return self.thumbnail_requirements.get(media_type, ())
 
     def _generate_thumbnail(
@@ -707,7 +711,7 @@ async def _generate_thumbnails(
 
         # We deduplicate the thumbnail sizes by ignoring the cropped versions if
         # they have the same dimensions of a scaled one.
-        thumbnails = {}  # type: Dict[Tuple[int, int, str], str]
+        thumbnails: Dict[Tuple[int, int, str], str] = {}
         for r_width, r_height, r_method, r_type in requirements:
             if r_method == "crop":
                 thumbnails.setdefault((r_width, r_height, r_type), r_method)
@@ -832,7 +836,9 @@ async def delete_old_remote_media(self, before_ts: int) -> Dict[str, int]:
 
         return {"deleted": deleted}
 
-    async def delete_local_media(self, media_id: str) -> Tuple[List[str], int]:
+    async def delete_local_media_ids(
+        self, media_ids: List[str]
+    ) -> Tuple[List[str], int]:
         """
         Delete the given local or remote media ID from this server
 
@@ -841,7 +847,7 @@ async def delete_local_media(self, media_id: str) -> Tuple[List[str], int]:
         Returns:
             A tuple of (list of deleted media IDs, total deleted media IDs).
         """
-        return await self._remove_local_media_from_disk([media_id])
+        return await self._remove_local_media_from_disk(media_ids)
 
     async def delete_old_local_media(
         self,
diff --git a/synapse/rest/media/v1/media_storage.py b/synapse/rest/media/v1/media_storage.py
index b1b1c9e6ec..56cdc1b4ed 100644
--- a/synapse/rest/media/v1/media_storage.py
+++ b/synapse/rest/media/v1/media_storage.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -192,7 +191,7 @@ async def fetch_media(self, file_info: FileInfo) -> Optional[Responder]:
 
         for provider in self.storage_providers:
             for path in paths:
-                res = await provider.fetch(path, file_info)  # type: Any
+                res: Any = await provider.fetch(path, file_info)
                 if res:
                     logger.debug("Streaming %s from %s", path, provider)
                     return res
@@ -234,7 +233,7 @@ async def ensure_media_is_in_local_cache(self, file_info: FileInfo) -> str:
             os.makedirs(dirname)
 
         for provider in self.storage_providers:
-            res = await provider.fetch(path, file_info)  # type: Any
+            res: Any = await provider.fetch(path, file_info)
             if res:
                 with res:
                     consumer = BackgroundFileConsumer(
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index c4ed9dfdb4..0f051d4041 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 #
@@ -59,9 +58,11 @@
 
 logger = logging.getLogger(__name__)
 
-_charset_match = re.compile(br'<\s*meta[^>]*charset\s*=\s*"?([a-z0-9-]+)"?', flags=re.I)
+_charset_match = re.compile(
+    br'<\s*meta[^>]*charset\s*=\s*"?([a-z0-9_-]+)"?', flags=re.I
+)
 _xml_encoding_match = re.compile(
-    br'\s*<\s*\?\s*xml[^>]*encoding="([a-z0-9-]+)"', flags=re.I
+    br'\s*<\s*\?\s*xml[^>]*encoding="([a-z0-9_-]+)"', flags=re.I
 )
 _content_type_match = re.compile(r'.*; *charset="?(.*?)"?(;|$)', flags=re.I)
 
@@ -170,7 +171,7 @@ def __init__(
 
         # memory cache mapping urls to an ObservableDeferred returning
         # JSON-encoded OG metadata
-        self._cache = ExpiringCache(
+        self._cache: ExpiringCache[str, ObservableDeferred] = ExpiringCache(
             cache_name="url_previews",
             clock=self.clock,
             # don't spider URLs more often than once an hour
@@ -187,15 +188,11 @@ async def _async_render_OPTIONS(self, request: Request) -> None:
         respond_with_json(request, 200, {}, send_cors=True)
 
     async def _async_render_GET(self, request: SynapseRequest) -> None:
-        # This will always be set by the time Twisted calls us.
-        assert request.args is not None
-
         # XXX: if get_user_by_req fails, what should we do in an async render?
         requester = await self.auth.get_user_by_req(request)
-        url = parse_string(request, "url")
-        if b"ts" in request.args:
-            ts = parse_integer(request, "ts")
-        else:
+        url = parse_string(request, "url", required=True)
+        ts = parse_integer(request, "ts")
+        if ts is None:
             ts = self.clock.time_msec()
 
         # XXX: we could move this into _do_preview if we wanted.
@@ -461,7 +458,7 @@ async def _download_url(self, url: str, user: str) -> Dict[str, Any]:
         file_info = FileInfo(server_name=None, file_id=file_id, url_cache=True)
 
         # If this URL can be accessed via oEmbed, use that instead.
-        url_to_download = url  # type: Optional[str]
+        url_to_download: Optional[str] = url
         oembed_url = self._get_oembed_url(url)
         if oembed_url:
             # The result might be a new URL to download, or it might be HTML content.
@@ -789,7 +786,7 @@ def _calc_og(tree: "etree.Element", media_uri: str) -> Dict[str, Optional[str]]:
     # "og:video:height" : "720",
     # "og:video:secure_url": "https://www.youtube.com/v/LXDBoHyjmtw?version=3",
 
-    og = {}  # type: Dict[str, Optional[str]]
+    og: Dict[str, Optional[str]] = {}
     for tag in tree.xpath("//*/meta[starts-with(@property, 'og:')]"):
         if "content" in tag.attrib:
             # if we've got more than 50 tags, someone is taking the piss
diff --git a/synapse/rest/media/v1/storage_provider.py b/synapse/rest/media/v1/storage_provider.py
index 031947557d..0ff6ad3c0c 100644
--- a/synapse/rest/media/v1/storage_provider.py
+++ b/synapse/rest/media/v1/storage_provider.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py
index af802bc0b1..a029d426f0 100644
--- a/synapse/rest/media/v1/thumbnail_resource.py
+++ b/synapse/rest/media/v1/thumbnail_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 #
diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py
index 988f52c78f..a65e9e1802 100644
--- a/synapse/rest/media/v1/thumbnailer.py
+++ b/synapse/rest/media/v1/thumbnailer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 #
@@ -41,6 +40,10 @@ class Thumbnailer:
 
     FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"}
 
+    @staticmethod
+    def set_limits(max_image_pixels: int):
+        Image.MAX_IMAGE_PIXELS = max_image_pixels
+
     def __init__(self, input_path: str):
         try:
             self.image = Image.open(input_path)
@@ -48,6 +51,11 @@ def __init__(self, input_path: str):
             # If an error occurs opening the image, a thumbnail won't be able to
             # be generated.
             raise ThumbnailError from e
+        except Image.DecompressionBombError as e:
+            # If an image decompression bomb error occurs opening the image,
+            # then the image exceeds the pixel limit and a thumbnail won't
+            # be able to be generated.
+            raise ThumbnailError from e
 
         self.width, self.height = self.image.size
         self.transpose_method = None
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index 0138b2e2d1..146adca8f1 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 #
@@ -15,13 +14,13 @@
 # limitations under the License.
 
 import logging
-from typing import IO, TYPE_CHECKING
+from typing import IO, TYPE_CHECKING, Dict, List, Optional
 
 from twisted.web.server import Request
 
 from synapse.api.errors import Codes, SynapseError
 from synapse.http.server import DirectServeJsonResource, respond_with_json
-from synapse.http.servlet import parse_string
+from synapse.http.servlet import parse_bytes_from_args
 from synapse.http.site import SynapseRequest
 from synapse.rest.media.v1.media_storage import SpamMediaException
 
@@ -52,8 +51,6 @@ async def _async_render_OPTIONS(self, request: Request) -> None:
 
     async def _async_render_POST(self, request: SynapseRequest) -> None:
         requester = await self.auth.get_user_by_req(request)
-        # TODO: The checks here are a bit late. The content will have
-        # already been uploaded to a tmp file at this point
         content_length = request.getHeader("Content-Length")
         if content_length is None:
             raise SynapseError(msg="Request must specify a Content-Length", code=400)
@@ -64,10 +61,11 @@ async def _async_render_POST(self, request: SynapseRequest) -> None:
                 errcode=Codes.TOO_LARGE,
             )
 
-        upload_name = parse_string(request, b"filename", encoding=None)
-        if upload_name:
+        args: Dict[bytes, List[bytes]] = request.args  # type: ignore
+        upload_name_bytes = parse_bytes_from_args(args, "filename")
+        if upload_name_bytes:
             try:
-                upload_name = upload_name.decode("utf8")
+                upload_name: Optional[str] = upload_name_bytes.decode("utf8")
             except UnicodeDecodeError:
                 raise SynapseError(
                     msg="Invalid UTF-8 filename parameter: %r" % (upload_name), code=400
@@ -91,7 +89,7 @@ async def _async_render_POST(self, request: SynapseRequest) -> None:
         # TODO(markjh): parse content-dispostion
 
         try:
-            content = request.content  # type: IO  # type: ignore
+            content: IO = request.content  # type: ignore
             content_uri = await self.media_repo.create_content(
                 media_type, upload_name, content, content_length, requester.user
             )
diff --git a/synapse/rest/synapse/__init__.py b/synapse/rest/synapse/__init__.py
index c0b733488b..6ef4fbe8f7 100644
--- a/synapse/rest/synapse/__init__.py
+++ b/synapse/rest/synapse/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/synapse/client/__init__.py b/synapse/rest/synapse/client/__init__.py
index 9eeb970580..47a2f72b32 100644
--- a/synapse/rest/synapse/client/__init__.py
+++ b/synapse/rest/synapse/client/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/synapse/client/new_user_consent.py b/synapse/rest/synapse/client/new_user_consent.py
index 78ee0b5e88..fc62a09b7f 100644
--- a/synapse/rest/synapse/client/new_user_consent.py
+++ b/synapse/rest/synapse/client/new_user_consent.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -47,6 +46,8 @@ def __init__(self, hs: "HomeServer"):
         self._consent_version = hs.config.consent.user_consent_version
 
         def template_search_dirs():
+            if hs.config.server.custom_template_directory:
+                yield hs.config.server.custom_template_directory
             if hs.config.sso.sso_template_dir:
                 yield hs.config.sso.sso_template_dir
             yield hs.config.sso.default_template_dir
@@ -62,6 +63,15 @@ async def _async_render_GET(self, request: Request) -> None:
             self._sso_handler.render_error(request, "bad_session", e.msg, code=e.code)
             return
 
+        # It should be impossible to get here without having first been through
+        # the pick-a-username step, which ensures chosen_localpart gets set.
+        if not session.chosen_localpart:
+            logger.warning("Session has no user name selected")
+            self._sso_handler.render_error(
+                request, "no_user", "No user name has been selected.", code=400
+            )
+            return
+
         user_id = UserID(session.chosen_localpart, self._server_name)
         user_profile = {
             "display_name": session.display_name,
diff --git a/synapse/rest/synapse/client/oidc/__init__.py b/synapse/rest/synapse/client/oidc/__init__.py
index 64c0deb75d..36ba401656 100644
--- a/synapse/rest/synapse/client/oidc/__init__.py
+++ b/synapse/rest/synapse/client/oidc/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Quentin Gliech
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/synapse/client/oidc/callback_resource.py b/synapse/rest/synapse/client/oidc/callback_resource.py
index 1af33f0a45..7785f17e90 100644
--- a/synapse/rest/synapse/client/oidc/callback_resource.py
+++ b/synapse/rest/synapse/client/oidc/callback_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Quentin Gliech
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/synapse/client/password_reset.py b/synapse/rest/synapse/client/password_reset.py
index d26ce46efc..f2800bf2db 100644
--- a/synapse/rest/synapse/client/password_reset.py
+++ b/synapse/rest/synapse/client/password_reset.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/synapse/client/pick_idp.py b/synapse/rest/synapse/client/pick_idp.py
index 9550b82998..d3a94a9349 100644
--- a/synapse/rest/synapse/client/pick_idp.py
+++ b/synapse/rest/synapse/client/pick_idp.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/synapse/client/pick_username.py b/synapse/rest/synapse/client/pick_username.py
index d9ffe84489..c15b83c387 100644
--- a/synapse/rest/synapse/client/pick_username.py
+++ b/synapse/rest/synapse/client/pick_username.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -75,6 +74,8 @@ def __init__(self, hs: "HomeServer"):
         self._sso_handler = hs.get_sso_handler()
 
         def template_search_dirs():
+            if hs.config.server.custom_template_directory:
+                yield hs.config.server.custom_template_directory
             if hs.config.sso.sso_template_dir:
                 yield hs.config.sso.sso_template_dir
             yield hs.config.sso.default_template_dir
@@ -119,9 +120,9 @@ async def _async_render_POST(self, request: SynapseRequest):
             use_display_name = parse_boolean(request, "use_display_name", default=False)
 
             try:
-                emails_to_use = [
+                emails_to_use: List[str] = [
                     val.decode("utf-8") for val in request.args.get(b"use_email", [])
-                ]  # type: List[str]
+                ]
             except ValueError:
                 raise SynapseError(400, "Query parameter use_email must be utf-8")
         except SynapseError as e:
diff --git a/synapse/rest/synapse/client/saml2/__init__.py b/synapse/rest/synapse/client/saml2/__init__.py
index 3e8235ee1e..781ccb237c 100644
--- a/synapse/rest/synapse/client/saml2/__init__.py
+++ b/synapse/rest/synapse/client/saml2/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/synapse/client/saml2/metadata_resource.py b/synapse/rest/synapse/client/saml2/metadata_resource.py
index 1e8526e22e..b37c7083dc 100644
--- a/synapse/rest/synapse/client/saml2/metadata_resource.py
+++ b/synapse/rest/synapse/client/saml2/metadata_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/synapse/client/saml2/response_resource.py b/synapse/rest/synapse/client/saml2/response_resource.py
index 4dfadf1bfb..774ccd870f 100644
--- a/synapse/rest/synapse/client/saml2/response_resource.py
+++ b/synapse/rest/synapse/client/saml2/response_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/rest/synapse/client/sso_register.py b/synapse/rest/synapse/client/sso_register.py
index f2acce2437..70cd148a76 100644
--- a/synapse/rest/synapse/client/sso_register.py
+++ b/synapse/rest/synapse/client/sso_register.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py
index f591cc6c5c..19ac3af337 100644
--- a/synapse/rest/well_known.py
+++ b/synapse/rest/well_known.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/secrets.py b/synapse/secrets.py
deleted file mode 100644
index 7939db75e7..0000000000
--- a/synapse/secrets.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Injectable secrets module for Synapse.
-
-See https://docs.python.org/3/library/secrets.html#module-secrets for the API
-used in Python 3.6, and the API emulated in Python 2.7.
-"""
-import sys
-
-# secrets is available since python 3.6
-if sys.version_info[0:2] >= (3, 6):
-    import secrets
-
-    class Secrets:
-        def token_bytes(self, nbytes: int = 32) -> bytes:
-            return secrets.token_bytes(nbytes)
-
-        def token_hex(self, nbytes: int = 32) -> str:
-            return secrets.token_hex(nbytes)
-
-
-else:
-    import binascii
-    import os
-
-    class Secrets:
-        def token_bytes(self, nbytes: int = 32) -> bytes:
-            return os.urandom(nbytes)
-
-        def token_hex(self, nbytes: int = 32) -> str:
-            return binascii.hexlify(self.token_bytes(nbytes)).decode("ascii")
diff --git a/synapse/server.py b/synapse/server.py
index e85b9391fa..de6517663e 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -1,7 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2017-2018 New Vector Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,9 +34,8 @@
 )
 
 import twisted.internet.tcp
-from twisted.internet import defer
-from twisted.mail.smtp import sendmail
 from twisted.web.iweb import IPolicyForHTTPS
+from twisted.web.resource import IResource
 
 from synapse.api.auth import Auth
 from synapse.api.filtering import Filtering
@@ -51,6 +47,7 @@
 from synapse.crypto.context_factory import RegularPolicyForHTTPS
 from synapse.crypto.keyring import Keyring
 from synapse.events.builder import EventBuilderFactory
+from synapse.events.presence_router import PresenceRouter
 from synapse.events.spamcheck import SpamChecker
 from synapse.events.third_party_rules import ThirdPartyEventRules
 from synapse.events.utils import EventClientSerializer
@@ -66,17 +63,17 @@
 from synapse.groups.groups_server import GroupsServerHandler, GroupsServerWorkerHandler
 from synapse.handlers.account_data import AccountDataHandler
 from synapse.handlers.account_validity import AccountValidityHandler
-from synapse.handlers.acme import AcmeHandler
 from synapse.handlers.admin import AdminHandler
 from synapse.handlers.appservice import ApplicationServicesHandler
 from synapse.handlers.auth import AuthHandler, MacaroonGenerator
-from synapse.handlers.cas_handler import CasHandler
+from synapse.handlers.cas import CasHandler
 from synapse.handlers.deactivate_account import DeactivateAccountHandler
 from synapse.handlers.device import DeviceHandler, DeviceWorkerHandler
 from synapse.handlers.devicemessage import DeviceMessageHandler
 from synapse.handlers.directory import DirectoryHandler
 from synapse.handlers.e2e_keys import E2eKeysHandler
 from synapse.handlers.e2e_room_keys import E2eRoomKeysHandler
+from synapse.handlers.event_auth import EventAuthHandler
 from synapse.handlers.events import EventHandler, EventStreamHandler
 from synapse.handlers.federation import FederationHandler
 from synapse.handlers.groups_local import GroupsLocalHandler, GroupsLocalWorkerHandler
@@ -85,7 +82,11 @@
 from synapse.handlers.message import EventCreationHandler, MessageHandler
 from synapse.handlers.pagination import PaginationHandler
 from synapse.handlers.password_policy import PasswordPolicyHandler
-from synapse.handlers.presence import PresenceHandler
+from synapse.handlers.presence import (
+    BasePresenceHandler,
+    PresenceHandler,
+    WorkerPresenceHandler,
+)
 from synapse.handlers.profile import ProfileHandler
 from synapse.handlers.read_marker import ReadMarkerHandler
 from synapse.handlers.receipts import ReceiptsHandler
@@ -98,9 +99,10 @@
 from synapse.handlers.room_list import RoomListHandler
 from synapse.handlers.room_member import RoomMemberHandler, RoomMemberMasterHandler
 from synapse.handlers.room_member_worker import RoomMemberWorkerHandler
+from synapse.handlers.room_summary import RoomSummaryHandler
 from synapse.handlers.search import SearchHandler
+from synapse.handlers.send_email import SendEmailHandler
 from synapse.handlers.set_password import SetPasswordHandler
-from synapse.handlers.space_summary import SpaceSummaryHandler
 from synapse.handlers.sso import SsoHandler
 from synapse.handlers.stats import StatsHandler
 from synapse.handlers.sync import SyncHandler
@@ -121,7 +123,6 @@
     MediaRepository,
     MediaRepositoryResource,
 )
-from synapse.secrets import Secrets
 from synapse.server_notices.server_notices_manager import ServerNoticesManager
 from synapse.server_notices.server_notices_sender import ServerNoticesSender
 from synapse.server_notices.worker_server_notices_sender import (
@@ -141,8 +142,8 @@
 if TYPE_CHECKING:
     from txredisapi import RedisProtocol
 
-    from synapse.handlers.oidc_handler import OidcHandler
-    from synapse.handlers.saml_handler import SamlHandler
+    from synapse.handlers.oidc import OidcHandler
+    from synapse.handlers.saml import SamlHandler
 
 
 T = TypeVar("T", bound=Callable[..., Any])
@@ -244,15 +245,47 @@ def __init__(
         # the key we use to sign events and requests
         self.signing_key = config.key.signing_key[0]
         self.config = config
-        self._listening_services = []  # type: List[twisted.internet.tcp.Port]
-        self.start_time = None  # type: Optional[int]
+        self._listening_services: List[twisted.internet.tcp.Port] = []
+        self.start_time: Optional[int] = None
 
         self._instance_id = random_string(5)
         self._instance_name = config.worker.instance_name
 
         self.version_string = version_string
 
-        self.datastores = None  # type: Optional[Databases]
+        self.datastores: Optional[Databases] = None
+
+        self._module_web_resources: Dict[str, IResource] = {}
+        self._module_web_resources_consumed = False
+
+    def register_module_web_resource(self, path: str, resource: IResource):
+        """Allows a module to register a web resource to be served at the given path.
+
+        If multiple modules register a resource for the same path, the module that
+        appears the highest in the configuration file takes priority.
+
+        Args:
+            path: The path to register the resource for.
+            resource: The resource to attach to this path.
+
+        Raises:
+            SynapseError(500): A module tried to register a web resource after the HTTP
+                listeners have been started.
+        """
+        if self._module_web_resources_consumed:
+            raise RuntimeError(
+                "Tried to register a web resource from a module after startup",
+            )
+
+        # Don't register a resource that's already been registered.
+        if path not in self._module_web_resources.keys():
+            self._module_web_resources[path] = resource
+        else:
+            logger.warning(
+                "Module tried to register a web resource for path %s but another module"
+                " has already registered a resource for this path.",
+                path,
+            )
 
     def get_instance_id(self) -> str:
         """A unique ID for this synapse process instance.
@@ -282,6 +315,14 @@ def setup(self) -> None:
         if self.config.run_background_tasks:
             self.setup_background_tasks()
 
+    def start_listening(self) -> None:
+        """Start the HTTP, manhole, metrics, etc listeners
+
+        Does nothing in this base class; overridden in derived classes to start the
+        appropriate listeners.
+        """
+        pass
+
     def setup_background_tasks(self) -> None:
         """
         Some handlers have side effects on instantiation (like registering
@@ -319,9 +360,6 @@ def get_datastores(self) -> Databases:
 
         return self.datastores
 
-    def get_config(self) -> HomeServerConfig:
-        return self.config
-
     @cache_in_self
     def get_distributor(self) -> Distributor:
         return Distributor()
@@ -329,6 +367,7 @@ def get_distributor(self) -> Distributor:
     @cache_in_self
     def get_registration_ratelimiter(self) -> Ratelimiter:
         return Ratelimiter(
+            store=self.get_datastore(),
             clock=self.get_clock(),
             rate_hz=self.config.rc_registration.per_second,
             burst_count=self.config.rc_registration.burst_count,
@@ -401,10 +440,6 @@ def get_room_creation_handler(self) -> RoomCreationHandler:
     def get_room_shutdown_handler(self) -> RoomShutdownHandler:
         return RoomShutdownHandler(self)
 
-    @cache_in_self
-    def get_sendmail(self) -> Callable[..., defer.Deferred]:
-        return sendmail
-
     @cache_in_self
     def get_state_handler(self) -> StateHandler:
         return StateHandler(self)
@@ -414,8 +449,11 @@ def get_state_resolution_handler(self) -> StateResolutionHandler:
         return StateResolutionHandler(self)
 
     @cache_in_self
-    def get_presence_handler(self) -> PresenceHandler:
-        return PresenceHandler(self)
+    def get_presence_handler(self) -> BasePresenceHandler:
+        if self.get_instance_name() in self.config.worker.writers.presence:
+            return PresenceHandler(self)
+        else:
+            return WorkerPresenceHandler(self)
 
     @cache_in_self
     def get_typing_writer_handler(self) -> TypingWriterHandler:
@@ -424,6 +462,10 @@ def get_typing_writer_handler(self) -> TypingWriterHandler:
         else:
             raise Exception("Workers cannot write typing")
 
+    @cache_in_self
+    def get_presence_router(self) -> PresenceRouter:
+        return PresenceRouter(self)
+
     @cache_in_self
     def get_typing_handler(self) -> FollowerTypingHandler:
         if self.config.worker.writers.typing == self.get_instance_name():
@@ -476,10 +518,6 @@ def get_e2e_keys_handler(self) -> E2eKeysHandler:
     def get_e2e_room_keys_handler(self) -> E2eRoomKeysHandler:
         return E2eRoomKeysHandler(self)
 
-    @cache_in_self
-    def get_acme_handler(self) -> AcmeHandler:
-        return AcmeHandler(self)
-
     @cache_in_self
     def get_admin_handler(self) -> AdminHandler:
         return AdminHandler(self)
@@ -532,6 +570,10 @@ def get_deactivate_account_handler(self) -> DeactivateAccountHandler:
     def get_search_handler(self) -> SearchHandler:
         return SearchHandler(self)
 
+    @cache_in_self
+    def get_send_email_handler(self) -> SendEmailHandler:
+        return SendEmailHandler(self)
+
     @cache_in_self
     def get_set_password_handler(self) -> SetPasswordHandler:
         return SetPasswordHandler(self)
@@ -623,17 +665,13 @@ def get_groups_attestation_signing(self) -> GroupAttestationSigning:
     def get_groups_attestation_renewer(self) -> GroupAttestionRenewer:
         return GroupAttestionRenewer(self)
 
-    @cache_in_self
-    def get_secrets(self) -> Secrets:
-        return Secrets()
-
     @cache_in_self
     def get_stats_handler(self) -> StatsHandler:
         return StatsHandler(self)
 
     @cache_in_self
     def get_spam_checker(self) -> SpamChecker:
-        return SpamChecker(self)
+        return SpamChecker()
 
     @cache_in_self
     def get_third_party_event_rules(self) -> ThirdPartyEventRules:
@@ -687,13 +725,13 @@ def get_cas_handler(self) -> CasHandler:
 
     @cache_in_self
     def get_saml_handler(self) -> "SamlHandler":
-        from synapse.handlers.saml_handler import SamlHandler
+        from synapse.handlers.saml import SamlHandler
 
         return SamlHandler(self)
 
     @cache_in_self
     def get_oidc_handler(self) -> "OidcHandler":
-        from synapse.handlers.oidc_handler import OidcHandler
+        from synapse.handlers.oidc import OidcHandler
 
         return OidcHandler(self)
 
@@ -734,8 +772,12 @@ def get_account_data_handler(self) -> AccountDataHandler:
         return AccountDataHandler(self)
 
     @cache_in_self
-    def get_space_summary_handler(self) -> SpaceSummaryHandler:
-        return SpaceSummaryHandler(self)
+    def get_room_summary_handler(self) -> RoomSummaryHandler:
+        return RoomSummaryHandler(self)
+
+    @cache_in_self
+    def get_event_auth_handler(self) -> EventAuthHandler:
+        return EventAuthHandler(self)
 
     @cache_in_self
     def get_external_cache(self) -> ExternalCache:
diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
index a9349bf9a1..4e0f814035 100644
--- a/synapse/server_notices/consent_server_notices.py
+++ b/synapse/server_notices/consent_server_notices.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -35,7 +34,7 @@ def __init__(self, hs: "HomeServer"):
         self._server_notices_manager = hs.get_server_notices_manager()
         self._store = hs.get_datastore()
 
-        self._users_in_progress = set()  # type: Set[str]
+        self._users_in_progress: Set[str] = set()
 
         self._current_consent_version = hs.config.user_consent_version
         self._server_notice_content = hs.config.user_consent_server_notice_content
diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index a18a2e76c9..073b0d754f 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -206,7 +205,7 @@ async def _is_room_currently_blocked(self, room_id: str) -> Tuple[bool, List[str
             # The user has yet to join the server notices room
             pass
 
-        referenced_events = []  # type: List[str]
+        referenced_events: List[str] = []
         if pinned_state_event is not None:
             referenced_events = list(pinned_state_event.content.get("pinned", []))
 
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index 144e1da78e..f19075b760 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/server_notices/server_notices_sender.py b/synapse/server_notices/server_notices_sender.py
index 965c645889..cdf0973d05 100644
--- a/synapse/server_notices/server_notices_sender.py
+++ b/synapse/server_notices/server_notices_sender.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,10 +32,12 @@ class ServerNoticesSender(WorkerServerNoticesSender):
 
     def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
-        self._server_notices = (
+        self._server_notices: Iterable[
+            Union[ConsentServerNotices, ResourceLimitsServerNotices]
+        ] = (
             ConsentServerNotices(hs),
             ResourceLimitsServerNotices(hs),
-        )  # type: Iterable[Union[ConsentServerNotices, ResourceLimitsServerNotices]]
+        )
 
     async def on_user_syncing(self, user_id: str) -> None:
         """Called when the user performs a sync operation.
diff --git a/synapse/server_notices/worker_server_notices_sender.py b/synapse/server_notices/worker_server_notices_sender.py
index c76bd57460..cc53318491 100644
--- a/synapse/server_notices/worker_server_notices_sender.py
+++ b/synapse/server_notices/worker_server_notices_sender.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/spam_checker_api/__init__.py b/synapse/spam_checker_api/__init__.py
index 3ce25bb012..73018f2d00 100644
--- a/synapse/spam_checker_api/__init__.py
+++ b/synapse/spam_checker_api/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index c3d6e80c49..463ce58dae 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -17,11 +16,14 @@
 import logging
 from collections import defaultdict, namedtuple
 from typing import (
+    TYPE_CHECKING,
     Any,
     Awaitable,
     Callable,
+    Collection,
     DefaultDict,
     Dict,
+    FrozenSet,
     Iterable,
     List,
     Optional,
@@ -46,11 +48,15 @@
 from synapse.state import v1, v2
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.storage.roommember import ProfileInfo
-from synapse.types import Collection, StateMap
+from synapse.types import StateMap
 from synapse.util.async_helpers import Linearizer
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.metrics import Measure, measure_func
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+    from synapse.storage.databases.main import DataStore
+
 logger = logging.getLogger(__name__)
 metrics_logger = logging.getLogger("synapse.state.metrics")
 
@@ -73,7 +79,7 @@
 POWER_KEY = (EventTypes.PowerLevels, "")
 
 
-def _gen_state_id():
+def _gen_state_id() -> str:
     global _NEXT_STATE_ID
     s = "X%d" % (_NEXT_STATE_ID,)
     _NEXT_STATE_ID += 1
@@ -108,7 +114,7 @@ def __init__(
         # `state_id` is either a state_group (and so an int) or a string. This
         # ensures we don't accidentally persist a state_id as a stateg_group
         if state_group:
-            self.state_id = state_group
+            self.state_id: Union[str, int] = state_group
         else:
             self.state_id = _gen_state_id()
 
@@ -121,7 +127,7 @@ class StateHandler:
     where necessary
     """
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.clock = hs.get_clock()
         self.store = hs.get_datastore()
         self.state_store = hs.get_storage().state
@@ -212,19 +218,23 @@ async def get_current_state_ids(
         return ret.state
 
     async def get_current_users_in_room(
-        self, room_id: str, latest_event_ids: Optional[List[str]] = None
+        self, room_id: str, latest_event_ids: List[str]
     ) -> Dict[str, ProfileInfo]:
         """
         Get the users who are currently in a room.
 
+        Note: This is much slower than using the equivalent method
+        `DataStore.get_users_in_room` or `DataStore.get_users_in_room_with_profiles`,
+        so this should only be used when wanting the users at a particular point
+        in the room.
+
         Args:
             room_id: The ID of the room.
             latest_event_ids: Precomputed list of latest event IDs. Will be computed if None.
         Returns:
             Dictionary of user IDs to their profileinfo.
         """
-        if not latest_event_ids:
-            latest_event_ids = await self.store.get_latest_event_ids_in_room(room_id)
+
         assert latest_event_ids is not None
 
         logger.debug("calling resolve_state_groups from get_current_users_in_room")
@@ -304,9 +314,9 @@ async def compute_event_context(
 
         if old_state:
             # if we're given the state before the event, then we use that
-            state_ids_before_event = {
+            state_ids_before_event: StateMap[str] = {
                 (s.type, s.state_key): s.event_id for s in old_state
-            }  # type: StateMap[str]
+            }
             state_group_before_event = None
             state_group_before_event_prev_group = None
             deltas_to_state_group_before_event = None
@@ -502,13 +512,15 @@ class StateResolutionHandler:
     be storage-independent.
     """
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.clock = hs.get_clock()
 
         self.resolve_linearizer = Linearizer(name="state_resolve_lock")
 
         # dict of set of event_ids -> _StateCacheEntry.
-        self._state_cache = ExpiringCache(
+        self._state_cache: ExpiringCache[
+            FrozenSet[int], _StateCacheEntry
+        ] = ExpiringCache(
             cache_name="state_cache",
             clock=self.clock,
             max_len=100000,
@@ -522,9 +534,9 @@ def __init__(self, hs):
         #
 
         # tracks the amount of work done on state res per room
-        self._state_res_metrics = defaultdict(
+        self._state_res_metrics: DefaultDict[str, _StateResMetrics] = defaultdict(
             _StateResMetrics
-        )  # type: DefaultDict[str, _StateResMetrics]
+        )
 
         self.clock.looping_call(self._report_metrics, 120 * 1000)
 
@@ -536,7 +548,7 @@ async def resolve_state_groups(
         state_groups_ids: Dict[int, StateMap[str]],
         event_map: Optional[Dict[str, EventBase]],
         state_res_store: "StateResolutionStore",
-    ):
+    ) -> _StateCacheEntry:
         """Resolves conflicts between a set of state groups
 
         Always generates a new state group (unless we hit the cache), so should
@@ -629,16 +641,20 @@ async def resolve_events_with_store(
         """
         try:
             with Measure(self.clock, "state._resolve_events") as m:
-                v = KNOWN_ROOM_VERSIONS[room_version]
-                if v.state_res == StateResolutionVersions.V1:
+                room_version_obj = KNOWN_ROOM_VERSIONS[room_version]
+                if room_version_obj.state_res == StateResolutionVersions.V1:
                     return await v1.resolve_events_with_store(
-                        room_id, state_sets, event_map, state_res_store.get_events
+                        room_id,
+                        room_version_obj,
+                        state_sets,
+                        event_map,
+                        state_res_store.get_events,
                     )
                 else:
                     return await v2.resolve_events_with_store(
                         self.clock,
                         room_id,
-                        room_version,
+                        room_version_obj,
                         state_sets,
                         event_map,
                         state_res_store,
@@ -646,13 +662,15 @@ async def resolve_events_with_store(
         finally:
             self._record_state_res_metrics(room_id, m.get_resource_usage())
 
-    def _record_state_res_metrics(self, room_id: str, rusage: ContextResourceUsage):
+    def _record_state_res_metrics(
+        self, room_id: str, rusage: ContextResourceUsage
+    ) -> None:
         room_metrics = self._state_res_metrics[room_id]
         room_metrics.cpu_time += rusage.ru_utime + rusage.ru_stime
         room_metrics.db_time += rusage.db_txn_duration_sec
         room_metrics.db_events += rusage.evt_db_fetch_count
 
-    def _report_metrics(self):
+    def _report_metrics(self) -> None:
         if not self._state_res_metrics:
             # no state res has happened since the last iteration: don't bother logging.
             return
@@ -695,9 +713,9 @@ def _report_biggest(
         items = self._state_res_metrics.items()
 
         # log the N biggest rooms
-        biggest = heapq.nlargest(
+        biggest: List[Tuple[str, _StateResMetrics]] = heapq.nlargest(
             n_to_log, items, key=lambda i: extract_key(i[1])
-        )  # type: List[Tuple[str, _StateResMetrics]]
+        )
         metrics_logger.debug(
             "%i biggest rooms for state-res by %s: %s",
             len(biggest),
@@ -749,7 +767,7 @@ def _make_state_cache_entry(
 
     # failing that, look for the closest match.
     prev_group = None
-    delta_ids = None  # type: Optional[StateMap[str]]
+    delta_ids: Optional[StateMap[str]] = None
 
     for old_group, old_state in state_groups_ids.items():
         n_delta_ids = {k: v for k, v in new_state.items() if old_state.get(k) != v}
@@ -762,16 +780,13 @@ def _make_state_cache_entry(
     )
 
 
-@attr.s(slots=True)
+@attr.s(slots=True, auto_attribs=True)
 class StateResolutionStore:
     """Interface that allows state resolution algorithms to access the database
     in well defined way.
-
-    Args:
-        store (DataStore)
     """
 
-    store = attr.ib()
+    store: "DataStore"
 
     def get_events(
         self, event_ids: Iterable[str], allow_rejected: bool = False
diff --git a/synapse/state/v1.py b/synapse/state/v1.py
index ce255da6fd..92336d7cc8 100644
--- a/synapse/state/v1.py
+++ b/synapse/state/v1.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,7 +29,7 @@
 from synapse import event_auth
 from synapse.api.constants import EventTypes
 from synapse.api.errors import AuthError
-from synapse.api.room_versions import RoomVersions
+from synapse.api.room_versions import RoomVersion, RoomVersions
 from synapse.events import EventBase
 from synapse.types import MutableStateMap, StateMap
 
@@ -42,6 +41,7 @@
 
 async def resolve_events_with_store(
     room_id: str,
+    room_version: RoomVersion,
     state_sets: Sequence[StateMap[str]],
     event_map: Optional[Dict[str, EventBase]],
     state_map_factory: Callable[[Iterable[str]], Awaitable[Dict[str, EventBase]]],
@@ -105,7 +105,7 @@ async def resolve_events_with_store(
     # get the ids of the auth events which allow us to authenticate the
     # conflicted state, picking only from the unconflicting state.
     auth_events = _create_auth_events_from_maps(
-        unconflicted_state, conflicted_state, state_map
+        room_version, unconflicted_state, conflicted_state, state_map
     )
 
     new_needed_events = set(auth_events.values())
@@ -133,7 +133,7 @@ async def resolve_events_with_store(
     state_map.update(state_map_new)
 
     return _resolve_with_state(
-        unconflicted_state, conflicted_state, auth_events, state_map
+        room_version, unconflicted_state, conflicted_state, auth_events, state_map
     )
 
 
@@ -160,7 +160,7 @@ def _seperate(
     """
     state_set_iterator = iter(state_sets)
     unconflicted_state = dict(next(state_set_iterator))
-    conflicted_state = {}  # type: MutableStateMap[Set[str]]
+    conflicted_state: MutableStateMap[Set[str]] = {}
 
     for state_set in state_set_iterator:
         for key, value in state_set.items():
@@ -188,6 +188,7 @@ def _seperate(
 
 
 def _create_auth_events_from_maps(
+    room_version: RoomVersion,
     unconflicted_state: StateMap[str],
     conflicted_state: StateMap[Set[str]],
     state_map: Dict[str, EventBase],
@@ -195,6 +196,7 @@ def _create_auth_events_from_maps(
     """
 
     Args:
+        room_version: The room version.
         unconflicted_state: The unconflicted state map.
         conflicted_state: The conflicted state map.
         state_map:
@@ -206,7 +208,9 @@ def _create_auth_events_from_maps(
     for event_ids in conflicted_state.values():
         for event_id in event_ids:
             if event_id in state_map:
-                keys = event_auth.auth_types_for_event(state_map[event_id])
+                keys = event_auth.auth_types_for_event(
+                    room_version, state_map[event_id]
+                )
                 for key in keys:
                     if key not in auth_events:
                         auth_event_id = unconflicted_state.get(key, None)
@@ -216,6 +220,7 @@ def _create_auth_events_from_maps(
 
 
 def _resolve_with_state(
+    room_version: RoomVersion,
     unconflicted_state_ids: MutableStateMap[str],
     conflicted_state_ids: StateMap[Set[str]],
     auth_event_ids: StateMap[str],
@@ -236,7 +241,9 @@ def _resolve_with_state(
     }
 
     try:
-        resolved_state = _resolve_state_events(conflicted_state, auth_events)
+        resolved_state = _resolve_state_events(
+            room_version, conflicted_state, auth_events
+        )
     except Exception:
         logger.exception("Failed to resolve state")
         raise
@@ -249,7 +256,9 @@ def _resolve_with_state(
 
 
 def _resolve_state_events(
-    conflicted_state: StateMap[List[EventBase]], auth_events: MutableStateMap[EventBase]
+    room_version: RoomVersion,
+    conflicted_state: StateMap[List[EventBase]],
+    auth_events: MutableStateMap[EventBase],
 ) -> StateMap[EventBase]:
     """This is where we actually decide which of the conflicted state to
     use.
@@ -264,21 +273,27 @@ def _resolve_state_events(
     if POWER_KEY in conflicted_state:
         events = conflicted_state[POWER_KEY]
         logger.debug("Resolving conflicted power levels %r", events)
-        resolved_state[POWER_KEY] = _resolve_auth_events(events, auth_events)
+        resolved_state[POWER_KEY] = _resolve_auth_events(
+            room_version, events, auth_events
+        )
 
     auth_events.update(resolved_state)
 
     for key, events in conflicted_state.items():
         if key[0] == EventTypes.JoinRules:
             logger.debug("Resolving conflicted join rules %r", events)
-            resolved_state[key] = _resolve_auth_events(events, auth_events)
+            resolved_state[key] = _resolve_auth_events(
+                room_version, events, auth_events
+            )
 
     auth_events.update(resolved_state)
 
     for key, events in conflicted_state.items():
         if key[0] == EventTypes.Member:
             logger.debug("Resolving conflicted member lists %r", events)
-            resolved_state[key] = _resolve_auth_events(events, auth_events)
+            resolved_state[key] = _resolve_auth_events(
+                room_version, events, auth_events
+            )
 
     auth_events.update(resolved_state)
 
@@ -291,12 +306,14 @@ def _resolve_state_events(
 
 
 def _resolve_auth_events(
-    events: List[EventBase], auth_events: StateMap[EventBase]
+    room_version: RoomVersion, events: List[EventBase], auth_events: StateMap[EventBase]
 ) -> EventBase:
     reverse = list(reversed(_ordered_events(events)))
 
     auth_keys = {
-        key for event in events for key in event_auth.auth_types_for_event(event)
+        key
+        for event in events
+        for key in event_auth.auth_types_for_event(room_version, event)
     }
 
     new_auth_events = {}
diff --git a/synapse/state/v2.py b/synapse/state/v2.py
index e73a548ee4..7b1e8361de 100644
--- a/synapse/state/v2.py
+++ b/synapse/state/v2.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,6 +18,7 @@
 from typing import (
     Any,
     Callable,
+    Collection,
     Dict,
     Generator,
     Iterable,
@@ -36,9 +36,9 @@
 from synapse import event_auth
 from synapse.api.constants import EventTypes
 from synapse.api.errors import AuthError
-from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
+from synapse.api.room_versions import RoomVersion
 from synapse.events import EventBase
-from synapse.types import Collection, MutableStateMap, StateMap
+from synapse.types import MutableStateMap, StateMap
 from synapse.util import Clock
 
 logger = logging.getLogger(__name__)
@@ -53,7 +53,7 @@
 async def resolve_events_with_store(
     clock: Clock,
     room_id: str,
-    room_version: str,
+    room_version: RoomVersion,
     state_sets: Sequence[StateMap[str]],
     event_map: Optional[Dict[str, EventBase]],
     state_res_store: "synapse.state.StateResolutionStore",
@@ -276,7 +276,7 @@ async def _get_auth_chain_difference(
     # event IDs if they appear in the `event_map`. This is the intersection of
     # the event's auth chain with the events in the `event_map` *plus* their
     # auth event IDs.
-    events_to_auth_chain = {}  # type: Dict[str, Set[str]]
+    events_to_auth_chain: Dict[str, Set[str]] = {}
     for event in event_map.values():
         chain = {event.event_id}
         events_to_auth_chain[event.event_id] = chain
@@ -301,17 +301,17 @@ async def _get_auth_chain_difference(
         # ((type, state_key)->event_id) mappings; and (b) we have stripped out
         # unpersisted events and replaced them with the persisted events in
         # their auth chain.
-        state_sets_ids = []  # type: List[Set[str]]
+        state_sets_ids: List[Set[str]] = []
 
         # For each state set, the unpersisted event IDs reachable (by their auth
         # chain) from the events in that set.
-        unpersisted_set_ids = []  # type: List[Set[str]]
+        unpersisted_set_ids: List[Set[str]] = []
 
         for state_set in state_sets:
-            set_ids = set()  # type: Set[str]
+            set_ids: Set[str] = set()
             state_sets_ids.append(set_ids)
 
-            unpersisted_ids = set()  # type: Set[str]
+            unpersisted_ids: Set[str] = set()
             unpersisted_set_ids.append(unpersisted_ids)
 
             for event_id in state_set.values():
@@ -334,7 +334,7 @@ async def _get_auth_chain_difference(
         union = unpersisted_set_ids[0].union(*unpersisted_set_ids[1:])
         intersection = unpersisted_set_ids[0].intersection(*unpersisted_set_ids[1:])
 
-        difference_from_event_map = union - intersection  # type: Collection[str]
+        difference_from_event_map: Collection[str] = union - intersection
     else:
         difference_from_event_map = ()
         state_sets_ids = [set(state_set.values()) for state_set in state_sets]
@@ -458,7 +458,7 @@ async def _reverse_topological_power_sort(
         The sorted list
     """
 
-    graph = {}  # type: Dict[str, Set[str]]
+    graph: Dict[str, Set[str]] = {}
     for idx, event_id in enumerate(event_ids, start=1):
         await _add_event_and_auth_chain_to_graph(
             graph, room_id, event_id, event_map, state_res_store, auth_diff
@@ -497,7 +497,7 @@ def _get_power_order(event_id):
 async def _iterative_auth_checks(
     clock: Clock,
     room_id: str,
-    room_version: str,
+    room_version: RoomVersion,
     event_ids: List[str],
     base_state: StateMap[str],
     event_map: Dict[str, EventBase],
@@ -519,7 +519,6 @@ async def _iterative_auth_checks(
         Returns the final updated state
     """
     resolved_state = dict(base_state)
-    room_version_obj = KNOWN_ROOM_VERSIONS[room_version]
 
     for idx, event_id in enumerate(event_ids, start=1):
         event = event_map[event_id]
@@ -538,7 +537,7 @@ async def _iterative_auth_checks(
                 if ev.rejected_reason is None:
                     auth_events[(ev.type, ev.state_key)] = ev
 
-        for key in event_auth.auth_types_for_event(event):
+        for key in event_auth.auth_types_for_event(room_version, event):
             if key in resolved_state:
                 ev_id = resolved_state[key]
                 ev = await _get_event(room_id, ev_id, event_map, state_res_store)
@@ -548,7 +547,7 @@ async def _iterative_auth_checks(
 
         try:
             event_auth.check(
-                room_version_obj,
+                room_version,
                 event,
                 auth_events,
                 do_sig_check=False,
@@ -657,7 +656,7 @@ async def _get_mainline_depth_for_event(
     """
 
     room_id = event.room_id
-    tmp_event = event  # type: Optional[EventBase]
+    tmp_event: Optional[EventBase] = event
 
     # We do an iterative search, replacing `event with the power level in its
     # auth events (if any)
@@ -767,7 +766,7 @@ def lexicographical_topological_sort(
     # outgoing edges, c.f.
     # https://en.wikipedia.org/wiki/Topological_sorting#Kahn's_algorithm
     outdegree_map = graph
-    reverse_graph = {}  # type: Dict[str, Set[str]]
+    reverse_graph: Dict[str, Set[str]] = {}
 
     # Lists of nodes with zero out degree. Is actually a tuple of
     # `(key(node), node)` so that sorting does the right thing
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 0b9007e51f..105e4e1fec 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018,2019 New Vector Ltd
 #
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 240905329f..0623da9aa1 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -15,15 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import random
 from abc import ABCMeta
-from typing import TYPE_CHECKING, Any, Iterable, Optional, Union
+from typing import TYPE_CHECKING, Any, Collection, Iterable, Optional, Union
 
 from synapse.storage.database import LoggingTransaction  # noqa: F401
 from synapse.storage.database import make_in_list_sql_clause  # noqa: F401
 from synapse.storage.database import DatabasePool
 from synapse.storage.types import Connection
-from synapse.types import Collection, StreamToken, get_domain_from_id
+from synapse.types import StreamToken, get_domain_from_id
 from synapse.util import json_decoder
 
 if TYPE_CHECKING:
@@ -45,7 +43,6 @@ def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer"
         self._clock = hs.get_clock()
         self.database_engine = database.engine
         self.db_pool = database
-        self.rand = random.SystemRandom()
 
     def process_replication_rows(
         self,
@@ -70,6 +67,7 @@ def _invalidate_state_caches(
             self._attempt_to_invalidate_cache("is_host_joined", (room_id, host))
 
         self._attempt_to_invalidate_cache("get_users_in_room", (room_id,))
+        self._attempt_to_invalidate_cache("get_users_in_room_with_profiles", (room_id,))
         self._attempt_to_invalidate_cache("get_room_summary", (room_id,))
         self._attempt_to_invalidate_cache("get_current_state_ids", (room_id,))
 
@@ -115,7 +113,7 @@ def db_to_json(db_content: Union[memoryview, bytes, bytearray, str]) -> Any:
         db_content = db_content.tobytes()
 
     # Decode it to a Unicode string before feeding it to the JSON decoder, since
-    # Python 3.5 does not support deserializing bytes.
+    # it only supports handling strings
     if isinstance(db_content, (bytes, bytearray)):
         db_content = db_content.decode("utf8")
 
diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py
index ccb06aab39..82b31d24f1 100644
--- a/synapse/storage/background_updates.py
+++ b/synapse/storage/background_updates.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -93,14 +92,12 @@ def __init__(self, hs: "HomeServer", database: "DatabasePool"):
         self.db_pool = database
 
         # if a background update is currently running, its name.
-        self._current_background_update = None  # type: Optional[str]
-
-        self._background_update_performance = (
-            {}
-        )  # type: Dict[str, BackgroundUpdatePerformance]
-        self._background_update_handlers = (
-            {}
-        )  # type: Dict[str, Callable[[JsonDict, int], Awaitable[int]]]
+        self._current_background_update: Optional[str] = None
+
+        self._background_update_performance: Dict[str, BackgroundUpdatePerformance] = {}
+        self._background_update_handlers: Dict[
+            str, Callable[[JsonDict, int], Awaitable[int]]
+        ] = {}
         self._all_done = False
 
     def start_doing_background_updates(self) -> None:
@@ -412,7 +409,7 @@ def create_index_sqlite(conn: Connection) -> None:
             c.execute(sql)
 
         if isinstance(self.db_pool.engine, engines.PostgresEngine):
-            runner = create_index_psql  # type: Optional[Callable[[Connection], None]]
+            runner: Optional[Callable[[Connection], None]] = create_index_psql
         elif psql_only:
             runner = None
         else:
diff --git a/synapse/storage/database.py b/synapse/storage/database.py
index 94590e7b45..95d2caff62 100644
--- a/synapse/storage/database.py
+++ b/synapse/storage/database.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -16,11 +15,13 @@
 # limitations under the License.
 import logging
 import time
+from collections import defaultdict
 from sys import intern
 from time import monotonic as monotonic_time
 from typing import (
     Any,
     Callable,
+    Collection,
     Dict,
     Iterable,
     Iterator,
@@ -40,6 +41,7 @@
 
 from synapse.api.errors import StoreError
 from synapse.config.database import DatabaseConnectionConfig
+from synapse.logging import opentracing
 from synapse.logging.context import (
     LoggingContext,
     current_context,
@@ -49,7 +51,6 @@
 from synapse.storage.background_updates import BackgroundUpdater
 from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
 from synapse.storage.types import Connection, Cursor
-from synapse.types import Collection
 
 # python 3 does not have a maximum int value
 MAX_TXN_ID = 2 ** 63 - 1
@@ -91,12 +92,18 @@ def make_pool(
     db_args = dict(db_config.config.get("args", {}))
     db_args.setdefault("cp_reconnect", True)
 
+    def _on_new_connection(conn):
+        # Ensure we have a logging context so we can correctly track queries,
+        # etc.
+        with LoggingContext("db.on_new_connection"):
+            engine.on_new_connection(
+                LoggingDatabaseConnection(conn, engine, "on_new_connection")
+            )
+
     return adbapi.ConnectionPool(
         db_config.config["name"],
         cp_reactor=reactor,
-        cp_openfun=lambda conn: engine.on_new_connection(
-            LoggingDatabaseConnection(conn, engine, "on_new_connection")
-        ),
+        cp_openfun=_on_new_connection,
         **db_args,
     )
 
@@ -105,7 +112,7 @@ def make_conn(
     db_config: DatabaseConnectionConfig,
     engine: BaseDatabaseEngine,
     default_txn_name: str,
-) -> Connection:
+) -> "LoggingDatabaseConnection":
     """Make a new connection to the database and return it.
 
     Returns:
@@ -172,10 +179,7 @@ def __getattr__(self, name):
 
 
 # The type of entry which goes on our after_callbacks and exception_callbacks lists.
-#
-# Python 3.5.2 doesn't support Callable with an ellipsis, so we wrap it in quotes so
-# that mypy sees the type but the runtime python doesn't.
-_CallbackListEntry = Tuple["Callable[..., None]", Iterable[Any], Dict[str, Any]]
+_CallbackListEntry = Tuple[Callable[..., None], Iterable[Any], Dict[str, Any]]
 
 
 R = TypeVar("R")
@@ -222,7 +226,7 @@ def __init__(
         self.after_callbacks = after_callbacks
         self.exception_callbacks = exception_callbacks
 
-    def call_after(self, callback: "Callable[..., None]", *args: Any, **kwargs: Any):
+    def call_after(self, callback: Callable[..., None], *args: Any, **kwargs: Any):
         """Call the given callback on the main twisted thread after the
         transaction has finished. Used to invalidate the caches on the
         correct thread.
@@ -234,7 +238,7 @@ def call_after(self, callback: "Callable[..., None]", *args: Any, **kwargs: Any)
         self.after_callbacks.append((callback, args, kwargs))
 
     def call_on_exception(
-        self, callback: "Callable[..., None]", *args: Any, **kwargs: Any
+        self, callback: Callable[..., None], *args: Any, **kwargs: Any
     ):
         # if self.exception_callbacks is None, that means that whatever constructed the
         # LoggingTransaction isn't expecting there to be any callbacks; assert that
@@ -317,7 +321,14 @@ def _do_execute(self, func: Callable[..., R], sql: str, *args: Any) -> R:
         start = time.time()
 
         try:
-            return func(sql, *args)
+            with opentracing.start_active_span(
+                "db.query",
+                tags={
+                    opentracing.tags.DATABASE_TYPE: "sql",
+                    opentracing.tags.DATABASE_STATEMENT: sql,
+                },
+            ):
+                return func(sql, *args)
         except Exception as e:
             sql_logger.debug("[SQL FAIL] {%s} %s", self.name, e)
             raise
@@ -387,6 +398,7 @@ def __init__(
     ):
         self.hs = hs
         self._clock = hs.get_clock()
+        self._txn_limit = database_config.config.get("txn_limit", 0)
         self._database_config = database_config
         self._db_pool = make_pool(hs.get_reactor(), database_config, engine)
 
@@ -396,6 +408,9 @@ def __init__(
         self._current_txn_total_time = 0.0
         self._previous_loop_ts = 0.0
 
+        # Transaction counter: key is the twisted thread id, value is the current count
+        self._txn_counters: Dict[int, int] = defaultdict(int)
+
         # TODO(paul): These can eventually be removed once the metrics code
         #   is running in mainline, and we have some nice monitoring frontends
         #   to watch it
@@ -486,9 +501,9 @@ def new_transaction(
         desc: str,
         after_callbacks: List[_CallbackListEntry],
         exception_callbacks: List[_CallbackListEntry],
-        func: "Callable[..., R]",
+        func: Callable[..., R],
         *args: Any,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> R:
         """Start a new database transaction with the given connection.
 
@@ -529,9 +544,17 @@ def new_transaction(
                     exception_callbacks=exception_callbacks,
                 )
                 try:
-                    r = func(cursor, *args, **kwargs)
-                    conn.commit()
-                    return r
+                    with opentracing.start_active_span(
+                        "db.txn",
+                        tags={
+                            opentracing.SynapseTags.DB_TXN_DESC: desc,
+                            opentracing.SynapseTags.DB_TXN_ID: name,
+                        },
+                    ):
+                        r = func(cursor, *args, **kwargs)
+                        opentracing.log_kv({"message": "commit"})
+                        conn.commit()
+                        return r
                 except self.engine.module.OperationalError as e:
                     # This can happen if the database disappears mid
                     # transaction.
@@ -545,7 +568,8 @@ def new_transaction(
                     if i < N:
                         i += 1
                         try:
-                            conn.rollback()
+                            with opentracing.start_active_span("db.rollback"):
+                                conn.rollback()
                         except self.engine.module.Error as e1:
                             transaction_logger.warning("[TXN EROLL] {%s} %s", name, e1)
                         continue
@@ -558,7 +582,8 @@ def new_transaction(
                         if i < N:
                             i += 1
                             try:
-                                conn.rollback()
+                                with opentracing.start_active_span("db.rollback"):
+                                    conn.rollback()
                             except self.engine.module.Error as e1:
                                 transaction_logger.warning(
                                     "[TXN EROLL] {%s} %s",
@@ -619,10 +644,10 @@ def new_transaction(
     async def runInteraction(
         self,
         desc: str,
-        func: "Callable[..., R]",
+        func: Callable[..., R],
         *args: Any,
         db_autocommit: bool = False,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> R:
         """Starts a transaction on the database and runs a given function
 
@@ -650,23 +675,24 @@ async def runInteraction(
         Returns:
             The result of func
         """
-        after_callbacks = []  # type: List[_CallbackListEntry]
-        exception_callbacks = []  # type: List[_CallbackListEntry]
+        after_callbacks: List[_CallbackListEntry] = []
+        exception_callbacks: List[_CallbackListEntry] = []
 
         if not current_context():
             logger.warning("Starting db txn '%s' from sentinel context", desc)
 
         try:
-            result = await self.runWithConnection(
-                self.new_transaction,
-                desc,
-                after_callbacks,
-                exception_callbacks,
-                func,
-                *args,
-                db_autocommit=db_autocommit,
-                **kwargs,
-            )
+            with opentracing.start_active_span(f"db.{desc}"):
+                result = await self.runWithConnection(
+                    self.new_transaction,
+                    desc,
+                    after_callbacks,
+                    exception_callbacks,
+                    func,
+                    *args,
+                    db_autocommit=db_autocommit,
+                    **kwargs,
+                )
 
             for after_callback, after_args, after_kwargs in after_callbacks:
                 after_callback(*after_args, **after_kwargs)
@@ -679,10 +705,10 @@ async def runInteraction(
 
     async def runWithConnection(
         self,
-        func: "Callable[..., R]",
+        func: Callable[..., R],
         *args: Any,
         db_autocommit: bool = False,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> R:
         """Wraps the .runWithConnection() method on the underlying db_pool.
 
@@ -719,26 +745,48 @@ def inner_func(conn, *args, **kwargs):
             # pool).
             assert not self.engine.in_transaction(conn)
 
-            with LoggingContext("runWithConnection", parent_context) as context:
-                sched_duration_sec = monotonic_time() - start_time
-                sql_scheduling_timer.observe(sched_duration_sec)
-                context.add_database_scheduled(sched_duration_sec)
-
-                if self.engine.is_connection_closed(conn):
-                    logger.debug("Reconnecting closed database connection")
-                    conn.reconnect()
-
-                try:
-                    if db_autocommit:
-                        self.engine.attempt_to_set_autocommit(conn, True)
-
-                    db_conn = LoggingDatabaseConnection(
-                        conn, self.engine, "runWithConnection"
-                    )
-                    return func(db_conn, *args, **kwargs)
-                finally:
-                    if db_autocommit:
-                        self.engine.attempt_to_set_autocommit(conn, False)
+            with LoggingContext(
+                str(curr_context), parent_context=parent_context
+            ) as context:
+                with opentracing.start_active_span(
+                    operation_name="db.connection",
+                ):
+                    sched_duration_sec = monotonic_time() - start_time
+                    sql_scheduling_timer.observe(sched_duration_sec)
+                    context.add_database_scheduled(sched_duration_sec)
+
+                    if self._txn_limit > 0:
+                        tid = self._db_pool.threadID()
+                        self._txn_counters[tid] += 1
+
+                        if self._txn_counters[tid] > self._txn_limit:
+                            logger.debug(
+                                "Reconnecting database connection over transaction limit"
+                            )
+                            conn.reconnect()
+                            opentracing.log_kv(
+                                {"message": "reconnected due to txn limit"}
+                            )
+                            self._txn_counters[tid] = 1
+
+                    if self.engine.is_connection_closed(conn):
+                        logger.debug("Reconnecting closed database connection")
+                        conn.reconnect()
+                        opentracing.log_kv({"message": "reconnected"})
+                        if self._txn_limit > 0:
+                            self._txn_counters[tid] = 1
+
+                    try:
+                        if db_autocommit:
+                            self.engine.attempt_to_set_autocommit(conn, True)
+
+                        db_conn = LoggingDatabaseConnection(
+                            conn, self.engine, "runWithConnection"
+                        )
+                        return func(db_conn, *args, **kwargs)
+                    finally:
+                        if db_autocommit:
+                            self.engine.attempt_to_set_autocommit(conn, False)
 
         return await make_deferred_yieldable(
             self._db_pool.runWithConnection(inner_func, *args, **kwargs)
@@ -775,7 +823,7 @@ async def execute(
         desc: str,
         decoder: Optional[Callable[[Cursor], R]],
         query: str,
-        *args: Any
+        *args: Any,
     ) -> R:
         """Runs a single query for a result set.
 
@@ -805,31 +853,16 @@ async def simple_insert(
         self,
         table: str,
         values: Dict[str, Any],
-        or_ignore: bool = False,
         desc: str = "simple_insert",
-    ) -> bool:
+    ) -> None:
         """Executes an INSERT query on the named table.
 
         Args:
             table: string giving the table name
             values: dict of new column names and values for them
-            or_ignore: bool stating whether an exception should be raised
-                when a conflicting row already exists. If True, False will be
-                returned by the function instead
             desc: description of the transaction, for logging and metrics
-
-        Returns:
-             Whether the row was inserted or not. Only useful when `or_ignore` is True
         """
-        try:
-            await self.runInteraction(desc, self.simple_insert_txn, table, values)
-        except self.engine.module.IntegrityError:
-            # We have to do or_ignore flag at this layer, since we can't reuse
-            # a cursor after we receive an error from the db.
-            if not or_ignore:
-                raise
-            return False
-        return True
+        await self.runInteraction(desc, self.simple_insert_txn, table, values)
 
     @staticmethod
     def simple_insert_txn(
@@ -880,7 +913,7 @@ def simple_insert_many_txn(
         # The sort is to ensure that we don't rely on dictionary iteration
         # order.
         keys, vals = zip(
-            *[zip(*(sorted(i.items(), key=lambda kv: kv[0]))) for i in values if i]
+            *(zip(*(sorted(i.items(), key=lambda kv: kv[0]))) for i in values if i)
         )
 
         for k in keys:
@@ -900,21 +933,21 @@ async def simple_upsert(
         table: str,
         keyvalues: Dict[str, Any],
         values: Dict[str, Any],
-        insertion_values: Dict[str, Any] = {},
+        insertion_values: Optional[Dict[str, Any]] = None,
         desc: str = "simple_upsert",
         lock: bool = True,
-    ) -> Optional[bool]:
+    ) -> bool:
         """
 
         `lock` should generally be set to True (the default), but can be set
         to False if either of the following are true:
-
-        * there is a UNIQUE INDEX on the key columns. In this case a conflict
-          will cause an IntegrityError in which case this function will retry
-          the update.
-
-        * we somehow know that we are the only thread which will be updating
-          this table.
+            1. there is a UNIQUE INDEX on the key columns. In this case a conflict
+            will cause an IntegrityError in which case this function will retry
+            the update.
+            2. we somehow know that we are the only thread which will be updating
+            this table.
+        As an additional note, this parameter only matters for old SQLite versions
+        because we will use native upserts otherwise.
 
         Args:
             table: The table to upsert into
@@ -924,9 +957,11 @@ async def simple_upsert(
             desc: description of the transaction, for logging and metrics
             lock: True to lock the table when doing the upsert.
         Returns:
-            Native upserts always return None. Emulated upserts return True if a
-            new entry was created, False if an existing one was updated.
+            Returns True if a row was inserted or updated (i.e. if `values` is
+            not empty then this always returns True)
         """
+        insertion_values = insertion_values or {}
+
         attempts = 0
         while True:
             try:
@@ -964,9 +999,9 @@ def simple_upsert_txn(
         table: str,
         keyvalues: Dict[str, Any],
         values: Dict[str, Any],
-        insertion_values: Dict[str, Any] = {},
+        insertion_values: Optional[Dict[str, Any]] = None,
         lock: bool = True,
-    ) -> Optional[bool]:
+    ) -> bool:
         """
         Pick the UPSERT method which works best on the platform. Either the
         native one (Pg9.5+, recent SQLites), or fall back to an emulated method.
@@ -979,14 +1014,15 @@ def simple_upsert_txn(
             insertion_values: additional key/values to use only when inserting
             lock: True to lock the table when doing the upsert.
         Returns:
-            Native upserts always return None. Emulated upserts return True if a
-            new entry was created, False if an existing one was updated.
+            Returns True if a row was inserted or updated (i.e. if `values` is
+            not empty then this always returns True)
         """
+        insertion_values = insertion_values or {}
+
         if self.engine.can_native_upsert and table not in self._unsafe_to_upsert_tables:
-            self.simple_upsert_txn_native_upsert(
+            return self.simple_upsert_txn_native_upsert(
                 txn, table, keyvalues, values, insertion_values=insertion_values
             )
-            return None
         else:
             return self.simple_upsert_txn_emulated(
                 txn,
@@ -1003,7 +1039,7 @@ def simple_upsert_txn_emulated(
         table: str,
         keyvalues: Dict[str, Any],
         values: Dict[str, Any],
-        insertion_values: Dict[str, Any] = {},
+        insertion_values: Optional[Dict[str, Any]] = None,
         lock: bool = True,
     ) -> bool:
         """
@@ -1014,9 +1050,11 @@ def simple_upsert_txn_emulated(
             insertion_values: additional key/values to use only when inserting
             lock: True to lock the table when doing the upsert.
         Returns:
-            Returns True if a new entry was created, False if an existing
-            one was updated.
+            Returns True if a row was inserted or updated (i.e. if `values` is
+            not empty then this always returns True)
         """
+        insertion_values = insertion_values or {}
+
         # We need to lock the table :(, unless we're *really* careful
         if lock:
             self.engine.lock_table(txn, table)
@@ -1053,11 +1091,10 @@ def _getwhere(key):
 
             txn.execute(sql, sqlargs)
             if txn.rowcount > 0:
-                # successfully updated at least one row.
-                return False
+                return True
 
         # We didn't find any existing rows, so insert a new one
-        allvalues = {}  # type: Dict[str, Any]
+        allvalues: Dict[str, Any] = {}
         allvalues.update(keyvalues)
         allvalues.update(values)
         allvalues.update(insertion_values)
@@ -1077,20 +1114,24 @@ def simple_upsert_txn_native_upsert(
         table: str,
         keyvalues: Dict[str, Any],
         values: Dict[str, Any],
-        insertion_values: Dict[str, Any] = {},
-    ) -> None:
+        insertion_values: Optional[Dict[str, Any]] = None,
+    ) -> bool:
         """
-        Use the native UPSERT functionality in recent PostgreSQL versions.
+        Use the native UPSERT functionality in PostgreSQL.
 
         Args:
             table: The table to upsert into
             keyvalues: The unique key tables and their new values
             values: The nonunique columns and their new values
             insertion_values: additional key/values to use only when inserting
+
+        Returns:
+            Returns True if a row was inserted or updated (i.e. if `values` is
+            not empty then this always returns True)
         """
-        allvalues = {}  # type: Dict[str, Any]
+        allvalues: Dict[str, Any] = {}
         allvalues.update(keyvalues)
-        allvalues.update(insertion_values)
+        allvalues.update(insertion_values or {})
 
         if not values:
             latter = "NOTHING"
@@ -1107,6 +1148,8 @@ def simple_upsert_txn_native_upsert(
         )
         txn.execute(sql, list(allvalues.values()))
 
+        return bool(txn.rowcount)
+
     async def simple_upsert_many(
         self,
         table: str,
@@ -1224,7 +1267,7 @@ def simple_upsert_many_txn_native_upsert(
             value_values: A list of each row's value column values.
                 Ignored if value_names is empty.
         """
-        allnames = []  # type: List[str]
+        allnames: List[str] = []
         allnames.extend(key_names)
         allnames.extend(value_names)
 
@@ -1513,7 +1556,7 @@ async def simple_select_many_batch(
         column: str,
         iterable: Iterable[Any],
         retcols: Iterable[str],
-        keyvalues: Dict[str, Any] = {},
+        keyvalues: Optional[Dict[str, Any]] = None,
         desc: str = "simple_select_many_batch",
         batch_size: int = 100,
     ) -> List[Any]:
@@ -1531,7 +1574,9 @@ async def simple_select_many_batch(
             desc: description of the transaction, for logging and metrics
             batch_size: the number of rows for each select query
         """
-        results = []  # type: List[Dict[str, Any]]
+        keyvalues = keyvalues or {}
+
+        results: List[Dict[str, Any]] = []
 
         if not iterable:
             return results
@@ -1943,7 +1988,7 @@ def simple_select_list_paginate_txn(
             raise ValueError("order_direction must be one of 'ASC' or 'DESC'.")
 
         where_clause = "WHERE " if filters or keyvalues or exclude_keyvalues else ""
-        arg_list = []  # type: List[Any]
+        arg_list: List[Any] = []
         if filters:
             where_clause += " AND ".join("%s LIKE ?" % (k,) for k in filters)
             arg_list += list(filters.values())
@@ -2059,69 +2104,18 @@ def make_in_list_sql_clause(
 KV = TypeVar("KV")
 
 
-def make_tuple_comparison_clause(
-    database_engine: BaseDatabaseEngine, keys: List[Tuple[str, KV]]
-) -> Tuple[str, List[KV]]:
+def make_tuple_comparison_clause(keys: List[Tuple[str, KV]]) -> Tuple[str, List[KV]]:
     """Returns a tuple comparison SQL clause
 
-    Depending what the SQL engine supports, builds a SQL clause that looks like either
-    "(a, b) > (?, ?)", or "(a > ?) OR (a == ? AND b > ?)".
+    Builds a SQL clause that looks like "(a, b) > (?, ?)"
 
     Args:
-        database_engine
         keys: A set of (column, value) pairs to be compared.
 
     Returns:
         A tuple of SQL query and the args
     """
-    if database_engine.supports_tuple_comparison:
-        return (
-            "(%s) > (%s)" % (",".join(k[0] for k in keys), ",".join("?" for _ in keys)),
-            [k[1] for k in keys],
-        )
-
-    # we want to build a clause
-    #    (a > ?) OR
-    #    (a == ? AND b > ?) OR
-    #    (a == ? AND b == ? AND c > ?)
-    #    ...
-    #    (a == ? AND b == ? AND ... AND z > ?)
-    #
-    # or, equivalently:
-    #
-    #  (a > ? OR (a == ? AND
-    #    (b > ? OR (b == ? AND
-    #      ...
-    #        (y > ? OR (y == ? AND
-    #          z > ?
-    #        ))
-    #      ...
-    #    ))
-    #  ))
-    #
-    # which itself is equivalent to (and apparently easier for the query optimiser):
-    #
-    #  (a >= ? AND (a > ? OR
-    #    (b >= ? AND (b > ? OR
-    #      ...
-    #        (y >= ? AND (y > ? OR
-    #          z > ?
-    #        ))
-    #      ...
-    #    ))
-    #  ))
-    #
-    #
-
-    clause = ""
-    args = []  # type: List[KV]
-    for k, v in keys[:-1]:
-        clause = clause + "(%s >= ? AND (%s > ? OR " % (k, k)
-        args.extend([v, v])
-
-    (k, v) = keys[-1]
-    clause += "%s > ?" % (k,)
-    args.append(v)
-
-    clause += "))" * (len(keys) - 1)
-    return clause, args
+    return (
+        "(%s) > (%s)" % (",".join(k[0] for k in keys), ",".join("?" for _ in keys)),
+        [k[1] for k in keys],
+    )
diff --git a/synapse/storage/databases/__init__.py b/synapse/storage/databases/__init__.py
index 379c78bb83..20b755056b 100644
--- a/synapse/storage/databases/__init__.py
+++ b/synapse/storage/databases/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py
index 1d44c3aa2c..01b918e12e 100644
--- a/synapse/storage/databases/main/__init__.py
+++ b/synapse/storage/databases/main/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 # Copyright 2019-2021 The Matrix.org Foundation C.I.C.
@@ -18,9 +17,9 @@
 import logging
 from typing import List, Optional, Tuple
 
-from synapse.api.constants import PresenceState
 from synapse.config.homeserver import HomeServerConfig
 from synapse.storage.database import DatabasePool
+from synapse.storage.databases.main.stats import UserSortOrder
 from synapse.storage.engines import PostgresEngine
 from synapse.storage.util.id_generators import (
     IdGenerator,
@@ -47,11 +46,12 @@
 from .filtering import FilteringStore
 from .group_server import GroupServerStore
 from .keys import KeyStore
+from .lock import LockStore
 from .media_repository import MediaRepositoryStore
 from .metrics import ServerMetricsStore
 from .monthly_active_users import MonthlyActiveUsersStore
 from .openid import OpenIdStore
-from .presence import PresenceStore, UserPresenceState
+from .presence import PresenceStore
 from .profile import ProfileStore
 from .purge_events import PurgeEventsStore
 from .push_rule import PushRuleStore
@@ -68,7 +68,7 @@
 from .stats import StatsStore
 from .stream import StreamStore
 from .tags import TagsStore
-from .transactions import TransactionStore
+from .transactions import TransactionWorkerStore
 from .ui_auth import UIAuthStore
 from .user_directory import UserDirectoryStore
 from .user_erasure_store import UserErasureStore
@@ -84,7 +84,7 @@ class DataStore(
     StreamStore,
     ProfileStore,
     PresenceStore,
-    TransactionStore,
+    TransactionWorkerStore,
     DirectoryStore,
     KeyStore,
     StateStore,
@@ -120,18 +120,13 @@ class DataStore(
     CacheInvalidationWorkerStore,
     ServerMetricsStore,
     EventForwardExtremitiesStore,
+    LockStore,
 ):
     def __init__(self, database: DatabasePool, db_conn, hs):
         self.hs = hs
         self._clock = hs.get_clock()
         self.database_engine = database.engine
 
-        self._presence_id_gen = StreamIdGenerator(
-            db_conn, "presence_stream", "stream_id"
-        )
-        self._public_room_id_gen = StreamIdGenerator(
-            db_conn, "public_room_list_stream", "stream_id"
-        )
         self._device_list_id_gen = StreamIdGenerator(
             db_conn,
             "device_lists_stream",
@@ -172,26 +167,12 @@ def __init__(self, database: DatabasePool, db_conn, hs):
                 sequence_name="cache_invalidation_stream_seq",
                 writers=[],
             )
+
         else:
             self._cache_id_gen = None
 
         super().__init__(database, db_conn, hs)
 
-        self._presence_on_startup = self._get_active_presence(db_conn)
-
-        presence_cache_prefill, min_presence_val = self.db_pool.get_cache_dict(
-            db_conn,
-            "presence_stream",
-            entity_column="user_id",
-            stream_column="stream_id",
-            max_value=self._presence_id_gen.get_current_token(),
-        )
-        self.presence_stream_cache = StreamChangeCache(
-            "PresenceStreamChangeCache",
-            min_presence_val,
-            prefilled_cache=presence_cache_prefill,
-        )
-
         device_list_max = self._device_list_id_gen.get_current_token()
         self._device_list_stream_cache = StreamChangeCache(
             "DeviceListStreamChangeCache", device_list_max
@@ -238,32 +219,6 @@ def __init__(self, database: DatabasePool, db_conn, hs):
     def get_device_stream_token(self) -> int:
         return self._device_list_id_gen.get_current_token()
 
-    def take_presence_startup_info(self):
-        active_on_startup = self._presence_on_startup
-        self._presence_on_startup = None
-        return active_on_startup
-
-    def _get_active_presence(self, db_conn):
-        """Fetch non-offline presence from the database so that we can register
-        the appropriate time outs.
-        """
-
-        sql = (
-            "SELECT user_id, state, last_active_ts, last_federation_update_ts,"
-            " last_user_sync_ts, status_msg, currently_active FROM presence_stream"
-            " WHERE state != ?"
-        )
-
-        txn = db_conn.cursor()
-        txn.execute(sql, (PresenceState.OFFLINE,))
-        rows = self.db_pool.cursor_to_dict(txn)
-        txn.close()
-
-        for row in rows:
-            row["currently_active"] = bool(row["currently_active"])
-
-        return [UserPresenceState(**row) for row in rows]
-
     async def get_users(self) -> List[JsonDict]:
         """Function to retrieve a list of users in users table.
 
@@ -292,6 +247,8 @@ async def get_users_paginate(
         name: Optional[str] = None,
         guests: bool = True,
         deactivated: bool = False,
+        order_by: str = UserSortOrder.USER_ID.value,
+        direction: str = "f",
     ) -> Tuple[List[JsonDict], int]:
         """Function to retrieve a paginated list of users from
         users list. This will return a json list of users and the
@@ -304,6 +261,8 @@ async def get_users_paginate(
             name: search for local part of user_id or display name
             guests: whether to in include guest users
             deactivated: whether to include deactivated users
+            order_by: the sort order of the returned list
+            direction: sort ascending or descending
         Returns:
             A tuple of a list of mappings from user to information and a count of total users.
         """
@@ -312,6 +271,14 @@ def get_users_paginate_txn(txn):
             filters = []
             args = [self.hs.config.server_name]
 
+            # Set ordering
+            order_by_column = UserSortOrder(order_by).value
+
+            if direction == "b":
+                order = "DESC"
+            else:
+                order = "ASC"
+
             # `name` is in database already in lower case
             if name:
                 filters.append("(name LIKE ? OR LOWER(displayname) LIKE ?)")
@@ -328,22 +295,22 @@ def get_users_paginate_txn(txn):
 
             where_clause = "WHERE " + " AND ".join(filters) if len(filters) > 0 else ""
 
-            sql_base = """
+            sql_base = f"""
                 FROM users as u
                 LEFT JOIN profiles AS p ON u.name = '@' || p.user_id || ':' || ?
-                {}
-                """.format(
-                where_clause
-            )
+                {where_clause}
+                """
             sql = "SELECT COUNT(*) as total_users " + sql_base
             txn.execute(sql, args)
             count = txn.fetchone()[0]
 
-            sql = (
-                "SELECT name, user_type, is_guest, admin, deactivated, shadow_banned, displayname, avatar_url "
-                + sql_base
-                + " ORDER BY u.name LIMIT ? OFFSET ?"
-            )
+            sql = f"""
+                SELECT name, user_type, is_guest, admin, deactivated, shadow_banned,
+                displayname, avatar_url, creation_ts * 1000 as creation_ts
+                {sql_base}
+                ORDER BY {order_by_column} {order}, u.name ASC
+                LIMIT ? OFFSET ?
+            """
             args += [limit, start]
             txn.execute(sql, args)
             users = self.db_pool.cursor_to_dict(txn)
diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py
index a277a1ef13..1d02795f43 100644
--- a/synapse/storage/databases/main/account_data.py
+++ b/synapse/storage/databases/main/account_data.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/storage/databases/main/appservice.py b/synapse/storage/databases/main/appservice.py
index 85bb853d33..e2d1b758bd 100644
--- a/synapse/storage/databases/main/appservice.py
+++ b/synapse/storage/databases/main/appservice.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -49,9 +48,7 @@ def _make_exclusive_regex(
     ]
     if exclusive_user_regexes:
         exclusive_user_regex = "|".join("(" + r + ")" for r in exclusive_user_regexes)
-        exclusive_user_pattern = re.compile(
-            exclusive_user_regex
-        )  # type: Optional[Pattern]
+        exclusive_user_pattern: Optional[Pattern] = re.compile(exclusive_user_regex)
     else:
         # We handle this case specially otherwise the constructed regex
         # will always match
diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py
index 1e7637a6f5..c57ae5ef15 100644
--- a/synapse/storage/databases/main/cache.py
+++ b/synapse/storage/databases/main/cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -169,10 +168,11 @@ def _invalidate_caches_for_event(
         backfilled,
     ):
         self._invalidate_get_event_cache(event_id)
+        self.have_seen_event.invalidate((room_id, event_id))
 
         self.get_latest_event_ids_in_room.invalidate((room_id,))
 
-        self.get_unread_event_push_actions_by_room_for_user.invalidate_many((room_id,))
+        self.get_unread_event_push_actions_by_room_for_user.invalidate((room_id,))
 
         if not backfilled:
             self._events_stream_cache.entity_has_changed(room_id, stream_ordering)
@@ -185,8 +185,8 @@ def _invalidate_caches_for_event(
             self.get_invited_rooms_for_local_user.invalidate((state_key,))
 
         if relates_to:
-            self.get_relations_for_event.invalidate_many((relates_to,))
-            self.get_aggregation_groups_for_event.invalidate_many((relates_to,))
+            self.get_relations_for_event.invalidate((relates_to,))
+            self.get_aggregation_groups_for_event.invalidate((relates_to,))
             self.get_applicable_edit.invalidate((relates_to,))
 
     async def invalidate_cache_and_stream(self, cache_name: str, keys: Tuple[Any, ...]):
diff --git a/synapse/storage/databases/main/censor_events.py b/synapse/storage/databases/main/censor_events.py
index 3e26d5ba87..f22c1f241b 100644
--- a/synapse/storage/databases/main/censor_events.py
+++ b/synapse/storage/databases/main/censor_events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py
index 6d18e692b0..074b077bef 100644
--- a/synapse/storage/databases/main/client_ips.py
+++ b/synapse/storage/databases/main/client_ips.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -298,7 +297,6 @@ def _devices_last_seen_update_txn(txn):
             #      times, which is fine.
 
             where_clause, where_args = make_tuple_comparison_clause(
-                self.database_engine,
                 [("user_id", last_user_id), ("device_id", last_device_id)],
             )
 
@@ -438,7 +436,7 @@ class ClientIpStore(ClientIpWorkerStore):
     def __init__(self, database: DatabasePool, db_conn, hs):
 
         self.client_ip_last_seen = LruCache(
-            cache_name="client_ip_last_seen", keylen=4, max_size=50000
+            cache_name="client_ip_last_seen", max_size=50000
         )
 
         super().__init__(database, db_conn, hs)
diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py
index 691080ce74..c55508867d 100644
--- a/synapse/storage/databases/main/deviceinbox.py
+++ b/synapse/storage/databases/main/deviceinbox.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,6 +15,7 @@
 import logging
 from typing import List, Optional, Tuple
 
+from synapse.logging import issue9533_logger
 from synapse.logging.opentracing import log_kv, set_tag, trace
 from synapse.replication.tcp.streams import ToDeviceStream
 from synapse.storage._base import SQLBaseStore, db_to_json
@@ -203,9 +203,7 @@ def delete_messages_for_device_txn(txn):
             "delete_messages_for_device", delete_messages_for_device_txn
         )
 
-        log_kv(
-            {"message": "deleted {} messages for device".format(count), "count": count}
-        )
+        log_kv({"message": f"deleted {count} messages for device", "count": count})
 
         # Update the cache, ensuring that we only ever increase the value
         last_deleted_stream_id = self._last_device_delete_cache.get(
@@ -405,6 +403,13 @@ def add_messages_txn(txn, now_ms, stream_id):
                 ],
             )
 
+            if remote_messages_by_destination:
+                issue9533_logger.debug(
+                    "Queued outgoing to-device messages with stream_id %i for %s",
+                    stream_id,
+                    list(remote_messages_by_destination.keys()),
+                )
+
         async with self._device_inbox_id_gen.get_next() as stream_id:
             now_ms = self.clock.time_msec()
             await self.db_pool.runInteraction(
@@ -534,6 +539,16 @@ def _add_messages_to_local_device_inbox_txn(
             ],
         )
 
+        issue9533_logger.debug(
+            "Stored to-device messages with stream_id %i for %s",
+            stream_id,
+            [
+                (user_id, device_id)
+                for (user_id, messages_by_device) in local_by_user_then_device.items()
+                for device_id in messages_by_device.keys()
+            ],
+        )
+
 
 class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
     DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop"
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index d327e9aa0b..3816a0ca53 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 # Copyright 2019,2020 The Matrix.org Foundation C.I.C.
@@ -16,7 +15,7 @@
 # limitations under the License.
 import abc
 import logging
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
+from typing import Any, Collection, Dict, Iterable, List, Optional, Set, Tuple
 
 from synapse.api.errors import Codes, StoreError
 from synapse.logging.opentracing import (
@@ -32,7 +31,7 @@
     LoggingTransaction,
     make_tuple_comparison_clause,
 )
-from synapse.types import Collection, JsonDict, get_verify_key_from_cross_signing_key
+from synapse.types import JsonDict, get_verify_key_from_cross_signing_key
 from synapse.util import json_decoder, json_encoder
 from synapse.util.caches.descriptors import cached, cachedList
 from synapse.util.caches.lrucache import LruCache
@@ -666,7 +665,7 @@ async def get_device_list_last_stream_id_for_remote(
         cached_method_name="get_device_list_last_stream_id_for_remote",
         list_name="user_ids",
     )
-    async def get_device_list_last_stream_id_for_remotes(self, user_ids: str):
+    async def get_device_list_last_stream_id_for_remotes(self, user_ids: Iterable[str]):
         rows = await self.db_pool.simple_select_many_batch(
             table="device_lists_remote_extremeties",
             column="user_id",
@@ -718,7 +717,15 @@ async def mark_remote_user_device_cache_as_stale(self, user_id: str) -> None:
             keyvalues={"user_id": user_id},
             values={},
             insertion_values={"added_ts": self._clock.time_msec()},
-            desc="make_remote_user_device_cache_as_stale",
+            desc="mark_remote_user_device_cache_as_stale",
+        )
+
+    async def mark_remote_user_device_cache_as_valid(self, user_id: str) -> None:
+        # Remove the database entry that says we need to resync devices, after a resync
+        await self.db_pool.simple_delete(
+            table="device_lists_remote_resync",
+            keyvalues={"user_id": user_id},
+            desc="mark_remote_user_device_cache_as_valid",
         )
 
     async def mark_remote_user_device_list_as_unsubscribed(self, user_id: str) -> None:
@@ -985,7 +992,7 @@ async def _remove_duplicate_outbound_pokes(self, progress, batch_size):
 
         def _txn(txn):
             clause, args = make_tuple_comparison_clause(
-                self.db_pool.engine, [(x, last_row[x]) for x in KEY_COLS]
+                [(x, last_row[x]) for x in KEY_COLS]
             )
             sql = """
                 SELECT stream_id, destination, user_id, device_id, MAX(ts) AS ts
@@ -1046,7 +1053,7 @@ def __init__(self, database: DatabasePool, db_conn, hs):
         # Map of (user_id, device_id) -> bool. If there is an entry that implies
         # the device exists.
         self.device_id_exists_cache = LruCache(
-            cache_name="device_id_exists", keylen=2, max_size=10000
+            cache_name="device_id_exists", max_size=10000
         )
 
     async def store_device(
@@ -1071,16 +1078,18 @@ async def store_device(
             return False
 
         try:
-            inserted = await self.db_pool.simple_insert(
+            inserted = await self.db_pool.simple_upsert(
                 "devices",
-                values={
+                keyvalues={
                     "user_id": user_id,
                     "device_id": device_id,
+                },
+                values={},
+                insertion_values={
                     "display_name": initial_device_display_name,
                     "hidden": False,
                 },
                 desc="store_device",
-                or_ignore=True,
             )
             if not inserted:
                 # if the device already exists, check if it's a real device, or
@@ -1092,6 +1101,7 @@ async def store_device(
                 )
                 if hidden:
                     raise StoreError(400, "The device ID is in use", Codes.FORBIDDEN)
+
             self.device_id_exists_cache.set(key, True)
             return inserted
         except StoreError:
@@ -1275,7 +1285,7 @@ def _update_remote_device_list_cache_txn(
         )
 
         txn.call_after(self.get_cached_devices_for_user.invalidate, (user_id,))
-        txn.call_after(self._get_cached_user_device.invalidate_many, (user_id,))
+        txn.call_after(self._get_cached_user_device.invalidate, (user_id,))
         txn.call_after(
             self.get_device_list_last_stream_id_for_remote.invalidate, (user_id,)
         )
@@ -1290,15 +1300,6 @@ def _update_remote_device_list_cache_txn(
             lock=False,
         )
 
-        # If we're replacing the remote user's device list cache presumably
-        # we've done a full resync, so we remove the entry that says we need
-        # to resync
-        self.db_pool.simple_delete_txn(
-            txn,
-            table="device_lists_remote_resync",
-            keyvalues={"user_id": user_id},
-        )
-
     async def add_device_change_to_streams(
         self, user_id: str, device_ids: Collection[str], hosts: List[str]
     ):
diff --git a/synapse/storage/databases/main/directory.py b/synapse/storage/databases/main/directory.py
index 267b948397..86075bc55b 100644
--- a/synapse/storage/databases/main/directory.py
+++ b/synapse/storage/databases/main/directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py
index 12cecceec2..b15fb71e62 100644
--- a/synapse/storage/databases/main/e2e_room_keys.py
+++ b/synapse/storage/databases/main/e2e_room_keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py
index f1e7859d26..1f0a39eac4 100644
--- a/synapse/storage/databases/main/end_to_end_keys.py
+++ b/synapse/storage/databases/main/end_to_end_keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 # Copyright 2019,2020 The Matrix.org Foundation C.I.C.
@@ -22,6 +21,7 @@
 
 from twisted.enterprise.adbapi import Connection
 
+from synapse.api.constants import DeviceKeyAlgorithms
 from synapse.logging.opentracing import log_kv, set_tag, trace
 from synapse.storage._base import SQLBaseStore, db_to_json
 from synapse.storage.database import DatabasePool, make_in_list_sql_clause
@@ -63,6 +63,13 @@ def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer"
 
 
 class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore):
+    def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer"):
+        super().__init__(database, db_conn, hs)
+
+        self._allow_device_name_lookup_over_federation = (
+            self.hs.config.federation.allow_device_name_lookup_over_federation
+        )
+
     async def get_e2e_device_keys_for_federation_query(
         self, user_id: str
     ) -> Tuple[int, List[JsonDict]]:
@@ -85,7 +92,9 @@ async def get_e2e_device_keys_for_federation_query(
                 if keys:
                     result["keys"] = keys
 
-                device_display_name = device.display_name
+                device_display_name = None
+                if self._allow_device_name_lookup_over_federation:
+                    device_display_name = device.display_name
                 if device_display_name:
                     result["device_display_name"] = device_display_name
 
@@ -239,7 +248,7 @@ def _get_e2e_device_keys_txn(
 
         txn.execute(sql, query_params)
 
-        result = {}  # type: Dict[str, Dict[str, Optional[DeviceKeyLookupResult]]]
+        result: Dict[str, Dict[str, Optional[DeviceKeyLookupResult]]] = {}
         for (user_id, device_id, display_name, key_json) in txn:
             if include_deleted_devices:
                 deleted_devices.remove((user_id, device_id))
@@ -373,9 +382,15 @@ def _count_e2e_one_time_keys(txn):
                 " GROUP BY algorithm"
             )
             txn.execute(sql, (user_id, device_id))
-            result = {}
+
+            # Initially set the key count to 0. This ensures that the client will always
+            # receive *some count*, even if it's 0.
+            result = {DeviceKeyAlgorithms.SIGNED_CURVE25519: 0}
+
+            # Override entries with the count of any keys we pulled from the database
             for algorithm, key_count in txn:
                 result[algorithm] = key_count
+
             return result
 
         return await self.db_pool.runInteraction(
@@ -472,7 +487,7 @@ def _get_bare_e2e_cross_signing_keys(self, user_id):
         num_args=1,
     )
     async def _get_bare_e2e_cross_signing_keys_bulk(
-        self, user_ids: List[str]
+        self, user_ids: Iterable[str]
     ) -> Dict[str, Dict[str, dict]]:
         """Returns the cross-signing keys for a set of users.  The output of this
         function should be passed to _get_e2e_cross_signing_signatures_txn if
@@ -496,7 +511,7 @@ async def _get_bare_e2e_cross_signing_keys_bulk(
     def _get_bare_e2e_cross_signing_keys_bulk_txn(
         self,
         txn: Connection,
-        user_ids: List[str],
+        user_ids: Iterable[str],
     ) -> Dict[str, Dict[str, dict]]:
         """Returns the cross-signing keys for a set of users.  The output of this
         function should be passed to _get_e2e_cross_signing_signatures_txn if
@@ -740,81 +755,145 @@ async def claim_e2e_one_time_keys(
         """
 
         @trace
-        def _claim_e2e_one_time_keys(txn):
-            sql = (
-                "SELECT key_id, key_json FROM e2e_one_time_keys_json"
-                " WHERE user_id = ? AND device_id = ? AND algorithm = ?"
-                " LIMIT 1"
+        def _claim_e2e_one_time_key_simple(
+            txn, user_id: str, device_id: str, algorithm: str
+        ) -> Optional[Tuple[str, str]]:
+            """Claim OTK for device for DBs that don't support RETURNING.
+
+            Returns:
+                A tuple of key name (algorithm + key ID) and key JSON, if an
+                OTK was found.
+            """
+
+            sql = """
+                SELECT key_id, key_json FROM e2e_one_time_keys_json
+                WHERE user_id = ? AND device_id = ? AND algorithm = ?
+                LIMIT 1
+            """
+
+            txn.execute(sql, (user_id, device_id, algorithm))
+            otk_row = txn.fetchone()
+            if otk_row is None:
+                return None
+
+            key_id, key_json = otk_row
+
+            self.db_pool.simple_delete_one_txn(
+                txn,
+                table="e2e_one_time_keys_json",
+                keyvalues={
+                    "user_id": user_id,
+                    "device_id": device_id,
+                    "algorithm": algorithm,
+                    "key_id": key_id,
+                },
             )
-            fallback_sql = (
-                "SELECT key_id, key_json, used FROM e2e_fallback_keys_json"
-                " WHERE user_id = ? AND device_id = ? AND algorithm = ?"
-                " LIMIT 1"
+            self._invalidate_cache_and_stream(
+                txn, self.count_e2e_one_time_keys, (user_id, device_id)
             )
-            result = {}
-            delete = []
-            used_fallbacks = []
-            for user_id, device_id, algorithm in query_list:
-                user_result = result.setdefault(user_id, {})
-                device_result = user_result.setdefault(device_id, {})
-                txn.execute(sql, (user_id, device_id, algorithm))
-                otk_row = txn.fetchone()
-                if otk_row is not None:
-                    key_id, key_json = otk_row
-                    device_result[algorithm + ":" + key_id] = key_json
-                    delete.append((user_id, device_id, algorithm, key_id))
-                else:
-                    # no one-time key available, so see if there's a fallback
-                    # key
-                    txn.execute(fallback_sql, (user_id, device_id, algorithm))
-                    fallback_row = txn.fetchone()
-                    if fallback_row is not None:
-                        key_id, key_json, used = fallback_row
-                        device_result[algorithm + ":" + key_id] = key_json
-                        if not used:
-                            used_fallbacks.append(
-                                (user_id, device_id, algorithm, key_id)
-                            )
-
-            # drop any one-time keys that were claimed
-            sql = (
-                "DELETE FROM e2e_one_time_keys_json"
-                " WHERE user_id = ? AND device_id = ? AND algorithm = ?"
-                " AND key_id = ?"
+
+            return f"{algorithm}:{key_id}", key_json
+
+        @trace
+        def _claim_e2e_one_time_key_returning(
+            txn, user_id: str, device_id: str, algorithm: str
+        ) -> Optional[Tuple[str, str]]:
+            """Claim OTK for device for DBs that support RETURNING.
+
+            Returns:
+                A tuple of key name (algorithm + key ID) and key JSON, if an
+                OTK was found.
+            """
+
+            # We can use RETURNING to do the fetch and DELETE in once step.
+            sql = """
+                DELETE FROM e2e_one_time_keys_json
+                WHERE user_id = ? AND device_id = ? AND algorithm = ?
+                    AND key_id IN (
+                        SELECT key_id FROM e2e_one_time_keys_json
+                        WHERE user_id = ? AND device_id = ? AND algorithm = ?
+                        LIMIT 1
+                    )
+                RETURNING key_id, key_json
+            """
+
+            txn.execute(
+                sql, (user_id, device_id, algorithm, user_id, device_id, algorithm)
             )
-            for user_id, device_id, algorithm, key_id in delete:
-                log_kv(
-                    {
-                        "message": "Executing claim e2e_one_time_keys transaction on database."
-                    }
-                )
-                txn.execute(sql, (user_id, device_id, algorithm, key_id))
-                log_kv({"message": "finished executing and invalidating cache"})
-                self._invalidate_cache_and_stream(
-                    txn, self.count_e2e_one_time_keys, (user_id, device_id)
+            otk_row = txn.fetchone()
+            if otk_row is None:
+                return None
+
+            key_id, key_json = otk_row
+            return f"{algorithm}:{key_id}", key_json
+
+        results = {}
+        for user_id, device_id, algorithm in query_list:
+            if self.database_engine.supports_returning:
+                # If we support RETURNING clause we can use a single query that
+                # allows us to use autocommit mode.
+                _claim_e2e_one_time_key = _claim_e2e_one_time_key_returning
+                db_autocommit = True
+            else:
+                _claim_e2e_one_time_key = _claim_e2e_one_time_key_simple
+                db_autocommit = False
+
+            row = await self.db_pool.runInteraction(
+                "claim_e2e_one_time_keys",
+                _claim_e2e_one_time_key,
+                user_id,
+                device_id,
+                algorithm,
+                db_autocommit=db_autocommit,
+            )
+            if row:
+                device_results = results.setdefault(user_id, {}).setdefault(
+                    device_id, {}
                 )
-            # mark fallback keys as used
-            for user_id, device_id, algorithm, key_id in used_fallbacks:
-                self.db_pool.simple_update_txn(
-                    txn,
-                    "e2e_fallback_keys_json",
-                    {
+                device_results[row[0]] = row[1]
+                continue
+
+            # No one-time key available, so see if there's a fallback
+            # key
+            row = await self.db_pool.simple_select_one(
+                table="e2e_fallback_keys_json",
+                keyvalues={
+                    "user_id": user_id,
+                    "device_id": device_id,
+                    "algorithm": algorithm,
+                },
+                retcols=("key_id", "key_json", "used"),
+                desc="_get_fallback_key",
+                allow_none=True,
+            )
+            if row is None:
+                continue
+
+            key_id = row["key_id"]
+            key_json = row["key_json"]
+            used = row["used"]
+
+            # Mark fallback key as used if not already.
+            if not used:
+                await self.db_pool.simple_update_one(
+                    table="e2e_fallback_keys_json",
+                    keyvalues={
                         "user_id": user_id,
                         "device_id": device_id,
                         "algorithm": algorithm,
                         "key_id": key_id,
                     },
-                    {"used": True},
+                    updatevalues={"used": True},
+                    desc="_get_fallback_key_set_used",
                 )
-                self._invalidate_cache_and_stream(
-                    txn, self.get_e2e_unused_fallback_key_types, (user_id, device_id)
+                await self.invalidate_cache_and_stream(
+                    "get_e2e_unused_fallback_key_types", (user_id, device_id)
                 )
 
-            return result
+            device_results = results.setdefault(user_id, {}).setdefault(device_id, {})
+            device_results[f"{algorithm}:{key_id}"] = key_json
 
-        return await self.db_pool.runInteraction(
-            "claim_e2e_one_time_keys", _claim_e2e_one_time_keys
-        )
+        return results
 
 
 class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore):
diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py
index a956be491a..bddf5ef192 100644
--- a/synapse/storage/databases/main/event_federation.py
+++ b/synapse/storage/databases/main/event_federation.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,22 +14,42 @@
 import itertools
 import logging
 from queue import Empty, PriorityQueue
-from typing import Dict, Iterable, List, Set, Tuple
+from typing import Collection, Dict, Iterable, List, Optional, Set, Tuple
 
+from prometheus_client import Counter, Gauge
+
+from synapse.api.constants import MAX_DEPTH
 from synapse.api.errors import StoreError
-from synapse.events import EventBase
+from synapse.api.room_versions import EventFormatVersions, RoomVersion
+from synapse.events import EventBase, make_event_from_dict
 from synapse.metrics.background_process_metrics import wrap_as_background_process
-from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause
+from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
 from synapse.storage.database import DatabasePool, LoggingTransaction
 from synapse.storage.databases.main.events_worker import EventsWorkerStore
 from synapse.storage.databases.main.signatures import SignatureWorkerStore
 from synapse.storage.engines import PostgresEngine
 from synapse.storage.types import Cursor
-from synapse.types import Collection
+from synapse.util import json_encoder
 from synapse.util.caches.descriptors import cached
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.iterutils import batch_iter
 
+oldest_pdu_in_federation_staging = Gauge(
+    "synapse_federation_server_oldest_inbound_pdu_in_staging",
+    "The age in seconds since we received the oldest pdu in the federation staging area",
+)
+
+number_pdus_in_federation_queue = Gauge(
+    "synapse_federation_server_number_inbound_pdu_in_staging",
+    "The total number of events in the inbound federation staging",
+)
+
+pdus_pruned_from_federation_queue = Counter(
+    "synapse_federation_server_number_inbound_pdu_pruned",
+    "The number of events in the inbound federation staging that have been "
+    "pruned due to the queue getting too long",
+)
+
 logger = logging.getLogger(__name__)
 
 
@@ -49,9 +68,11 @@ def __init__(self, database: DatabasePool, db_conn, hs):
             )
 
         # Cache of event ID to list of auth event IDs and their depths.
-        self._event_auth_cache = LruCache(
+        self._event_auth_cache: LruCache[str, List[Tuple[str, int]]] = LruCache(
             500000, "_event_auth_cache", size_callback=len
-        )  # type: LruCache[str, List[Tuple[str, int]]]
+        )
+
+        self._clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000)
 
     async def get_auth_chain(
         self, room_id: str, event_ids: Collection[str], include_given: bool = False
@@ -122,10 +143,10 @@ def _get_auth_chain_ids_using_cover_index_txn(
         initial_events = set(event_ids)
 
         # All the events that we've found that are reachable from the events.
-        seen_events = set()  # type: Set[str]
+        seen_events: Set[str] = set()
 
         # A map from chain ID to max sequence number of the given events.
-        event_chains = {}  # type: Dict[int, int]
+        event_chains: Dict[int, int] = {}
 
         sql = """
             SELECT event_id, chain_id, sequence_number
@@ -167,7 +188,7 @@ def _get_auth_chain_ids_using_cover_index_txn(
         """
 
         # A map from chain ID to max sequence number *reachable* from any event ID.
-        chains = {}  # type: Dict[int, int]
+        chains: Dict[int, int] = {}
 
         # Add all linked chains reachable from initial set of chains.
         for batch in batch_iter(event_chains, 1000):
@@ -338,14 +359,14 @@ def _get_auth_chain_difference_using_cover_index_txn(
         initial_events = set(state_sets[0]).union(*state_sets[1:])
 
         # Map from event_id -> (chain ID, seq no)
-        chain_info = {}  # type: Dict[str, Tuple[int, int]]
+        chain_info: Dict[str, Tuple[int, int]] = {}
 
         # Map from chain ID -> seq no -> event Id
-        chain_to_event = {}  # type: Dict[int, Dict[int, str]]
+        chain_to_event: Dict[int, Dict[int, str]] = {}
 
         # All the chains that we've found that are reachable from the state
         # sets.
-        seen_chains = set()  # type: Set[int]
+        seen_chains: Set[int] = set()
 
         sql = """
             SELECT event_id, chain_id, sequence_number
@@ -377,9 +398,9 @@ def _get_auth_chain_difference_using_cover_index_txn(
 
         # Corresponds to `state_sets`, except as a map from chain ID to max
         # sequence number reachable from the state set.
-        set_to_chain = []  # type: List[Dict[int, int]]
+        set_to_chain: List[Dict[int, int]] = []
         for state_set in state_sets:
-            chains = {}  # type: Dict[int, int]
+            chains: Dict[int, int] = {}
             set_to_chain.append(chains)
 
             for event_id in state_set:
@@ -431,7 +452,7 @@ def _get_auth_chain_difference_using_cover_index_txn(
 
         # Mapping from chain ID to the range of sequence numbers that should be
         # pulled from the database.
-        chain_to_gap = {}  # type: Dict[int, Tuple[int, int]]
+        chain_to_gap: Dict[int, Tuple[int, int]] = {}
 
         for chain_id in seen_chains:
             min_seq_no = min(chains.get(chain_id, 0) for chains in set_to_chain)
@@ -540,7 +561,7 @@ def _get_auth_chain_difference_txn(
         }
 
         # The sorted list of events whose auth chains we should walk.
-        search = []  # type: List[Tuple[int, str]]
+        search: List[Tuple[int, str]] = []
 
         # We need to get the depth of the initial events for sorting purposes.
         sql = """
@@ -563,7 +584,7 @@ def _get_auth_chain_difference_txn(
         search.sort()
 
         # Map from event to its auth events
-        event_to_auth_events = {}  # type: Dict[str, Set[str]]
+        event_to_auth_events: Dict[str, Set[str]] = {}
 
         base_sql = """
             SELECT a.event_id, auth_id, depth
@@ -650,30 +671,100 @@ def _get_auth_chain_difference_txn(
         # Return all events where not all sets can reach them.
         return {eid for eid, n in event_to_missing_sets.items() if n}
 
-    async def get_oldest_events_with_depth_in_room(self, room_id):
+    async def get_oldest_event_ids_with_depth_in_room(self, room_id) -> Dict[str, int]:
+        """Gets the oldest events(backwards extremities) in the room along with the
+        aproximate depth.
+
+        We use this function so that we can compare and see if someones current
+        depth at their current scrollback is within pagination range of the
+        event extremeties. If the current depth is close to the depth of given
+        oldest event, we can trigger a backfill.
+
+        Args:
+            room_id: Room where we want to find the oldest events
+
+        Returns:
+            Map from event_id to depth
+        """
+
+        def get_oldest_event_ids_with_depth_in_room_txn(txn, room_id):
+            # Assemble a dictionary with event_id -> depth for the oldest events
+            # we know of in the room. Backwards extremeties are the oldest
+            # events we know of in the room but we only know of them because
+            # some other event referenced them by prev_event and aren't peristed
+            # in our database yet (meaning we don't know their depth
+            # specifically). So we need to look for the aproximate depth from
+            # the events connected to the current backwards extremeties.
+            sql = """
+                SELECT b.event_id, MAX(e.depth) FROM events as e
+                /**
+                 * Get the edge connections from the event_edges table
+                 * so we can see whether this event's prev_events points
+                 * to a backward extremity in the next join.
+                 */
+                INNER JOIN event_edges as g
+                ON g.event_id = e.event_id
+                /**
+                 * We find the "oldest" events in the room by looking for
+                 * events connected to backwards extremeties (oldest events
+                 * in the room that we know of so far).
+                 */
+                INNER JOIN event_backward_extremities as b
+                ON g.prev_event_id = b.event_id
+                WHERE b.room_id = ? AND g.is_state is ?
+                GROUP BY b.event_id
+            """
+
+            txn.execute(sql, (room_id, False))
+
+            return dict(txn)
+
         return await self.db_pool.runInteraction(
-            "get_oldest_events_with_depth_in_room",
-            self.get_oldest_events_with_depth_in_room_txn,
+            "get_oldest_event_ids_with_depth_in_room",
+            get_oldest_event_ids_with_depth_in_room_txn,
             room_id,
         )
 
-    def get_oldest_events_with_depth_in_room_txn(self, txn, room_id):
-        sql = (
-            "SELECT b.event_id, MAX(e.depth) FROM events as e"
-            " INNER JOIN event_edges as g"
-            " ON g.event_id = e.event_id"
-            " INNER JOIN event_backward_extremities as b"
-            " ON g.prev_event_id = b.event_id"
-            " WHERE b.room_id = ? AND g.is_state is ?"
-            " GROUP BY b.event_id"
-        )
+    async def get_insertion_event_backwards_extremities_in_room(
+        self, room_id
+    ) -> Dict[str, int]:
+        """Get the insertion events we know about that we haven't backfilled yet.
+
+        We use this function so that we can compare and see if someones current
+        depth at their current scrollback is within pagination range of the
+        insertion event. If the current depth is close to the depth of given
+        insertion event, we can trigger a backfill.
 
-        txn.execute(sql, (room_id, False))
+        Args:
+            room_id: Room where we want to find the oldest events
+
+        Returns:
+            Map from event_id to depth
+        """
+
+        def get_insertion_event_backwards_extremities_in_room_txn(txn, room_id):
+            sql = """
+                SELECT b.event_id, MAX(e.depth) FROM insertion_events as i
+                /* We only want insertion events that are also marked as backwards extremities */
+                INNER JOIN insertion_event_extremities as b USING (event_id)
+                /* Get the depth of the insertion event from the events table */
+                INNER JOIN events AS e USING (event_id)
+                WHERE b.room_id = ?
+                GROUP BY b.event_id
+            """
+
+            txn.execute(sql, (room_id,))
 
-        return dict(txn)
+            return dict(txn)
+
+        return await self.db_pool.runInteraction(
+            "get_insertion_event_backwards_extremities_in_room",
+            get_insertion_event_backwards_extremities_in_room_txn,
+            room_id,
+        )
 
-    async def get_max_depth_of(self, event_ids: List[str]) -> int:
-        """Returns the max depth of a set of event IDs
+    async def get_max_depth_of(self, event_ids: List[str]) -> Tuple[str, int]:
+        """Returns the event ID and depth for the event that has the max depth from a set of event IDs
 
         Args:
             event_ids: The event IDs to calculate the max depth of.
@@ -682,14 +773,53 @@ async def get_max_depth_of(self, event_ids: List[str]) -> int:
             table="events",
             column="event_id",
             iterable=event_ids,
-            retcols=("depth",),
+            retcols=(
+                "event_id",
+                "depth",
+            ),
             desc="get_max_depth_of",
         )
 
         if not rows:
-            return 0
+            return None, 0
         else:
-            return max(row["depth"] for row in rows)
+            max_depth_event_id = ""
+            current_max_depth = 0
+            for row in rows:
+                if row["depth"] > current_max_depth:
+                    max_depth_event_id = row["event_id"]
+                    current_max_depth = row["depth"]
+
+            return max_depth_event_id, current_max_depth
+
+    async def get_min_depth_of(self, event_ids: List[str]) -> Tuple[str, int]:
+        """Returns the event ID and depth for the event that has the min depth from a set of event IDs
+
+        Args:
+            event_ids: The event IDs to calculate the max depth of.
+        """
+        rows = await self.db_pool.simple_select_many_batch(
+            table="events",
+            column="event_id",
+            iterable=event_ids,
+            retcols=(
+                "event_id",
+                "depth",
+            ),
+            desc="get_min_depth_of",
+        )
+
+        if not rows:
+            return None, 0
+        else:
+            min_depth_event_id = ""
+            current_min_depth = MAX_DEPTH
+            for row in rows:
+                if row["depth"] < current_min_depth:
+                    min_depth_event_id = row["event_id"]
+                    current_min_depth = row["depth"]
+
+            return min_depth_event_id, current_min_depth
 
     async def get_prev_events_for_room(self, room_id: str) -> List[str]:
         """
@@ -882,15 +1012,46 @@ def _get_backfill_events(self, txn, room_id, event_list, limit):
         # We want to make sure that we do a breadth-first, "depth" ordered
         # search.
 
-        query = (
-            "SELECT depth, prev_event_id FROM event_edges"
-            " INNER JOIN events"
-            " ON prev_event_id = events.event_id"
-            " WHERE event_edges.event_id = ?"
-            " AND event_edges.is_state = ?"
-            " LIMIT ?"
-        )
+        # Look for the prev_event_id connected to the given event_id
+        query = """
+            SELECT depth, prev_event_id FROM event_edges
+            /* Get the depth of the prev_event_id from the events table */
+            INNER JOIN events
+            ON prev_event_id = events.event_id
+            /* Find an event which matches the given event_id */
+            WHERE event_edges.event_id = ?
+            AND event_edges.is_state = ?
+            LIMIT ?
+        """
 
+        # Look for the "insertion" events connected to the given event_id
+        connected_insertion_event_query = """
+            SELECT e.depth, i.event_id FROM insertion_event_edges AS i
+            /* Get the depth of the insertion event from the events table */
+            INNER JOIN events AS e USING (event_id)
+            /* Find an insertion event which points via prev_events to the given event_id */
+            WHERE i.insertion_prev_event_id = ?
+            LIMIT ?
+        """
+
+        # Find any chunk connections of a given insertion event
+        chunk_connection_query = """
+            SELECT e.depth, c.event_id FROM insertion_events AS i
+            /* Find the chunk that connects to the given insertion event */
+            INNER JOIN chunk_events AS c
+            ON i.next_chunk_id = c.chunk_id
+            /* Get the depth of the chunk start event from the events table */
+            INNER JOIN events AS e USING (event_id)
+            /* Find an insertion event which matches the given event_id */
+            WHERE i.event_id = ?
+            LIMIT ?
+        """
+
+        # In a PriorityQueue, the lowest valued entries are retrieved first.
+        # We're using depth as the priority in the queue.
+        # Depth is lowest at the oldest-in-time message and highest and
+        # newest-in-time message. We add events to the queue with a negative depth so that
+        # we process the newest-in-time messages first going backwards in time.
         queue = PriorityQueue()
 
         for event_id in event_list:
@@ -916,9 +1077,47 @@ def _get_backfill_events(self, txn, room_id, event_list, limit):
 
             event_results.add(event_id)
 
+            # Try and find any potential historical chunks of message history.
+            #
+            # First we look for an insertion event connected to the current
+            # event (by prev_event). If we find any, we need to go and try to
+            # find any chunk events connected to the insertion event (by
+            # chunk_id). If we find any, we'll add them to the queue and
+            # navigate up the DAG like normal in the next iteration of the loop.
+            txn.execute(
+                connected_insertion_event_query, (event_id, limit - len(event_results))
+            )
+            connected_insertion_event_id_results = txn.fetchall()
+            logger.debug(
+                "_get_backfill_events: connected_insertion_event_query %s",
+                connected_insertion_event_id_results,
+            )
+            for row in connected_insertion_event_id_results:
+                connected_insertion_event_depth = row[0]
+                connected_insertion_event = row[1]
+                queue.put((-connected_insertion_event_depth, connected_insertion_event))
+
+                # Find any chunk connections for the given insertion event
+                txn.execute(
+                    chunk_connection_query,
+                    (connected_insertion_event, limit - len(event_results)),
+                )
+                chunk_start_event_id_results = txn.fetchall()
+                logger.debug(
+                    "_get_backfill_events: chunk_start_event_id_results %s",
+                    chunk_start_event_id_results,
+                )
+                for row in chunk_start_event_id_results:
+                    if row[1] not in event_results:
+                        queue.put((-row[0], row[1]))
+
             txn.execute(query, (event_id, False, limit - len(event_results)))
+            prev_event_id_results = txn.fetchall()
+            logger.debug(
+                "_get_backfill_events: prev_event_ids %s", prev_event_id_results
+            )
 
-            for row in txn:
+            for row in prev_event_id_results:
                 if row[1] not in event_results:
                     queue.put((-row[0], row[1]))
 
@@ -1006,6 +1205,299 @@ def _delete_old_forward_extrem_cache_txn(txn):
             _delete_old_forward_extrem_cache_txn,
         )
 
+    async def insert_insertion_extremity(self, event_id: str, room_id: str) -> None:
+        await self.db_pool.simple_upsert(
+            table="insertion_event_extremities",
+            keyvalues={"event_id": event_id},
+            values={
+                "event_id": event_id,
+                "room_id": room_id,
+            },
+            insertion_values={},
+            desc="insert_insertion_extremity",
+            lock=False,
+        )
+
+    async def insert_received_event_to_staging(
+        self, origin: str, event: EventBase
+    ) -> None:
+        """Insert a newly received event from federation into the staging area."""
+
+        # We use an upsert here to handle the case where we see the same event
+        # from the same server multiple times.
+        await self.db_pool.simple_upsert(
+            table="federation_inbound_events_staging",
+            keyvalues={
+                "origin": origin,
+                "event_id": event.event_id,
+            },
+            values={},
+            insertion_values={
+                "room_id": event.room_id,
+                "received_ts": self._clock.time_msec(),
+                "event_json": json_encoder.encode(event.get_dict()),
+                "internal_metadata": json_encoder.encode(
+                    event.internal_metadata.get_dict()
+                ),
+            },
+            desc="insert_received_event_to_staging",
+        )
+
+    async def remove_received_event_from_staging(
+        self,
+        origin: str,
+        event_id: str,
+    ) -> Optional[int]:
+        """Remove the given event from the staging area.
+
+        Returns:
+            The received_ts of the row that was deleted, if any.
+        """
+        if self.db_pool.engine.supports_returning:
+
+            def _remove_received_event_from_staging_txn(txn):
+                sql = """
+                    DELETE FROM federation_inbound_events_staging
+                    WHERE origin = ? AND event_id = ?
+                    RETURNING received_ts
+                """
+
+                txn.execute(sql, (origin, event_id))
+                return txn.fetchone()
+
+            row = await self.db_pool.runInteraction(
+                "remove_received_event_from_staging",
+                _remove_received_event_from_staging_txn,
+                db_autocommit=True,
+            )
+            if row is None:
+                return None
+
+            return row[0]
+
+        else:
+
+            def _remove_received_event_from_staging_txn(txn):
+                received_ts = self.db_pool.simple_select_one_onecol_txn(
+                    txn,
+                    table="federation_inbound_events_staging",
+                    keyvalues={
+                        "origin": origin,
+                        "event_id": event_id,
+                    },
+                    retcol="received_ts",
+                    allow_none=True,
+                )
+                self.db_pool.simple_delete_txn(
+                    txn,
+                    table="federation_inbound_events_staging",
+                    keyvalues={
+                        "origin": origin,
+                        "event_id": event_id,
+                    },
+                )
+
+                return received_ts
+
+            return await self.db_pool.runInteraction(
+                "remove_received_event_from_staging",
+                _remove_received_event_from_staging_txn,
+            )
+
+    async def get_next_staged_event_id_for_room(
+        self,
+        room_id: str,
+    ) -> Optional[Tuple[str, str]]:
+        """Get the next event ID in the staging area for the given room."""
+
+        def _get_next_staged_event_id_for_room_txn(txn):
+            sql = """
+                SELECT origin, event_id
+                FROM federation_inbound_events_staging
+                WHERE room_id = ?
+                ORDER BY received_ts ASC
+                LIMIT 1
+            """
+
+            txn.execute(sql, (room_id,))
+
+            return txn.fetchone()
+
+        return await self.db_pool.runInteraction(
+            "get_next_staged_event_id_for_room", _get_next_staged_event_id_for_room_txn
+        )
+
+    async def get_next_staged_event_for_room(
+        self,
+        room_id: str,
+        room_version: RoomVersion,
+    ) -> Optional[Tuple[str, EventBase]]:
+        """Get the next event in the staging area for the given room."""
+
+        def _get_next_staged_event_for_room_txn(txn):
+            sql = """
+                SELECT event_json, internal_metadata, origin
+                FROM federation_inbound_events_staging
+                WHERE room_id = ?
+                ORDER BY received_ts ASC
+                LIMIT 1
+            """
+            txn.execute(sql, (room_id,))
+
+            return txn.fetchone()
+
+        row = await self.db_pool.runInteraction(
+            "get_next_staged_event_for_room", _get_next_staged_event_for_room_txn
+        )
+
+        if not row:
+            return None
+
+        event_d = db_to_json(row[0])
+        internal_metadata_d = db_to_json(row[1])
+        origin = row[2]
+
+        event = make_event_from_dict(
+            event_dict=event_d,
+            room_version=room_version,
+            internal_metadata_dict=internal_metadata_d,
+        )
+
+        return origin, event
+
+    async def prune_staged_events_in_room(
+        self,
+        room_id: str,
+        room_version: RoomVersion,
+    ) -> bool:
+        """Checks if there are lots of staged events for the room, and if so
+        prune them down.
+
+        Returns:
+            Whether any events were pruned
+        """
+
+        # First check the size of the queue.
+        count = await self.db_pool.simple_select_one_onecol(
+            table="federation_inbound_events_staging",
+            keyvalues={"room_id": room_id},
+            retcol="COALESCE(COUNT(*), 0)",
+            desc="prune_staged_events_in_room_count",
+        )
+
+        if count < 100:
+            return False
+
+        # If the queue is too large, then we want clear the entire queue,
+        # keeping only the forward extremities (i.e. the events not referenced
+        # by other events in the queue). We do this so that we can always
+        # backpaginate in all the events we have dropped.
+        rows = await self.db_pool.simple_select_list(
+            table="federation_inbound_events_staging",
+            keyvalues={"room_id": room_id},
+            retcols=("event_id", "event_json"),
+            desc="prune_staged_events_in_room_fetch",
+        )
+
+        # Find the set of events referenced by those in the queue, as well as
+        # collecting all the event IDs in the queue.
+        referenced_events: Set[str] = set()
+        seen_events: Set[str] = set()
+        for row in rows:
+            event_id = row["event_id"]
+            seen_events.add(event_id)
+            event_d = db_to_json(row["event_json"])
+
+            # We don't bother parsing the dicts into full blown event objects,
+            # as that is needlessly expensive.
+
+            # We haven't checked that the `prev_events` have the right format
+            # yet, so we check as we go.
+            prev_events = event_d.get("prev_events", [])
+            if not isinstance(prev_events, list):
+                logger.info("Invalid prev_events for %s", event_id)
+                continue
+
+            if room_version.event_format == EventFormatVersions.V1:
+                for prev_event_tuple in prev_events:
+                    if not isinstance(prev_event_tuple, list) or len(prev_events) != 2:
+                        logger.info("Invalid prev_events for %s", event_id)
+                        break
+
+                    prev_event_id = prev_event_tuple[0]
+                    if not isinstance(prev_event_id, str):
+                        logger.info("Invalid prev_events for %s", event_id)
+                        break
+
+                    referenced_events.add(prev_event_id)
+            else:
+                for prev_event_id in prev_events:
+                    if not isinstance(prev_event_id, str):
+                        logger.info("Invalid prev_events for %s", event_id)
+                        break
+
+                    referenced_events.add(prev_event_id)
+
+        to_delete = referenced_events & seen_events
+        if not to_delete:
+            return False
+
+        pdus_pruned_from_federation_queue.inc(len(to_delete))
+        logger.info(
+            "Pruning %d events in room %s from federation queue",
+            len(to_delete),
+            room_id,
+        )
+
+        await self.db_pool.simple_delete_many(
+            table="federation_inbound_events_staging",
+            keyvalues={"room_id": room_id},
+            iterable=to_delete,
+            column="event_id",
+            desc="prune_staged_events_in_room_delete",
+        )
+
+        return True
+
+    async def get_all_rooms_with_staged_incoming_events(self) -> List[str]:
+        """Get the room IDs of all events currently staged."""
+        return await self.db_pool.simple_select_onecol(
+            table="federation_inbound_events_staging",
+            keyvalues={},
+            retcol="DISTINCT room_id",
+            desc="get_all_rooms_with_staged_incoming_events",
+        )
+
+    @wrap_as_background_process("_get_stats_for_federation_staging")
+    async def _get_stats_for_federation_staging(self):
+        """Update the prometheus metrics for the inbound federation staging area."""
+
+        def _get_stats_for_federation_staging_txn(txn):
+            txn.execute(
+                "SELECT coalesce(count(*), 0) FROM federation_inbound_events_staging"
+            )
+            (count,) = txn.fetchone()
+
+            txn.execute(
+                "SELECT min(received_ts) FROM federation_inbound_events_staging"
+            )
+
+            (received_ts,) = txn.fetchone()
+
+            # If there is nothing in the staging area default it to 0.
+            age = 0
+            if received_ts is not None:
+                age = self._clock.time_msec() - received_ts
+
+            return count, age
+
+        count, age = await self.db_pool.runInteraction(
+            "_get_stats_for_federation_staging", _get_stats_for_federation_staging_txn
+        )
+
+        number_pdus_in_federation_queue.set(count)
+        oldest_pdu_in_federation_staging.set(age)
+
 
 class EventFederationStore(EventFederationWorkerStore):
     """Responsible for storing and serving up the various graphs associated
diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py
index 78245ad5bd..55caa6bbe7 100644
--- a/synapse/storage/databases/main/event_push_actions.py
+++ b/synapse/storage/databases/main/event_push_actions.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -760,7 +759,7 @@ def _rotate_notifs_before_txn(self, txn, rotate_to_stream_ordering):
         # object because we might not have the same amount of rows in each of them. To do
         # this, we use a dict indexed on the user ID and room ID to make it easier to
         # populate.
-        summaries = {}  # type: Dict[Tuple[str, str], _EventPushSummary]
+        summaries: Dict[Tuple[str, str], _EventPushSummary] = {}
         for row in txn:
             summaries[(row[0], row[1])] = _EventPushSummary(
                 unread_count=row[2],
@@ -861,7 +860,7 @@ def _remove_old_push_actions_before_txn(
                                   not be deleted.
         """
         txn.call_after(
-            self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
+            self.get_unread_event_push_actions_by_room_for_user.invalidate,
             (room_id, user_id),
         )
 
diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py
index 98dac19a95..40b53274fb 100644
--- a/synapse/storage/databases/main/events.py
+++ b/synapse/storage/databases/main/events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018-2019 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -110,10 +109,8 @@ def __init__(
 
         # Ideally we'd move these ID gens here, unfortunately some other ID
         # generators are chained off them so doing so is a bit of a PITA.
-        self._backfill_id_gen = (
-            self.store._backfill_id_gen
-        )  # type: MultiWriterIdGenerator
-        self._stream_id_gen = self.store._stream_id_gen  # type: MultiWriterIdGenerator
+        self._backfill_id_gen: MultiWriterIdGenerator = self.store._backfill_id_gen
+        self._stream_id_gen: MultiWriterIdGenerator = self.store._stream_id_gen
 
         # This should only exist on instances that are configured to write
         assert (
@@ -171,7 +168,7 @@ async def _persist_events_and_state_updates(
             )
 
         async with stream_ordering_manager as stream_orderings:
-            for (event, context), stream in zip(events_and_contexts, stream_orderings):
+            for (event, _), stream in zip(events_and_contexts, stream_orderings):
                 event.internal_metadata.stream_ordering = stream
 
             await self.db_pool.runInteraction(
@@ -222,7 +219,7 @@ async def _get_events_which_are_prevs(self, event_ids: Iterable[str]) -> List[st
         Returns:
             Filtered event ids
         """
-        results = []  # type: List[str]
+        results: List[str] = []
 
         def _get_events_which_are_prevs_txn(txn, batch):
             sql = """
@@ -298,7 +295,7 @@ def _get_prevs_before_rejected_txn(txn, batch):
                 txn.execute(sql + clause, args)
                 to_recursively_check = []
 
-                for event_id, prev_event_id, metadata, rejected in txn:
+                for _, prev_event_id, metadata, rejected in txn:
                     if prev_event_id in existing_prevs:
                         continue
 
@@ -320,8 +317,8 @@ def _persist_events_txn(
         txn: LoggingTransaction,
         events_and_contexts: List[Tuple[EventBase, EventContext]],
         backfilled: bool,
-        state_delta_for_room: Dict[str, DeltaState] = {},
-        new_forward_extremeties: Dict[str, List[str]] = {},
+        state_delta_for_room: Optional[Dict[str, DeltaState]] = None,
+        new_forward_extremeties: Optional[Dict[str, List[str]]] = None,
     ):
         """Insert some number of room events into the necessary database tables.
 
@@ -342,6 +339,9 @@ def _persist_events_txn(
                 extremities.
 
         """
+        state_delta_for_room = state_delta_for_room or {}
+        new_forward_extremeties = new_forward_extremeties or {}
+
         all_events_and_contexts = events_and_contexts
 
         min_stream_order = events_and_contexts[0][0].internal_metadata.stream_ordering
@@ -506,7 +506,7 @@ def _add_chain_cover_index(
         """
 
         # Map from event ID to chain ID/sequence number.
-        chain_map = {}  # type: Dict[str, Tuple[int, int]]
+        chain_map: Dict[str, Tuple[int, int]] = {}
 
         # Set of event IDs to calculate chain ID/seq numbers for.
         events_to_calc_chain_id_for = set(event_to_room_id)
@@ -815,8 +815,8 @@ def _allocate_chain_ids(
         #      new chain if the sequence number has already been allocated.
         #
 
-        existing_chains = set()  # type: Set[int]
-        tree = []  # type: List[Tuple[str, Optional[str]]]
+        existing_chains: Set[int] = set()
+        tree: List[Tuple[str, Optional[str]]] = []
 
         # We need to do this in a topologically sorted order as we want to
         # generate chain IDs/sequence numbers of an event's auth events before
@@ -846,7 +846,7 @@ def _allocate_chain_ids(
         )
         txn.execute(sql % (clause,), args)
 
-        chain_to_max_seq_no = {row[0]: row[1] for row in txn}  # type: Dict[Any, int]
+        chain_to_max_seq_no: Dict[Any, int] = {row[0]: row[1] for row in txn}
 
         # Allocate the new events chain ID/sequence numbers.
         #
@@ -856,8 +856,8 @@ def _allocate_chain_ids(
         # number of new chain IDs in one call, replacing all temporary
         # objects with real allocated chain IDs.
 
-        unallocated_chain_ids = set()  # type: Set[object]
-        new_chain_tuples = {}  # type: Dict[str, Tuple[Any, int]]
+        unallocated_chain_ids: Set[object] = set()
+        new_chain_tuples: Dict[str, Tuple[Any, int]] = {}
         for event_id, auth_event_id in tree:
             # If we reference an auth_event_id we fetch the allocated chain ID,
             # either from the existing `chain_map` or the newly generated
@@ -868,7 +868,7 @@ def _allocate_chain_ids(
                 if not existing_chain_id:
                     existing_chain_id = chain_map[auth_event_id]
 
-            new_chain_tuple = None  # type: Optional[Tuple[Any, int]]
+            new_chain_tuple: Optional[Tuple[Any, int]] = None
             if existing_chain_id:
                 # We found a chain ID/sequence number candidate, check its
                 # not already taken.
@@ -895,9 +895,9 @@ def _allocate_chain_ids(
         )
 
         # Map from potentially temporary chain ID to real chain ID
-        chain_id_to_allocated_map = dict(
+        chain_id_to_allocated_map: Dict[Any, int] = dict(
             zip(unallocated_chain_ids, newly_allocated_chain_ids)
-        )  # type: Dict[Any, int]
+        )
         chain_id_to_allocated_map.update((c, c) for c in existing_chains)
 
         return {
@@ -1125,7 +1125,7 @@ def _upsert_room_version_txn(self, txn: LoggingTransaction, room_id: str):
     def _update_forward_extremities_txn(
         self, txn, new_forward_extremities, max_stream_order
     ):
-        for room_id, new_extrem in new_forward_extremities.items():
+        for room_id in new_forward_extremities.keys():
             self.db_pool.simple_delete_txn(
                 txn, table="event_forward_extremities", keyvalues={"room_id": room_id}
             )
@@ -1173,9 +1173,9 @@ def _filter_events_and_contexts_for_duplicates(
         Returns:
             list[(EventBase, EventContext)]: filtered list
         """
-        new_events_and_contexts = (
-            OrderedDict()
-        )  # type: OrderedDict[str, Tuple[EventBase, EventContext]]
+        new_events_and_contexts: OrderedDict[
+            str, Tuple[EventBase, EventContext]
+        ] = OrderedDict()
         for event, context in events_and_contexts:
             prev_event_context = new_events_and_contexts.get(event.event_id)
             if prev_event_context:
@@ -1203,7 +1203,7 @@ def _update_room_depths_txn(
                 we are persisting
             backfilled (bool): True if the events were backfilled
         """
-        depth_updates = {}  # type: Dict[str, int]
+        depth_updates: Dict[str, int] = {}
         for event, context in events_and_contexts:
             # Remove the any existing cache entries for the event_ids
             txn.call_after(self.store._invalidate_get_event_cache, event.event_id)
@@ -1376,24 +1376,28 @@ def get_internal_metadata(event):
             ],
         )
 
-        for event, _ in events_and_contexts:
-            if not event.internal_metadata.is_redacted():
-                # If we're persisting an unredacted event we go and ensure
-                # that we mark any redactions that reference this event as
-                # requiring censoring.
-                self.db_pool.simple_update_txn(
-                    txn,
-                    table="redactions",
-                    keyvalues={"redacts": event.event_id},
-                    updatevalues={"have_censored": False},
+        # If we're persisting an unredacted event we go and ensure
+        # that we mark any redactions that reference this event as
+        # requiring censoring.
+        sql = "UPDATE redactions SET have_censored = ? WHERE redacts = ?"
+        txn.execute_batch(
+            sql,
+            (
+                (
+                    False,
+                    event.event_id,
                 )
+                for event, _ in events_and_contexts
+                if not event.internal_metadata.is_redacted()
+            ),
+        )
 
         state_events_and_contexts = [
             ec for ec in events_and_contexts if ec[0].is_state()
         ]
 
         state_values = []
-        for event, context in state_events_and_contexts:
+        for event, _ in state_events_and_contexts:
             vals = {
                 "event_id": event.event_id,
                 "room_id": event.room_id,
@@ -1462,7 +1466,7 @@ def _update_metadata_tables_txn(
             # nothing to do here
             return
 
-        for event, context in events_and_contexts:
+        for event, _ in events_and_contexts:
             if event.type == EventTypes.Redaction and event.redacts is not None:
                 # Remove the entries in the event_push_actions table for the
                 # redacted event.
@@ -1498,6 +1502,9 @@ def _update_metadata_tables_txn(
 
             self._handle_event_relations(txn, event)
 
+            self._handle_insertion_event(txn, event)
+            self._handle_chunk_event(txn, event)
+
             # Store the labels for this event.
             labels = event.content.get(EventContentFields.LABELS)
             if labels:
@@ -1574,11 +1581,11 @@ def _store_redaction(self, txn, event):
         # invalidate the cache for the redacted event
         txn.call_after(self.store._invalidate_get_event_cache, event.redacts)
 
-        self.db_pool.simple_insert_txn(
+        self.db_pool.simple_upsert_txn(
             txn,
             table="redactions",
+            keyvalues={"event_id": event.event_id},
             values={
-                "event_id": event.event_id,
                 "redacts": event.redacts,
                 "received_ts": self._clock.time_msec(),
             },
@@ -1742,14 +1749,114 @@ def _handle_event_relations(self, txn, event):
             },
         )
 
-        txn.call_after(self.store.get_relations_for_event.invalidate_many, (parent_id,))
+        txn.call_after(self.store.get_relations_for_event.invalidate, (parent_id,))
         txn.call_after(
-            self.store.get_aggregation_groups_for_event.invalidate_many, (parent_id,)
+            self.store.get_aggregation_groups_for_event.invalidate, (parent_id,)
         )
 
         if rel_type == RelationTypes.REPLACE:
             txn.call_after(self.store.get_applicable_edit.invalidate, (parent_id,))
 
+    def _handle_insertion_event(self, txn: LoggingTransaction, event: EventBase):
+        """Handles keeping track of insertion events and edges/connections.
+        Part of MSC2716.
+
+        Args:
+            txn: The database transaction object
+            event: The event to process
+        """
+
+        if event.type != EventTypes.MSC2716_INSERTION:
+            # Not a insertion event
+            return
+
+        # Skip processing a insertion event if the room version doesn't
+        # support it.
+        room_version = self.store.get_room_version_txn(txn, event.room_id)
+        if not room_version.msc2716_historical:
+            return
+
+        next_chunk_id = event.content.get(EventContentFields.MSC2716_NEXT_CHUNK_ID)
+        if next_chunk_id is None:
+            # Invalid insertion event without next chunk ID
+            return
+
+        logger.debug(
+            "_handle_insertion_event (next_chunk_id=%s) %s", next_chunk_id, event
+        )
+
+        # Keep track of the insertion event and the chunk ID
+        self.db_pool.simple_insert_txn(
+            txn,
+            table="insertion_events",
+            values={
+                "event_id": event.event_id,
+                "room_id": event.room_id,
+                "next_chunk_id": next_chunk_id,
+            },
+        )
+
+        # Insert an edge for every prev_event connection
+        for prev_event_id in event.prev_events:
+            self.db_pool.simple_insert_txn(
+                txn,
+                table="insertion_event_edges",
+                values={
+                    "event_id": event.event_id,
+                    "room_id": event.room_id,
+                    "insertion_prev_event_id": prev_event_id,
+                },
+            )
+
+    def _handle_chunk_event(self, txn: LoggingTransaction, event: EventBase):
+        """Handles inserting the chunk edges/connections between the chunk event
+        and an insertion event. Part of MSC2716.
+
+        Args:
+            txn: The database transaction object
+            event: The event to process
+        """
+
+        if event.type != EventTypes.MSC2716_CHUNK:
+            # Not a chunk event
+            return
+
+        # Skip processing a chunk event if the room version doesn't
+        # support it.
+        room_version = self.store.get_room_version_txn(txn, event.room_id)
+        if not room_version.msc2716_historical:
+            return
+
+        chunk_id = event.content.get(EventContentFields.MSC2716_CHUNK_ID)
+        if chunk_id is None:
+            # Invalid chunk event without a chunk ID
+            return
+
+        logger.debug("_handle_chunk_event chunk_id=%s %s", chunk_id, event)
+
+        # Keep track of the insertion event and the chunk ID
+        self.db_pool.simple_insert_txn(
+            txn,
+            table="chunk_events",
+            values={
+                "event_id": event.event_id,
+                "room_id": event.room_id,
+                "chunk_id": chunk_id,
+            },
+        )
+
+        # When we receive an event with a `chunk_id` referencing the
+        # `next_chunk_id` of the insertion event, we can remove it from the
+        # `insertion_event_extremities` table.
+        sql = """
+            DELETE FROM insertion_event_extremities WHERE event_id IN (
+                SELECT event_id FROM insertion_events
+                WHERE next_chunk_id = ?
+            )
+        """
+
+        txn.execute(sql, (chunk_id,))
+
     def _handle_redaction(self, txn, redacted_event_id):
         """Handles receiving a redaction and checking whether we need to remove
         any redacted relations from the database.
@@ -1879,20 +1986,28 @@ def _set_push_actions_for_event_and_users_txn(
                 ),
             )
 
-        for event, _ in events_and_contexts:
-            user_ids = self.db_pool.simple_select_onecol_txn(
-                txn,
-                table="event_push_actions_staging",
-                keyvalues={"event_id": event.event_id},
-                retcol="user_id",
-            )
+            room_to_event_ids: Dict[str, List[str]] = {}
+            for e, _ in events_and_contexts:
+                room_to_event_ids.setdefault(e.room_id, []).append(e.event_id)
 
-            for uid in user_ids:
-                txn.call_after(
-                    self.store.get_unread_event_push_actions_by_room_for_user.invalidate_many,
-                    (event.room_id, uid),
+            for room_id, event_ids in room_to_event_ids.items():
+                rows = self.db_pool.simple_select_many_txn(
+                    txn,
+                    table="event_push_actions_staging",
+                    column="event_id",
+                    iterable=event_ids,
+                    keyvalues={},
+                    retcols=("user_id",),
                 )
 
+                user_ids = {row["user_id"] for row in rows}
+
+                for user_id in user_ids:
+                    txn.call_after(
+                        self.store.get_unread_event_push_actions_by_room_for_user.invalidate,
+                        (room_id, user_id),
+                    )
+
         # Now we delete the staging area for *all* events that were being
         # persisted.
         txn.execute_batch(
@@ -1903,7 +2018,7 @@ def _set_push_actions_for_event_and_users_txn(
     def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id):
         # Sad that we have to blow away the cache for the whole room here
         txn.call_after(
-            self.store.get_unread_event_push_actions_by_room_for_user.invalidate_many,
+            self.store.get_unread_event_push_actions_by_room_for_user.invalidate,
             (room_id,),
         )
         txn.execute(
@@ -1998,19 +2113,17 @@ def _update_backward_extremeties(self, txn, events):
 
         Forward extremities are handled when we first start persisting the events.
         """
-        events_by_room = {}  # type: Dict[str, List[EventBase]]
-        for ev in events:
-            events_by_room.setdefault(ev.room_id, []).append(ev)
-
+        # From the events passed in, add all of the prev events as backwards extremities.
+        # Ignore any events that are already backwards extrems or outliers.
         query = (
             "INSERT INTO event_backward_extremities (event_id, room_id)"
             " SELECT ?, ? WHERE NOT EXISTS ("
-            " SELECT 1 FROM event_backward_extremities"
-            " WHERE event_id = ? AND room_id = ?"
+            "   SELECT 1 FROM event_backward_extremities"
+            "   WHERE event_id = ? AND room_id = ?"
             " )"
             " AND NOT EXISTS ("
-            " SELECT 1 FROM events WHERE event_id = ? AND room_id = ? "
-            " AND outlier = ?"
+            "   SELECT 1 FROM events WHERE event_id = ? AND room_id = ? "
+            "   AND outlier = ?"
             " )"
         )
 
@@ -2024,6 +2137,8 @@ def _update_backward_extremeties(self, txn, events):
             ],
         )
 
+        # Delete all these events that we've already fetched and now know that their
+        # prev events are the new backwards extremeties.
         query = (
             "DELETE FROM event_backward_extremities"
             " WHERE event_id = ? AND room_id = ?"
diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py
index 78367ea58d..6fcb2b8353 100644
--- a/synapse/storage/databases/main/events_bg_updates.py
+++ b/synapse/storage/databases/main/events_bg_updates.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,6 +29,34 @@
 logger = logging.getLogger(__name__)
 
 
+_REPLACE_STREAM_ORDERING_SQL_COMMANDS = (
+    # there should be no leftover rows without a stream_ordering2, but just in case...
+    "UPDATE events SET stream_ordering2 = stream_ordering WHERE stream_ordering2 IS NULL",
+    # now we can drop the rule and switch the columns
+    "DROP RULE populate_stream_ordering2 ON events",
+    "ALTER TABLE events DROP COLUMN stream_ordering",
+    "ALTER TABLE events RENAME COLUMN stream_ordering2 TO stream_ordering",
+    # ... and finally, rename the indexes into place for consistency with sqlite
+    "ALTER INDEX event_contains_url_index2 RENAME TO event_contains_url_index",
+    "ALTER INDEX events_order_room2 RENAME TO events_order_room",
+    "ALTER INDEX events_room_stream2 RENAME TO events_room_stream",
+    "ALTER INDEX events_ts2 RENAME TO events_ts",
+)
+
+
+class _BackgroundUpdates:
+    EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
+    EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url"
+    DELETE_SOFT_FAILED_EXTREMITIES = "delete_soft_failed_extremities"
+    POPULATE_STREAM_ORDERING2 = "populate_stream_ordering2"
+    INDEX_STREAM_ORDERING2 = "index_stream_ordering2"
+    INDEX_STREAM_ORDERING2_CONTAINS_URL = "index_stream_ordering2_contains_url"
+    INDEX_STREAM_ORDERING2_ROOM_ORDER = "index_stream_ordering2_room_order"
+    INDEX_STREAM_ORDERING2_ROOM_STREAM = "index_stream_ordering2_room_stream"
+    INDEX_STREAM_ORDERING2_TS = "index_stream_ordering2_ts"
+    REPLACE_STREAM_ORDERING_COLUMN = "replace_stream_ordering_column"
+
+
 @attr.s(slots=True, frozen=True)
 class _CalculateChainCover:
     """Return value for _calculate_chain_cover_txn."""
@@ -49,19 +76,15 @@ class _CalculateChainCover:
 
 
 class EventsBackgroundUpdatesStore(SQLBaseStore):
-
-    EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
-    EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url"
-    DELETE_SOFT_FAILED_EXTREMITIES = "delete_soft_failed_extremities"
-
     def __init__(self, database: DatabasePool, db_conn, hs):
         super().__init__(database, db_conn, hs)
 
         self.db_pool.updates.register_background_update_handler(
-            self.EVENT_ORIGIN_SERVER_TS_NAME, self._background_reindex_origin_server_ts
+            _BackgroundUpdates.EVENT_ORIGIN_SERVER_TS_NAME,
+            self._background_reindex_origin_server_ts,
         )
         self.db_pool.updates.register_background_update_handler(
-            self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME,
+            _BackgroundUpdates.EVENT_FIELDS_SENDER_URL_UPDATE_NAME,
             self._background_reindex_fields_sender,
         )
 
@@ -86,7 +109,8 @@ def __init__(self, database: DatabasePool, db_conn, hs):
         )
 
         self.db_pool.updates.register_background_update_handler(
-            self.DELETE_SOFT_FAILED_EXTREMITIES, self._cleanup_extremities_bg_update
+            _BackgroundUpdates.DELETE_SOFT_FAILED_EXTREMITIES,
+            self._cleanup_extremities_bg_update,
         )
 
         self.db_pool.updates.register_background_update_handler(
@@ -140,6 +164,59 @@ def __init__(self, database: DatabasePool, db_conn, hs):
             self._purged_chain_cover_index,
         )
 
+        ################################################################################
+
+        # bg updates for replacing stream_ordering with a BIGINT
+        # (these only run on postgres.)
+
+        self.db_pool.updates.register_background_update_handler(
+            _BackgroundUpdates.POPULATE_STREAM_ORDERING2,
+            self._background_populate_stream_ordering2,
+        )
+        # CREATE UNIQUE INDEX events_stream_ordering ON events(stream_ordering2);
+        self.db_pool.updates.register_background_index_update(
+            _BackgroundUpdates.INDEX_STREAM_ORDERING2,
+            index_name="events_stream_ordering",
+            table="events",
+            columns=["stream_ordering2"],
+            unique=True,
+        )
+        # CREATE INDEX event_contains_url_index ON events(room_id, topological_ordering, stream_ordering) WHERE contains_url = true AND outlier = false;
+        self.db_pool.updates.register_background_index_update(
+            _BackgroundUpdates.INDEX_STREAM_ORDERING2_CONTAINS_URL,
+            index_name="event_contains_url_index2",
+            table="events",
+            columns=["room_id", "topological_ordering", "stream_ordering2"],
+            where_clause="contains_url = true AND outlier = false",
+        )
+        # CREATE INDEX events_order_room ON events(room_id, topological_ordering, stream_ordering);
+        self.db_pool.updates.register_background_index_update(
+            _BackgroundUpdates.INDEX_STREAM_ORDERING2_ROOM_ORDER,
+            index_name="events_order_room2",
+            table="events",
+            columns=["room_id", "topological_ordering", "stream_ordering2"],
+        )
+        # CREATE INDEX events_room_stream ON events(room_id, stream_ordering);
+        self.db_pool.updates.register_background_index_update(
+            _BackgroundUpdates.INDEX_STREAM_ORDERING2_ROOM_STREAM,
+            index_name="events_room_stream2",
+            table="events",
+            columns=["room_id", "stream_ordering2"],
+        )
+        # CREATE INDEX events_ts ON events(origin_server_ts, stream_ordering);
+        self.db_pool.updates.register_background_index_update(
+            _BackgroundUpdates.INDEX_STREAM_ORDERING2_TS,
+            index_name="events_ts2",
+            table="events",
+            columns=["origin_server_ts", "stream_ordering2"],
+        )
+        self.db_pool.updates.register_background_update_handler(
+            _BackgroundUpdates.REPLACE_STREAM_ORDERING_COLUMN,
+            self._background_replace_stream_ordering_column,
+        )
+
+        ################################################################################
+
     async def _background_reindex_fields_sender(self, progress, batch_size):
         target_min_stream_id = progress["target_min_stream_id_inclusive"]
         max_stream_id = progress["max_stream_id_exclusive"]
@@ -191,18 +268,18 @@ def reindex_txn(txn):
             }
 
             self.db_pool.updates._background_update_progress_txn(
-                txn, self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, progress
+                txn, _BackgroundUpdates.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, progress
             )
 
             return len(rows)
 
         result = await self.db_pool.runInteraction(
-            self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, reindex_txn
+            _BackgroundUpdates.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, reindex_txn
         )
 
         if not result:
             await self.db_pool.updates._end_background_update(
-                self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME
+                _BackgroundUpdates.EVENT_FIELDS_SENDER_URL_UPDATE_NAME
             )
 
         return result
@@ -265,18 +342,18 @@ def reindex_search_txn(txn):
             }
 
             self.db_pool.updates._background_update_progress_txn(
-                txn, self.EVENT_ORIGIN_SERVER_TS_NAME, progress
+                txn, _BackgroundUpdates.EVENT_ORIGIN_SERVER_TS_NAME, progress
             )
 
             return len(rows_to_update)
 
         result = await self.db_pool.runInteraction(
-            self.EVENT_ORIGIN_SERVER_TS_NAME, reindex_search_txn
+            _BackgroundUpdates.EVENT_ORIGIN_SERVER_TS_NAME, reindex_search_txn
         )
 
         if not result:
             await self.db_pool.updates._end_background_update(
-                self.EVENT_ORIGIN_SERVER_TS_NAME
+                _BackgroundUpdates.EVENT_ORIGIN_SERVER_TS_NAME
             )
 
         return result
@@ -455,7 +532,7 @@ def _cleanup_extremities_bg_update_txn(txn):
 
         if not num_handled:
             await self.db_pool.updates._end_background_update(
-                self.DELETE_SOFT_FAILED_EXTREMITIES
+                _BackgroundUpdates.DELETE_SOFT_FAILED_EXTREMITIES
             )
 
             def _drop_table_txn(txn):
@@ -838,7 +915,6 @@ def _calculate_chain_cover_txn(
         # We want to do a `(topological_ordering, stream_ordering) > (?,?)`
         # comparison, but that is not supported on older SQLite versions
         tuple_clause, tuple_args = make_tuple_comparison_clause(
-            self.database_engine,
             [
                 ("events.room_id", last_room_id),
                 ("topological_ordering", last_depth),
@@ -884,9 +960,9 @@ def _calculate_chain_cover_txn(
         event_to_types = {row[0]: (row[1], row[2]) for row in rows}
 
         # Calculate the new last position we've processed up to.
-        new_last_depth = rows[-1][3] if rows else last_depth  # type: int
-        new_last_stream = rows[-1][4] if rows else last_stream  # type: int
-        new_last_room_id = rows[-1][5] if rows else ""  # type: str
+        new_last_depth: int = rows[-1][3] if rows else last_depth
+        new_last_stream: int = rows[-1][4] if rows else last_stream
+        new_last_room_id: str = rows[-1][5] if rows else ""
 
         # Map from room_id to last depth/stream_ordering processed for the room,
         # excluding the last room (which we're likely still processing). We also
@@ -913,7 +989,7 @@ def _calculate_chain_cover_txn(
             retcols=("event_id", "auth_id"),
         )
 
-        event_to_auth_chain = {}  # type: Dict[str, List[str]]
+        event_to_auth_chain: Dict[str, List[str]] = {}
         for row in auth_events:
             event_to_auth_chain.setdefault(row["event_id"], []).append(row["auth_id"])
 
@@ -1011,3 +1087,81 @@ def purged_chain_cover_txn(txn) -> int:
             await self.db_pool.updates._end_background_update("purged_chain_cover")
 
         return result
+
+    async def _background_populate_stream_ordering2(
+        self, progress: JsonDict, batch_size: int
+    ) -> int:
+        """Populate events.stream_ordering2, then replace stream_ordering
+
+        This is to deal with the fact that stream_ordering was initially created as a
+        32-bit integer field.
+        """
+        batch_size = max(batch_size, 1)
+
+        def process(txn: Cursor) -> int:
+            last_stream = progress.get("last_stream", -(1 << 31))
+            txn.execute(
+                """
+                UPDATE events SET stream_ordering2=stream_ordering
+                WHERE stream_ordering IN (
+                   SELECT stream_ordering FROM events WHERE stream_ordering > ?
+                   ORDER BY stream_ordering LIMIT ?
+                )
+                RETURNING stream_ordering;
+                """,
+                (last_stream, batch_size),
+            )
+            row_count = txn.rowcount
+            if row_count == 0:
+                return 0
+            last_stream = max(row[0] for row in txn)
+            logger.info("populated stream_ordering2 up to %i", last_stream)
+
+            self.db_pool.updates._background_update_progress_txn(
+                txn,
+                _BackgroundUpdates.POPULATE_STREAM_ORDERING2,
+                {"last_stream": last_stream},
+            )
+            return row_count
+
+        result = await self.db_pool.runInteraction(
+            "_background_populate_stream_ordering2", process
+        )
+
+        if result != 0:
+            return result
+
+        await self.db_pool.updates._end_background_update(
+            _BackgroundUpdates.POPULATE_STREAM_ORDERING2
+        )
+        return 0
+
+    async def _background_replace_stream_ordering_column(
+        self, progress: JsonDict, batch_size: int
+    ) -> int:
+        """Drop the old 'stream_ordering' column and rename 'stream_ordering2' into its place."""
+
+        def process(txn: Cursor) -> None:
+            for sql in _REPLACE_STREAM_ORDERING_SQL_COMMANDS:
+                logger.info("completing stream_ordering migration: %s", sql)
+                txn.execute(sql)
+
+        # ANALYZE the new column to build stats on it, to encourage PostgreSQL to use the
+        # indexes on it.
+        # We need to pass execute a dummy function to handle the txn's result otherwise
+        # it tries to call fetchall() on it and fails because there's no result to fetch.
+        await self.db_pool.execute(
+            "background_analyze_new_stream_ordering_column",
+            lambda txn: None,
+            "ANALYZE events(stream_ordering2)",
+        )
+
+        await self.db_pool.runInteraction(
+            "_background_replace_stream_ordering_column", process
+        )
+
+        await self.db_pool.updates._end_background_update(
+            _BackgroundUpdates.REPLACE_STREAM_ORDERING_COLUMN
+        )
+
+        return 0
diff --git a/synapse/storage/databases/main/events_forward_extremities.py b/synapse/storage/databases/main/events_forward_extremities.py
index b3703ae161..6d2688d711 100644
--- a/synapse/storage/databases/main/events_forward_extremities.py
+++ b/synapse/storage/databases/main/events_forward_extremities.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py
index dfb638ea54..375463e4e9 100644
--- a/synapse/storage/databases/main/events_worker.py
+++ b/synapse/storage/databases/main/events_worker.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,9 +14,19 @@
 
 import logging
 import threading
-from collections import namedtuple
-from typing import Dict, Iterable, List, Optional, Tuple, overload
+from typing import (
+    Collection,
+    Container,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    overload,
+)
 
+import attr
 from constantly import NamedConstant, Names
 from typing_extensions import Literal
 
@@ -33,7 +42,11 @@
 from synapse.events import EventBase, make_event_from_dict
 from synapse.events.snapshot import EventContext
 from synapse.events.utils import prune_event
-from synapse.logging.context import PreserveLoggingContext, current_context
+from synapse.logging.context import (
+    PreserveLoggingContext,
+    current_context,
+    make_deferred_yieldable,
+)
 from synapse.metrics.background_process_metrics import (
     run_as_background_process,
     wrap_as_background_process,
@@ -46,8 +59,10 @@
 from synapse.storage.engines import PostgresEngine
 from synapse.storage.util.id_generators import MultiWriterIdGenerator, StreamIdGenerator
 from synapse.storage.util.sequence import build_sequence_generator
-from synapse.types import Collection, JsonDict, get_domain_from_id
-from synapse.util.caches.descriptors import cached
+from synapse.types import JsonDict, get_domain_from_id
+from synapse.util import unwrapFirstError
+from synapse.util.async_helpers import ObservableDeferred
+from synapse.util.caches.descriptors import cached, cachedList
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.iterutils import batch_iter
 from synapse.util.metrics import Measure
@@ -65,7 +80,10 @@
 EVENT_QUEUE_TIMEOUT_S = 0.1  # Timeout when waiting for requests for events
 
 
-_EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
+@attr.s(slots=True, auto_attribs=True)
+class _EventCacheEntry:
+    event: EventBase
+    redacted_event: Optional[EventBase]
 
 
 class EventRedactBehaviour(Names):
@@ -149,10 +167,16 @@ def __init__(self, database: DatabasePool, db_conn, hs):
 
         self._get_event_cache = LruCache(
             cache_name="*getEvent*",
-            keylen=3,
             max_size=hs.config.caches.event_cache_size,
         )
 
+        # Map from event ID to a deferred that will result in a map from event
+        # ID to cache entry. Note that the returned dict may not have the
+        # requested event in it if the event isn't in the DB.
+        self._current_event_fetches: Dict[
+            str, ObservableDeferred[Dict[str, _EventCacheEntry]]
+        ] = {}
+
         self._event_fetch_lock = threading.Condition()
         self._event_fetch_list = []
         self._event_fetch_ongoing = 0
@@ -468,7 +492,9 @@ async def get_events_as_list(
 
         return events
 
-    async def _get_events_from_cache_or_db(self, event_ids, allow_rejected=False):
+    async def _get_events_from_cache_or_db(
+        self, event_ids: Iterable[str], allow_rejected: bool = False
+    ) -> Dict[str, _EventCacheEntry]:
         """Fetch a bunch of events from the cache or the database.
 
         If events are pulled from the database, they will be cached for future lookups.
@@ -477,53 +503,107 @@ async def _get_events_from_cache_or_db(self, event_ids, allow_rejected=False):
 
         Args:
 
-            event_ids (Iterable[str]): The event_ids of the events to fetch
+            event_ids: The event_ids of the events to fetch
 
-            allow_rejected (bool): Whether to include rejected events. If False,
+            allow_rejected: Whether to include rejected events. If False,
                 rejected events are omitted from the response.
 
         Returns:
-            Dict[str, _EventCacheEntry]:
-                map from event id to result
+            map from event id to result
         """
         event_entry_map = self._get_events_from_cache(
-            event_ids, allow_rejected=allow_rejected
+            event_ids,
         )
 
-        missing_events_ids = [e for e in event_ids if e not in event_entry_map]
+        missing_events_ids = {e for e in event_ids if e not in event_entry_map}
+
+        # We now look up if we're already fetching some of the events in the DB,
+        # if so we wait for those lookups to finish instead of pulling the same
+        # events out of the DB multiple times.
+        already_fetching: Dict[str, defer.Deferred] = {}
+
+        for event_id in missing_events_ids:
+            deferred = self._current_event_fetches.get(event_id)
+            if deferred is not None:
+                # We're already pulling the event out of the DB. Add the deferred
+                # to the collection of deferreds to wait on.
+                already_fetching[event_id] = deferred.observe()
+
+        missing_events_ids.difference_update(already_fetching)
 
         if missing_events_ids:
             log_ctx = current_context()
             log_ctx.record_event_fetch(len(missing_events_ids))
 
+            # Add entries to `self._current_event_fetches` for each event we're
+            # going to pull from the DB. We use a single deferred that resolves
+            # to all the events we pulled from the DB (this will result in this
+            # function returning more events than requested, but that can happen
+            # already due to `_get_events_from_db`).
+            fetching_deferred: ObservableDeferred[
+                Dict[str, _EventCacheEntry]
+            ] = ObservableDeferred(defer.Deferred())
+            for event_id in missing_events_ids:
+                self._current_event_fetches[event_id] = fetching_deferred
+
             # Note that _get_events_from_db is also responsible for turning db rows
             # into FrozenEvents (via _get_event_from_row), which involves seeing if
             # the events have been redacted, and if so pulling the redaction event out
             # of the database to check it.
             #
-            missing_events = await self._get_events_from_db(
-                missing_events_ids, allow_rejected=allow_rejected
-            )
+            try:
+                missing_events = await self._get_events_from_db(
+                    missing_events_ids,
+                )
+
+                event_entry_map.update(missing_events)
+            except Exception as e:
+                with PreserveLoggingContext():
+                    fetching_deferred.errback(e)
+                raise e
+            finally:
+                # Ensure that we mark these events as no longer being fetched.
+                for event_id in missing_events_ids:
+                    self._current_event_fetches.pop(event_id, None)
+
+            with PreserveLoggingContext():
+                fetching_deferred.callback(missing_events)
+
+        if already_fetching:
+            # Wait for the other event requests to finish and add their results
+            # to ours.
+            results = await make_deferred_yieldable(
+                defer.gatherResults(
+                    already_fetching.values(),
+                    consumeErrors=True,
+                )
+            ).addErrback(unwrapFirstError)
 
-            event_entry_map.update(missing_events)
+            for result in results:
+                event_entry_map.update(result)
+
+        if not allow_rejected:
+            event_entry_map = {
+                event_id: entry
+                for event_id, entry in event_entry_map.items()
+                if not entry.event.rejected_reason
+            }
 
         return event_entry_map
 
     def _invalidate_get_event_cache(self, event_id):
         self._get_event_cache.invalidate((event_id,))
 
-    def _get_events_from_cache(self, events, allow_rejected, update_metrics=True):
-        """Fetch events from the caches
+    def _get_events_from_cache(
+        self, events: Iterable[str], update_metrics: bool = True
+    ) -> Dict[str, _EventCacheEntry]:
+        """Fetch events from the caches.
 
-        Args:
-            events (Iterable[str]): list of event_ids to fetch
-            allow_rejected (bool): Whether to return events that were rejected
-            update_metrics (bool): Whether to update the cache hit ratio metrics
+        May return rejected events.
 
-        Returns:
-            dict of event_id -> _EventCacheEntry for each event_id in cache. If
-            allow_rejected is `False` then there will still be an entry but it
-            will be `None`
+        Args:
+            events: list of event_ids to fetch
+            update_metrics: Whether to update the cache hit ratio metrics
         """
         event_map = {}
 
@@ -534,17 +614,14 @@ def _get_events_from_cache(self, events, allow_rejected, update_metrics=True):
             if not ret:
                 continue
 
-            if allow_rejected or not ret.event.rejected_reason:
-                event_map[event_id] = ret
-            else:
-                event_map[event_id] = None
+            event_map[event_id] = ret
 
         return event_map
 
     async def get_stripped_room_state_from_event_context(
         self,
         context: EventContext,
-        state_types_to_include: List[str],
+        state_types_to_include: Container[str],
         membership_user_id: Optional[str] = None,
     ) -> List[JsonDict]:
         """
@@ -664,23 +741,23 @@ def fire(evs, exc):
                 with PreserveLoggingContext():
                     self.hs.get_reactor().callFromThread(fire, event_list, e)
 
-    async def _get_events_from_db(self, event_ids, allow_rejected=False):
+    async def _get_events_from_db(
+        self, event_ids: Iterable[str]
+    ) -> Dict[str, _EventCacheEntry]:
         """Fetch a bunch of events from the database.
 
+        May return rejected events.
+
         Returned events will be added to the cache for future lookups.
 
         Unknown events are omitted from the response.
 
         Args:
-            event_ids (Iterable[str]): The event_ids of the events to fetch
-
-            allow_rejected (bool): Whether to include rejected events. If False,
-                rejected events are omitted from the response.
+            event_ids: The event_ids of the events to fetch
 
         Returns:
-            Dict[str, _EventCacheEntry]:
-                map from event id to result. May return extra events which
-                weren't asked for.
+            map from event id to result. May return extra events which
+            weren't asked for.
         """
         fetched_events = {}
         events_to_fetch = event_ids
@@ -709,9 +786,6 @@ async def _get_events_from_db(self, event_ids, allow_rejected=False):
 
             rejected_reason = row["rejected_reason"]
 
-            if not allow_rejected and rejected_reason:
-                continue
-
             # If the event or metadata cannot be parsed, log the error and act
             # as if the event is unknown.
             try:
@@ -1038,32 +1112,74 @@ async def have_events_in_timeline(self, event_ids):
 
         return {r["event_id"] for r in rows}
 
-    async def have_seen_events(self, event_ids):
+    async def have_seen_events(
+        self, room_id: str, event_ids: Iterable[str]
+    ) -> Set[str]:
         """Given a list of event ids, check if we have already processed them.
 
+        The room_id is only used to structure the cache (so that it can later be
+        invalidated by room_id) - there is no guarantee that the events are actually
+        in the room in question.
+
         Args:
-            event_ids (iterable[str]):
+            room_id: Room we are polling
+            event_ids: events we are looking for
 
         Returns:
             set[str]: The events we have already seen.
         """
+        res = await self._have_seen_events_dict(
+            (room_id, event_id) for event_id in event_ids
+        )
+        return {eid for ((_rid, eid), have_event) in res.items() if have_event}
+
+    @cachedList("have_seen_event", "keys")
+    async def _have_seen_events_dict(
+        self, keys: Iterable[Tuple[str, str]]
+    ) -> Dict[Tuple[str, str], bool]:
+        """Helper for have_seen_events
+
+        Returns:
+             a dict {(room_id, event_id)-> bool}
+        """
         # if the event cache contains the event, obviously we've seen it.
-        results = {x for x in event_ids if self._get_event_cache.contains(x)}
 
-        def have_seen_events_txn(txn, chunk):
-            sql = "SELECT event_id FROM events as e WHERE "
+        cache_results = {
+            (rid, eid) for (rid, eid) in keys if self._get_event_cache.contains((eid,))
+        }
+        results = {x: True for x in cache_results}
+
+        def have_seen_events_txn(txn, chunk: Tuple[Tuple[str, str], ...]):
+            # we deliberately do *not* query the database for room_id, to make the
+            # query an index-only lookup on `events_event_id_key`.
+            #
+            # We therefore pull the events from the database into a set...
+
+            sql = "SELECT event_id FROM events AS e WHERE "
             clause, args = make_in_list_sql_clause(
-                txn.database_engine, "e.event_id", chunk
+                txn.database_engine, "e.event_id", [eid for (_rid, eid) in chunk]
             )
             txn.execute(sql + clause, args)
-            results.update(row[0] for row in txn)
+            found_events = {eid for eid, in txn}
 
-        for chunk in batch_iter((x for x in event_ids if x not in results), 100):
+            # ... and then we can update the results for each row in the batch
+            results.update({(rid, eid): (eid in found_events) for (rid, eid) in chunk})
+
+        # each batch requires its own index scan, so we make the batches as big as
+        # possible.
+        for chunk in batch_iter((k for k in keys if k not in cache_results), 500):
             await self.db_pool.runInteraction(
                 "have_seen_events", have_seen_events_txn, chunk
             )
+
         return results
 
+    @cached(max_entries=100000, tree=True)
+    async def have_seen_event(self, room_id: str, event_id: str):
+        # this only exists for the benefit of the @cachedList descriptor on
+        # _have_seen_events_dict
+        raise NotImplementedError()
+
     def _get_current_state_event_counts_txn(self, txn, room_id):
         """
         See get_current_state_event_counts.
@@ -1315,10 +1431,10 @@ def get_deltas_for_stream_id_txn(txn, stream_id):
         # we need to make sure that, for every stream id in the results, we get *all*
         # the rows with that stream id.
 
-        rows = await self.db_pool.runInteraction(
+        rows: List[Tuple] = await self.db_pool.runInteraction(
             "get_all_updated_current_state_deltas",
             get_all_updated_current_state_deltas_txn,
-        )  # type: List[Tuple]
+        )
 
         # if we've got fewer rows than the limit, we're good
         if len(rows) < target_row_count:
@@ -1419,7 +1535,7 @@ async def get_already_persisted_events(
         """
 
         mapping = {}
-        txn_id_to_event = {}  # type: Dict[Tuple[str, int, str], str]
+        txn_id_to_event: Dict[Tuple[str, int, str], str] = {}
 
         for event in events:
             token_id = getattr(event.internal_metadata, "token_id", None)
diff --git a/synapse/storage/databases/main/filtering.py b/synapse/storage/databases/main/filtering.py
index d2f5b9a502..bb244a03c0 100644
--- a/synapse/storage/databases/main/filtering.py
+++ b/synapse/storage/databases/main/filtering.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/group_server.py b/synapse/storage/databases/main/group_server.py
index ac07e0197b..e70d3649ff 100644
--- a/synapse/storage/databases/main/group_server.py
+++ b/synapse/storage/databases/main/group_server.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -28,8 +27,11 @@
 _DEFAULT_CATEGORY_ID = ""
 _DEFAULT_ROLE_ID = ""
 
+
 # A room in a group.
-_RoomInGroup = TypedDict("_RoomInGroup", {"room_id": str, "is_public": bool})
+class _RoomInGroup(TypedDict):
+    room_id: str
+    is_public: bool
 
 
 class GroupServerWorkerStore(SQLBaseStore):
@@ -93,6 +95,7 @@ async def get_rooms_in_group(
               "is_public": False                    # Whether this is a public room or not
             }
         """
+
         # TODO: Pagination
 
         def _get_rooms_in_group_txn(txn):
@@ -1027,8 +1030,8 @@ async def add_user_to_group(
         user_id: str,
         is_admin: bool = False,
         is_public: bool = True,
-        local_attestation: dict = None,
-        remote_attestation: dict = None,
+        local_attestation: Optional[dict] = None,
+        remote_attestation: Optional[dict] = None,
     ) -> None:
         """Add a user to the group server.
 
@@ -1171,7 +1174,7 @@ async def register_user_group_membership(
         user_id: str,
         membership: str,
         is_admin: bool = False,
-        content: JsonDict = {},
+        content: Optional[JsonDict] = None,
         local_attestation: Optional[dict] = None,
         remote_attestation: Optional[dict] = None,
         is_publicised: bool = False,
@@ -1192,6 +1195,8 @@ async def register_user_group_membership(
             is_publicised: Whether this should be publicised.
         """
 
+        content = content or {}
+
         def _register_user_group_membership_txn(txn, next_id):
             # TODO: Upsert?
             self.db_pool.simple_delete_txn(
diff --git a/synapse/storage/databases/main/keys.py b/synapse/storage/databases/main/keys.py
index d504323b03..6990f3ed1d 100644
--- a/synapse/storage/databases/main/keys.py
+++ b/synapse/storage/databases/main/keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd.
 #
@@ -56,7 +55,7 @@ async def get_server_verify_keys(
         """
         keys = {}
 
-        def _get_keys(txn: Cursor, batch: Tuple[Tuple[str, str]]) -> None:
+        def _get_keys(txn: Cursor, batch: Tuple[Tuple[str, str], ...]) -> None:
             """Processes a batch of keys to fetch, and adds the result to `keys`."""
 
             # batch_iter always returns tuples so it's safe to do len(batch)
diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py
new file mode 100644
index 0000000000..3d1dff660b
--- /dev/null
+++ b/synapse/storage/databases/main/lock.py
@@ -0,0 +1,349 @@
+# Copyright 2021 Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from types import TracebackType
+from typing import TYPE_CHECKING, Dict, Optional, Tuple, Type
+
+from twisted.internet.interfaces import IReactorCore
+
+from synapse.metrics.background_process_metrics import wrap_as_background_process
+from synapse.storage._base import SQLBaseStore
+from synapse.storage.database import DatabasePool, LoggingTransaction
+from synapse.storage.types import Connection
+from synapse.util import Clock
+from synapse.util.stringutils import random_string
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+
+logger = logging.getLogger(__name__)
+
+
+# How often to renew an acquired lock by updating the `last_renewed_ts` time in
+# the lock table.
+_RENEWAL_INTERVAL_MS = 30 * 1000
+
+# How long before an acquired lock times out.
+_LOCK_TIMEOUT_MS = 2 * 60 * 1000
+
+
+class LockStore(SQLBaseStore):
+    """Provides a best effort distributed lock between worker instances.
+
+    Locks are identified by a name and key. A lock is acquired by inserting into
+    the `worker_locks` table if a) there is no existing row for the name/key or
+    b) the existing row has a `last_renewed_ts` older than `_LOCK_TIMEOUT_MS`.
+
+    When a lock is taken out the instance inserts a random `token`, the instance
+    that holds that token holds the lock until it drops (or times out).
+
+    The instance that holds the lock should regularly update the
+    `last_renewed_ts` column with the current time.
+    """
+
+    def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer"):
+        super().__init__(database, db_conn, hs)
+
+        self._reactor = hs.get_reactor()
+        self._instance_name = hs.get_instance_id()
+
+        # A map from `(lock_name, lock_key)` to the token of any locks that we
+        # think we currently hold.
+        self._live_tokens: Dict[Tuple[str, str], str] = {}
+
+        # When we shut down we want to remove the locks. Technically this can
+        # lead to a race, as we may drop the lock while we are still processing.
+        # However, a) it should be a small window, b) the lock is best effort
+        # anyway and c) we want to really avoid leaking locks when we restart.
+        hs.get_reactor().addSystemEventTrigger(
+            "before",
+            "shutdown",
+            self._on_shutdown,
+        )
+
+    @wrap_as_background_process("LockStore._on_shutdown")
+    async def _on_shutdown(self) -> None:
+        """Called when the server is shutting down"""
+        logger.info("Dropping held locks due to shutdown")
+
+        # We need to take a copy of the tokens dict as dropping the locks will
+        # cause the dictionary to change.
+        tokens = dict(self._live_tokens)
+
+        for (lock_name, lock_key), token in tokens.items():
+            await self._drop_lock(lock_name, lock_key, token)
+
+        logger.info("Dropped locks due to shutdown")
+
+    async def try_acquire_lock(self, lock_name: str, lock_key: str) -> Optional["Lock"]:
+        """Try to acquire a lock for the given name/key. Will return an async
+        context manager if the lock is successfully acquired, which *must* be
+        used (otherwise the lock will leak).
+        """
+
+        now = self._clock.time_msec()
+        token = random_string(6)
+
+        if self.db_pool.engine.can_native_upsert:
+
+            def _try_acquire_lock_txn(txn: LoggingTransaction) -> bool:
+                # We take out the lock if either a) there is no row for the lock
+                # already or b) the existing row has timed out.
+                sql = """
+                    INSERT INTO worker_locks (lock_name, lock_key, instance_name, token, last_renewed_ts)
+                    VALUES (?, ?, ?, ?, ?)
+                    ON CONFLICT (lock_name, lock_key)
+                    DO UPDATE
+                        SET
+                            token = EXCLUDED.token,
+                            instance_name = EXCLUDED.instance_name,
+                            last_renewed_ts = EXCLUDED.last_renewed_ts
+                        WHERE
+                            worker_locks.last_renewed_ts < ?
+                """
+                txn.execute(
+                    sql,
+                    (
+                        lock_name,
+                        lock_key,
+                        self._instance_name,
+                        token,
+                        now,
+                        now - _LOCK_TIMEOUT_MS,
+                    ),
+                )
+
+                # We only acquired the lock if we inserted or updated the table.
+                return bool(txn.rowcount)
+
+            did_lock = await self.db_pool.runInteraction(
+                "try_acquire_lock",
+                _try_acquire_lock_txn,
+                # We can autocommit here as we're executing a single query, this
+                # will avoid serialization errors.
+                db_autocommit=True,
+            )
+            if not did_lock:
+                return None
+
+        else:
+            # If we're on an old SQLite we emulate the above logic by first
+            # clearing out any existing stale locks and then upserting.
+
+            def _try_acquire_lock_emulated_txn(txn: LoggingTransaction) -> bool:
+                sql = """
+                    DELETE FROM worker_locks
+                    WHERE
+                        lock_name = ?
+                        AND lock_key = ?
+                        AND last_renewed_ts < ?
+                """
+                txn.execute(
+                    sql,
+                    (lock_name, lock_key, now - _LOCK_TIMEOUT_MS),
+                )
+
+                inserted = self.db_pool.simple_upsert_txn_emulated(
+                    txn,
+                    table="worker_locks",
+                    keyvalues={
+                        "lock_name": lock_name,
+                        "lock_key": lock_key,
+                    },
+                    values={},
+                    insertion_values={
+                        "token": token,
+                        "last_renewed_ts": self._clock.time_msec(),
+                        "instance_name": self._instance_name,
+                    },
+                )
+
+                return inserted
+
+            did_lock = await self.db_pool.runInteraction(
+                "try_acquire_lock_emulated", _try_acquire_lock_emulated_txn
+            )
+
+            if not did_lock:
+                return None
+
+        self._live_tokens[(lock_name, lock_key)] = token
+
+        return Lock(
+            self._reactor,
+            self._clock,
+            self,
+            lock_name=lock_name,
+            lock_key=lock_key,
+            token=token,
+        )
+
+    async def _is_lock_still_valid(
+        self, lock_name: str, lock_key: str, token: str
+    ) -> bool:
+        """Checks whether this instance still holds the lock."""
+        last_renewed_ts = await self.db_pool.simple_select_one_onecol(
+            table="worker_locks",
+            keyvalues={
+                "lock_name": lock_name,
+                "lock_key": lock_key,
+                "token": token,
+            },
+            retcol="last_renewed_ts",
+            allow_none=True,
+            desc="is_lock_still_valid",
+        )
+        return (
+            last_renewed_ts is not None
+            and self._clock.time_msec() - _LOCK_TIMEOUT_MS < last_renewed_ts
+        )
+
+    async def _renew_lock(self, lock_name: str, lock_key: str, token: str) -> None:
+        """Attempt to renew the lock if we still hold it."""
+        await self.db_pool.simple_update(
+            table="worker_locks",
+            keyvalues={
+                "lock_name": lock_name,
+                "lock_key": lock_key,
+                "token": token,
+            },
+            updatevalues={"last_renewed_ts": self._clock.time_msec()},
+            desc="renew_lock",
+        )
+
+    async def _drop_lock(self, lock_name: str, lock_key: str, token: str) -> None:
+        """Attempt to drop the lock, if we still hold it"""
+        await self.db_pool.simple_delete(
+            table="worker_locks",
+            keyvalues={
+                "lock_name": lock_name,
+                "lock_key": lock_key,
+                "token": token,
+            },
+            desc="drop_lock",
+        )
+
+        self._live_tokens.pop((lock_name, lock_key), None)
+
+
+class Lock:
+    """An async context manager that manages an acquired lock, ensuring it is
+    regularly renewed and dropping it when the context manager exits.
+
+    The lock object has an `is_still_valid` method which can be used to
+    double-check the lock is still valid, if e.g. processing work in a loop.
+
+    For example:
+
+        lock = await self.store.try_acquire_lock(...)
+        if not lock:
+            return
+
+        async with lock:
+            for item in work:
+                await process(item)
+
+                if not await lock.is_still_valid():
+                    break
+    """
+
+    def __init__(
+        self,
+        reactor: IReactorCore,
+        clock: Clock,
+        store: LockStore,
+        lock_name: str,
+        lock_key: str,
+        token: str,
+    ) -> None:
+        self._reactor = reactor
+        self._clock = clock
+        self._store = store
+        self._lock_name = lock_name
+        self._lock_key = lock_key
+
+        self._token = token
+
+        self._looping_call = clock.looping_call(
+            self._renew, _RENEWAL_INTERVAL_MS, store, lock_name, lock_key, token
+        )
+
+        self._dropped = False
+
+    @staticmethod
+    @wrap_as_background_process("Lock._renew")
+    async def _renew(
+        store: LockStore,
+        lock_name: str,
+        lock_key: str,
+        token: str,
+    ) -> None:
+        """Renew the lock.
+
+        Note: this is a static method, rather than using self.*, so that we
+        don't end up with a reference to `self` in the reactor, which would stop
+        this from being cleaned up if we dropped the context manager.
+        """
+        await store._renew_lock(lock_name, lock_key, token)
+
+    async def is_still_valid(self) -> bool:
+        """Check if the lock is still held by us"""
+        return await self._store._is_lock_still_valid(
+            self._lock_name, self._lock_key, self._token
+        )
+
+    async def __aenter__(self) -> None:
+        if self._dropped:
+            raise Exception("Cannot reuse a Lock object")
+
+    async def __aexit__(
+        self,
+        _exctype: Optional[Type[BaseException]],
+        _excinst: Optional[BaseException],
+        _exctb: Optional[TracebackType],
+    ) -> bool:
+        await self.release()
+
+        return False
+
+    async def release(self) -> None:
+        """Release the lock.
+
+        This is automatically called when using the lock as a context manager.
+        """
+
+        if self._dropped:
+            return
+
+        if self._looping_call.running:
+            self._looping_call.stop()
+
+        await self._store._drop_lock(self._lock_name, self._lock_key, self._token)
+        self._dropped = True
+
+    def __del__(self) -> None:
+        if not self._dropped:
+            # We should not be dropped without the lock being released (unless
+            # we're shutting down), but if we are then let's at least stop
+            # renewing the lock.
+            if self._looping_call.running:
+                self._looping_call.stop()
+
+            if self._reactor.running:
+                logger.error(
+                    "Lock for (%s, %s) dropped without being released",
+                    self._lock_name,
+                    self._lock_key,
+                )
diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py
index 4f3d192562..2fa945d171 100644
--- a/synapse/storage/databases/main/media_repository.py
+++ b/synapse/storage/databases/main/media_repository.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020-2021 The Matrix.org Foundation C.I.C.
 #
@@ -22,6 +21,9 @@
 BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD = (
     "media_repository_drop_index_wo_method"
 )
+BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD_2 = (
+    "media_repository_drop_index_wo_method_2"
+)
 
 
 class MediaSortOrder(Enum):
@@ -85,23 +87,35 @@ def __init__(self, database: DatabasePool, db_conn, hs):
             unique=True,
         )
 
+        # the original impl of _drop_media_index_without_method was broken (see
+        # https://github.com/matrix-org/synapse/issues/8649), so we replace the original
+        # impl with a no-op and run the fixed migration as
+        # media_repository_drop_index_wo_method_2.
+        self.db_pool.updates.register_noop_background_update(
+            BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD
+        )
         self.db_pool.updates.register_background_update_handler(
-            BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD,
+            BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD_2,
             self._drop_media_index_without_method,
         )
 
     async def _drop_media_index_without_method(self, progress, batch_size):
+        """background update handler which removes the old constraints.
+
+        Note that this is only run on postgres.
+        """
+
         def f(txn):
             txn.execute(
                 "ALTER TABLE local_media_repository_thumbnails DROP CONSTRAINT IF EXISTS local_media_repository_thumbn_media_id_thumbnail_width_thum_key"
             )
             txn.execute(
-                "ALTER TABLE remote_media_cache_thumbnails DROP CONSTRAINT IF EXISTS remote_media_repository_thumbn_media_id_thumbnail_width_thum_key"
+                "ALTER TABLE remote_media_cache_thumbnails DROP CONSTRAINT IF EXISTS remote_media_cache_thumbnails_media_origin_media_id_thumbna_key"
             )
 
         await self.db_pool.runInteraction("drop_media_indices_without_method", f)
         await self.db_pool.updates._end_background_update(
-            BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD
+            BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD_2
         )
         return 1
 
@@ -129,6 +143,7 @@ async def get_local_media(self, media_id: str) -> Optional[Dict[str, Any]]:
                 "created_ts",
                 "quarantined_by",
                 "url_cache",
+                "safe_from_quarantine",
             ),
             allow_none=True,
             desc="get_local_media",
@@ -282,12 +297,12 @@ async def store_local_media(
             desc="store_local_media",
         )
 
-    async def mark_local_media_as_safe(self, media_id: str) -> None:
-        """Mark a local media as safe from quarantining."""
+    async def mark_local_media_as_safe(self, media_id: str, safe: bool = True) -> None:
+        """Mark a local media as safe or unsafe from quarantining."""
         await self.db_pool.simple_update_one(
             table="local_media_repository",
             keyvalues={"media_id": media_id},
-            updatevalues={"safe_from_quarantine": True},
+            updatevalues={"safe_from_quarantine": safe},
             desc="mark_local_media_as_safe",
         )
 
diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py
index 614a418a15..dc0bbc56ac 100644
--- a/synapse/storage/databases/main/metrics.py
+++ b/synapse/storage/databases/main/metrics.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -317,11 +316,140 @@ def _count_r30_users(txn):
 
         return await self.db_pool.runInteraction("count_r30_users", _count_r30_users)
 
+    async def count_r30v2_users(self) -> Dict[str, int]:
+        """
+        Counts the number of 30 day retained users, defined as users that:
+         - Appear more than once in the past 60 days
+         - Have more than 30 days between the most and least recent appearances that
+           occurred in the past 60 days.
+
+        (This is the second version of this metric, hence R30'v2')
+
+        Returns:
+             A mapping from client type to the number of 30-day retained users for that client.
+
+             The dict keys are:
+              - "all" (a combined number of users across any and all clients)
+              - "android" (Element Android)
+              - "ios" (Element iOS)
+              - "electron" (Element Desktop)
+              - "web" (any web application -- it's not possible to distinguish Element Web here)
+        """
+
+        def _count_r30v2_users(txn):
+            thirty_days_in_secs = 86400 * 30
+            now = int(self._clock.time())
+            sixty_days_ago_in_secs = now - 2 * thirty_days_in_secs
+            one_day_from_now_in_secs = now + 86400
+
+            # This is the 'per-platform' count.
+            sql = """
+                SELECT
+                    client_type,
+                    count(client_type)
+                FROM
+                    (
+                        SELECT
+                            user_id,
+                            CASE
+                                WHEN
+                                    LOWER(user_agent) LIKE '%%riot%%' OR
+                                    LOWER(user_agent) LIKE '%%element%%'
+                                    THEN CASE
+                                        WHEN
+                                            LOWER(user_agent) LIKE '%%electron%%'
+                                            THEN 'electron'
+                                        WHEN
+                                            LOWER(user_agent) LIKE '%%android%%'
+                                            THEN 'android'
+                                        WHEN
+                                            LOWER(user_agent) LIKE '%%ios%%'
+                                            THEN 'ios'
+                                        ELSE 'unknown'
+                                    END
+                                WHEN
+                                    LOWER(user_agent) LIKE '%%mozilla%%' OR
+                                    LOWER(user_agent) LIKE '%%gecko%%'
+                                    THEN 'web'
+                                ELSE 'unknown'
+                            END as client_type
+                        FROM
+                            user_daily_visits
+                        WHERE
+                            timestamp > ?
+                            AND
+                            timestamp < ?
+                        GROUP BY
+                            user_id,
+                            client_type
+                        HAVING
+                            max(timestamp) - min(timestamp) > ?
+                    ) AS temp
+                GROUP BY
+                    client_type
+                ;
+            """
+
+            # We initialise all the client types to zero, so we get an explicit
+            # zero if they don't appear in the query results
+            results = {"ios": 0, "android": 0, "web": 0, "electron": 0}
+            txn.execute(
+                sql,
+                (
+                    sixty_days_ago_in_secs * 1000,
+                    one_day_from_now_in_secs * 1000,
+                    thirty_days_in_secs * 1000,
+                ),
+            )
+
+            for row in txn:
+                if row[0] == "unknown":
+                    continue
+                results[row[0]] = row[1]
+
+            # This is the 'all users' count.
+            sql = """
+                SELECT COUNT(*) FROM (
+                    SELECT
+                        1
+                    FROM
+                        user_daily_visits
+                    WHERE
+                        timestamp > ?
+                        AND
+                        timestamp < ?
+                    GROUP BY
+                        user_id
+                    HAVING
+                        max(timestamp) - min(timestamp) > ?
+                ) AS r30_users
+            """
+
+            txn.execute(
+                sql,
+                (
+                    sixty_days_ago_in_secs * 1000,
+                    one_day_from_now_in_secs * 1000,
+                    thirty_days_in_secs * 1000,
+                ),
+            )
+            row = txn.fetchone()
+            if row is None:
+                results["all"] = 0
+            else:
+                results["all"] = row[0]
+
+            return results
+
+        return await self.db_pool.runInteraction(
+            "count_r30v2_users", _count_r30v2_users
+        )
+
     def _get_start_of_day(self):
         """
         Returns millisecond unixtime for start of UTC day.
         """
-        now = time.gmtime()
+        now = time.gmtime(self._clock.time())
         today_start = calendar.timegm((now.tm_year, now.tm_mon, now.tm_mday, 0, 0, 0))
         return today_start * 1000
 
@@ -353,7 +481,7 @@ def _generate_user_daily_visits(txn):
                     ) udv
                     ON u.user_id = udv.user_id AND u.device_id=udv.device_id
                     INNER JOIN users ON users.name=u.user_id
-                    WHERE last_seen > ? AND last_seen <= ?
+                    WHERE ? <= last_seen AND last_seen < ?
                     AND udv.timestamp IS NULL AND users.is_guest=0
                     AND users.appservice_id IS NULL
                     GROUP BY u.user_id, u.device_id
diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py
index 757da3d55d..d213b26703 100644
--- a/synapse/storage/databases/main/monthly_active_users.py
+++ b/synapse/storage/databases/main/monthly_active_users.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -298,17 +297,13 @@ def upsert_monthly_active_user_txn(self, txn, user_id):
         Args:
             txn (cursor):
             user_id (str): user to add/update
-
-        Returns:
-            bool: True if a new entry was created, False if an
-            existing one was updated.
         """
 
         # Am consciously deciding to lock the table on the basis that is ought
         # never be a big table and alternative approaches (batching multiple
         # upserts into a single txn) introduced a lot of extra complexity.
         # See https://github.com/matrix-org/synapse/issues/3854 for more
-        is_insert = self.db_pool.simple_upsert_txn(
+        self.db_pool.simple_upsert_txn(
             txn,
             table="monthly_active_users",
             keyvalues={"user_id": user_id},
@@ -323,8 +318,6 @@ def upsert_monthly_active_user_txn(self, txn, user_id):
             txn, self.user_last_seen_monthly_active, (user_id,)
         )
 
-        return is_insert
-
     async def populate_monthly_active_users(self, user_id):
         """Checks on the state of monthly active user limits and optionally
         add the user to the monthly active tables
diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py
index 0ff693a310..1388771c40 100644
--- a/synapse/storage/databases/main/presence.py
+++ b/synapse/storage/databases/main/presence.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,16 +12,70 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Dict, List, Tuple
+from typing import TYPE_CHECKING, Dict, Iterable, List, Tuple
 
-from synapse.api.presence import UserPresenceState
+from synapse.api.presence import PresenceState, UserPresenceState
+from synapse.replication.tcp.streams import PresenceStream
 from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause
+from synapse.storage.database import DatabasePool
+from synapse.storage.engines import PostgresEngine
+from synapse.storage.types import Connection
+from synapse.storage.util.id_generators import MultiWriterIdGenerator, StreamIdGenerator
 from synapse.util.caches.descriptors import cached, cachedList
+from synapse.util.caches.stream_change_cache import StreamChangeCache
 from synapse.util.iterutils import batch_iter
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 
 class PresenceStore(SQLBaseStore):
+    def __init__(
+        self,
+        database: DatabasePool,
+        db_conn: Connection,
+        hs: "HomeServer",
+    ):
+        super().__init__(database, db_conn, hs)
+
+        self._can_persist_presence = (
+            hs.get_instance_name() in hs.config.worker.writers.presence
+        )
+
+        if isinstance(database.engine, PostgresEngine):
+            self._presence_id_gen = MultiWriterIdGenerator(
+                db_conn=db_conn,
+                db=database,
+                stream_name="presence_stream",
+                instance_name=self._instance_name,
+                tables=[("presence_stream", "instance_name", "stream_id")],
+                sequence_name="presence_stream_sequence",
+                writers=hs.config.worker.writers.presence,
+            )
+        else:
+            self._presence_id_gen = StreamIdGenerator(
+                db_conn, "presence_stream", "stream_id"
+            )
+
+        self.hs = hs
+        self._presence_on_startup = self._get_active_presence(db_conn)
+
+        presence_cache_prefill, min_presence_val = self.db_pool.get_cache_dict(
+            db_conn,
+            "presence_stream",
+            entity_column="user_id",
+            stream_column="stream_id",
+            max_value=self._presence_id_gen.get_current_token(),
+        )
+        self.presence_stream_cache = StreamChangeCache(
+            "PresenceStreamChangeCache",
+            min_presence_val,
+            prefilled_cache=presence_cache_prefill,
+        )
+
     async def update_presence(self, presence_states):
+        assert self._can_persist_presence
+
         stream_ordering_manager = self._presence_id_gen.get_next_mult(
             len(presence_states)
         )
@@ -44,6 +97,15 @@ def _update_presence_txn(self, txn, stream_orderings, presence_states):
             )
             txn.call_after(self._get_presence_for_user.invalidate, (state.user_id,))
 
+        # Delete old rows to stop database from getting really big
+        sql = "DELETE FROM presence_stream WHERE stream_id < ? AND "
+
+        for states in batch_iter(presence_states, 50):
+            clause, args = make_in_list_sql_clause(
+                self.database_engine, "user_id", [s.user_id for s in states]
+            )
+            txn.execute(sql + clause, [stream_id] + list(args))
+
         # Actually insert new rows
         self.db_pool.simple_insert_many_txn(
             txn,
@@ -58,20 +120,12 @@ def _update_presence_txn(self, txn, stream_orderings, presence_states):
                     "last_user_sync_ts": state.last_user_sync_ts,
                     "status_msg": state.status_msg,
                     "currently_active": state.currently_active,
+                    "instance_name": self._instance_name,
                 }
                 for stream_id, state in zip(stream_orderings, presence_states)
             ],
         )
 
-        # Delete old rows to stop database from getting really big
-        sql = "DELETE FROM presence_stream WHERE stream_id < ? AND "
-
-        for states in batch_iter(presence_states, 50):
-            clause, args = make_in_list_sql_clause(
-                self.database_engine, "user_id", [s.user_id for s in states]
-            )
-            txn.execute(sql + clause, [stream_id] + list(args))
-
     async def get_all_presence_updates(
         self, instance_name: str, last_id: int, current_id: int, limit: int
     ) -> Tuple[List[Tuple[int, list]], int, bool]:
@@ -157,6 +211,61 @@ async def get_presence_for_users(self, user_ids):
 
         return {row["user_id"]: UserPresenceState(**row) for row in rows}
 
+    async def should_user_receive_full_presence_with_token(
+        self,
+        user_id: str,
+        from_token: int,
+    ) -> bool:
+        """Check whether the given user should receive full presence using the stream token
+        they're updating from.
+
+        Args:
+            user_id: The ID of the user to check.
+            from_token: The stream token included in their /sync token.
+
+        Returns:
+            True if the user should have full presence sent to them, False otherwise.
+        """
+
+        def _should_user_receive_full_presence_with_token_txn(txn):
+            sql = """
+                SELECT 1 FROM users_to_send_full_presence_to
+                WHERE user_id = ?
+                AND presence_stream_id >= ?
+            """
+            txn.execute(sql, (user_id, from_token))
+            return bool(txn.fetchone())
+
+        return await self.db_pool.runInteraction(
+            "should_user_receive_full_presence_with_token",
+            _should_user_receive_full_presence_with_token_txn,
+        )
+
+    async def add_users_to_send_full_presence_to(self, user_ids: Iterable[str]):
+        """Adds to the list of users who should receive a full snapshot of presence
+        upon their next sync.
+
+        Args:
+            user_ids: An iterable of user IDs.
+        """
+        # Add user entries to the table, updating the presence_stream_id column if the user already
+        # exists in the table.
+        await self.db_pool.simple_upsert_many(
+            table="users_to_send_full_presence_to",
+            key_names=("user_id",),
+            key_values=[(user_id,) for user_id in user_ids],
+            value_names=("presence_stream_id",),
+            # We save the current presence stream ID token along with the user ID entry so
+            # that when a user /sync's, even if they syncing multiple times across separate
+            # devices at different times, each device will receive full presence once - when
+            # the presence stream ID in their sync token is less than the one in the table
+            # for their user ID.
+            value_values=(
+                (self._presence_id_gen.get_current_token(),) for _ in user_ids
+            ),
+            desc="add_users_to_send_full_presence_to",
+        )
+
     async def get_presence_for_all_users(
         self,
         include_offline: bool = True,
@@ -217,3 +326,37 @@ async def get_presence_for_all_users(
 
     def get_current_presence_token(self):
         return self._presence_id_gen.get_current_token()
+
+    def _get_active_presence(self, db_conn: Connection):
+        """Fetch non-offline presence from the database so that we can register
+        the appropriate time outs.
+        """
+
+        sql = (
+            "SELECT user_id, state, last_active_ts, last_federation_update_ts,"
+            " last_user_sync_ts, status_msg, currently_active FROM presence_stream"
+            " WHERE state != ?"
+        )
+
+        txn = db_conn.cursor()
+        txn.execute(sql, (PresenceState.OFFLINE,))
+        rows = self.db_pool.cursor_to_dict(txn)
+        txn.close()
+
+        for row in rows:
+            row["currently_active"] = bool(row["currently_active"])
+
+        return [UserPresenceState(**row) for row in rows]
+
+    def take_presence_startup_info(self):
+        active_on_startup = self._presence_on_startup
+        self._presence_on_startup = None
+        return active_on_startup
+
+    def process_replication_rows(self, stream_name, instance_name, token, rows):
+        if stream_name == PresenceStream.NAME:
+            self._presence_id_gen.advance(instance_name, token)
+            for row in rows:
+                self.presence_stream_cache.entity_has_changed(row.user_id, token)
+                self._get_presence_for_user.invalidate((row.user_id,))
+        return super().process_replication_rows(stream_name, instance_name, token, rows)
diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py
index 1e65cb8a04..1a34118f7e 100644
--- a/synapse/storage/databases/main/profile.py
+++ b/synapse/storage/databases/main/profile.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py
index 41f4fe7f95..664c65dac5 100644
--- a/synapse/storage/databases/main/purge_events.py
+++ b/synapse/storage/databases/main/purge_events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,14 +16,14 @@
 from typing import Any, List, Set, Tuple
 
 from synapse.api.errors import SynapseError
-from synapse.storage._base import SQLBaseStore
+from synapse.storage.databases.main import CacheInvalidationWorkerStore
 from synapse.storage.databases.main.state import StateGroupWorkerStore
 from synapse.types import RoomStreamToken
 
 logger = logging.getLogger(__name__)
 
 
-class PurgeEventsStore(StateGroupWorkerStore, SQLBaseStore):
+class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore):
     async def purge_history(
         self, room_id: str, token: str, delete_local_events: bool
     ) -> Set[int]:
@@ -116,7 +115,7 @@ def _purge_history_txn(
         logger.info("[purge] looking for events to delete")
 
         should_delete_expr = "state_key IS NULL"
-        should_delete_params = ()  # type: Tuple[Any, ...]
+        should_delete_params: Tuple[Any, ...] = ()
         if not delete_local_events:
             should_delete_expr += " AND event_id NOT LIKE ?"
 
@@ -204,8 +203,6 @@ def _purge_history_txn(
             "DELETE FROM event_to_state_groups "
             "WHERE event_id IN (SELECT event_id from events_to_purge)"
         )
-        for event_id, _ in event_rows:
-            txn.call_after(self._get_state_group_for_event.invalidate, (event_id,))
 
         # Delete all remote non-state events
         for table in (
@@ -218,6 +215,7 @@ def _purge_history_txn(
             "event_relations",
             "event_search",
             "rejections",
+            "redactions",
         ):
             logger.info("[purge] removing events from %s", table)
 
@@ -284,6 +282,20 @@ def _purge_history_txn(
         # so make sure to keep this actually last.
         txn.execute("DROP TABLE events_to_purge")
 
+        for event_id, should_delete in event_rows:
+            self._invalidate_cache_and_stream(
+                txn, self._get_state_group_for_event, (event_id,)
+            )
+
+            # XXX: This is racy, since have_seen_events could be called between the
+            #    transaction completing and the invalidation running. On the other hand,
+            #    that's no different to calling `have_seen_events` just before the
+            #    event is deleted from the database.
+            if should_delete:
+                self._invalidate_cache_and_stream(
+                    txn, self.have_seen_event, (room_id, event_id)
+                )
+
         logger.info("[purge] done")
 
         return referenced_state_groups
@@ -381,7 +393,6 @@ def _purge_room_txn(self, txn, room_id: str) -> List[int]:
             "room_memberships",
             "room_stats_state",
             "room_stats_current",
-            "room_stats_historical",
             "room_stats_earliest_token",
             "rooms",
             "stream_ordering_to_exterm",
@@ -423,7 +434,11 @@ def _purge_room_txn(self, txn, room_id: str) -> List[int]:
         #       index on them. In any case we should be clearing out 'stream' tables
         #       periodically anyway (#5888)
 
-        # TODO: we could probably usefully do a bunch of cache invalidation here
+        # TODO: we could probably usefully do a bunch more cache invalidation here
+
+        # XXX: as with purge_history, this is racy, but no worse than other races
+        #   that already exist.
+        self._invalidate_cache_and_stream(txn, self.have_seen_event, (room_id,))
 
         logger.info("[purge] done")
 
diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py
index 9e58dc0e6a..a7fb8cd848 100644
--- a/synapse/storage/databases/main/push_rule.py
+++ b/synapse/storage/databases/main/push_rule.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -80,9 +79,9 @@ def __init__(self, database: DatabasePool, db_conn, hs):
         super().__init__(database, db_conn, hs)
 
         if hs.config.worker.worker_app is None:
-            self._push_rules_stream_id_gen = StreamIdGenerator(
-                db_conn, "push_rules_stream", "stream_id"
-            )  # type: Union[StreamIdGenerator, SlavedIdTracker]
+            self._push_rules_stream_id_gen: Union[
+                StreamIdGenerator, SlavedIdTracker
+            ] = StreamIdGenerator(db_conn, "push_rules_stream", "stream_id")
         else:
             self._push_rules_stream_id_gen = SlavedIdTracker(
                 db_conn, "push_rules_stream", "stream_id"
diff --git a/synapse/storage/databases/main/pusher.py b/synapse/storage/databases/main/pusher.py
index c65558c280..b48fe086d4 100644
--- a/synapse/storage/databases/main/pusher.py
+++ b/synapse/storage/databases/main/pusher.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py
index 43c852c96c..edeaacd7a6 100644
--- a/synapse/storage/databases/main/receipts.py
+++ b/synapse/storage/databases/main/receipts.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -461,7 +460,7 @@ def _invalidate_get_users_with_receipts_in_room(
 
     def invalidate_caches_for_receipt(self, room_id, receipt_type, user_id):
         self.get_receipts_for_user.invalidate((user_id, receipt_type))
-        self._get_linearized_receipts_for_room.invalidate_many((room_id,))
+        self._get_linearized_receipts_for_room.invalidate((room_id,))
         self.get_last_receipt_event_id_for_user.invalidate(
             (user_id, room_id, receipt_type)
         )
@@ -660,9 +659,7 @@ def insert_graph_receipt_txn(
         )
         txn.call_after(self.get_receipts_for_user.invalidate, (user_id, receipt_type))
         # FIXME: This shouldn't invalidate the whole cache
-        txn.call_after(
-            self._get_linearized_receipts_for_room.invalidate_many, (room_id,)
-        )
+        txn.call_after(self._get_linearized_receipts_for_room.invalidate, (room_id,))
 
         self.db_pool.simple_delete_txn(
             txn,
diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py
index c439b58db6..1909aaea0a 100644
--- a/synapse/storage/databases/main/registration.py
+++ b/synapse/storage/databases/main/registration.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019,2020 The Matrix.org Foundation C.I.C.
@@ -15,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+import random
 import re
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
@@ -29,7 +29,7 @@
 from synapse.storage.types import Connection, Cursor
 from synapse.storage.util.id_generators import IdGenerator
 from synapse.storage.util.sequence import build_sequence_generator
-from synapse.types import UserID
+from synapse.types import UserID, UserInfo
 from synapse.util.caches.descriptors import cached
 
 if TYPE_CHECKING:
@@ -53,6 +53,9 @@ class TokenLookupResult:
         valid_until_ms: The timestamp the token expires, if any.
         token_owner: The "owner" of the token. This is either the same as the
             user, or a server admin who is logged in as the user.
+        token_used: True if this token was used at least once in a request.
+            This field can be out of date since `get_user_by_access_token` is
+            cached.
     """
 
     user_id = attr.ib(type=str)
@@ -62,6 +65,7 @@ class TokenLookupResult:
     device_id = attr.ib(type=Optional[str], default=None)
     valid_until_ms = attr.ib(type=Optional[int], default=None)
     token_owner = attr.ib(type=str)
+    token_used = attr.ib(type=bool, default=False)
 
     # Make the token owner default to the user ID, which is the common case.
     @token_owner.default
@@ -69,6 +73,29 @@ def _default_token_owner(self):
         return self.user_id
 
 
+@attr.s(frozen=True, slots=True)
+class RefreshTokenLookupResult:
+    """Result of looking up a refresh token."""
+
+    user_id = attr.ib(type=str)
+    """The user this token belongs to."""
+
+    device_id = attr.ib(type=str)
+    """The device associated with this refresh token."""
+
+    token_id = attr.ib(type=int)
+    """The ID of this refresh token."""
+
+    next_token_id = attr.ib(type=Optional[int])
+    """The ID of the refresh token which replaced this one."""
+
+    has_next_refresh_token_been_refreshed = attr.ib(type=bool)
+    """True if the next refresh token was used for another refresh."""
+
+    has_next_access_token_been_used = attr.ib(type=bool)
+    """True if the next access token was already used at least once."""
+
+
 class RegistrationWorkerStore(CacheInvalidationWorkerStore):
     def __init__(
         self,
@@ -92,11 +119,17 @@ def __init__(
             id_column=None,
         )
 
-        self._account_validity_enabled = hs.config.account_validity_enabled
+        self._account_validity_enabled = (
+            hs.config.account_validity.account_validity_enabled
+        )
+        self._account_validity_period = None
+        self._account_validity_startup_job_max_delta = None
         if self._account_validity_enabled:
-            self._account_validity_period = hs.config.account_validity_period
+            self._account_validity_period = (
+                hs.config.account_validity.account_validity_period
+            )
             self._account_validity_startup_job_max_delta = (
-                hs.config.account_validity_startup_job_max_delta
+                hs.config.account_validity.account_validity_startup_job_max_delta
             )
 
             if hs.config.run_background_tasks:
@@ -113,6 +146,7 @@ def __init__(
 
     @cached()
     async def get_user_by_id(self, user_id: str) -> Optional[Dict[str, Any]]:
+        """Deprecated: use get_userinfo_by_id instead"""
         return await self.db_pool.simple_select_one(
             table="users",
             keyvalues={"name": user_id},
@@ -133,6 +167,33 @@ async def get_user_by_id(self, user_id: str) -> Optional[Dict[str, Any]]:
             desc="get_user_by_id",
         )
 
+    async def get_userinfo_by_id(self, user_id: str) -> Optional[UserInfo]:
+        """Get a UserInfo object for a user by user ID.
+
+        Note! Currently uses the cache of `get_user_by_id`. Once that deprecated method is removed,
+        this method should be cached.
+
+        Args:
+             user_id: The user to fetch user info for.
+        Returns:
+            `UserInfo` object if user found, otherwise `None`.
+        """
+        user_data = await self.get_user_by_id(user_id)
+        if not user_data:
+            return None
+        return UserInfo(
+            appservice_id=user_data["appservice_id"],
+            consent_server_notice_sent=user_data["consent_server_notice_sent"],
+            consent_version=user_data["consent_version"],
+            creation_ts=user_data["creation_ts"],
+            is_admin=bool(user_data["admin"]),
+            is_deactivated=bool(user_data["deactivated"]),
+            is_guest=bool(user_data["is_guest"]),
+            is_shadow_banned=bool(user_data["shadow_banned"]),
+            user_id=UserID.from_string(user_data["name"]),
+            user_type=user_data["user_type"],
+        )
+
     async def is_trial_user(self, user_id: str) -> bool:
         """Checks if user is in the "trial" period, i.e. within the first
         N days of registration defined by `mau_trial_days` config
@@ -515,7 +576,8 @@ def _query_for_auth(self, txn, token: str) -> Optional[TokenLookupResult]:
                 access_tokens.id as token_id,
                 access_tokens.device_id,
                 access_tokens.valid_until_ms,
-                access_tokens.user_id as token_owner
+                access_tokens.user_id as token_owner,
+                access_tokens.used as token_used
             FROM users
             INNER JOIN access_tokens on users.name = COALESCE(puppets_user_id, access_tokens.user_id)
             WHERE token = ?
@@ -523,8 +585,15 @@ def _query_for_auth(self, txn, token: str) -> Optional[TokenLookupResult]:
 
         txn.execute(sql, (token,))
         rows = self.db_pool.cursor_to_dict(txn)
+
         if rows:
-            return TokenLookupResult(**rows[0])
+            row = rows[0]
+
+            # This field is nullable, ensure it comes out as a boolean
+            if row["token_used"] is None:
+                row["token_used"] = False
+
+            return TokenLookupResult(**row)
 
         return None
 
@@ -610,6 +679,28 @@ async def record_user_external_id(
             desc="record_user_external_id",
         )
 
+    async def remove_user_external_id(
+        self, auth_provider: str, external_id: str, user_id: str
+    ) -> None:
+        """Remove a mapping from an external user id to a mxid
+
+        If the mapping is not found, this method does nothing.
+
+        Args:
+            auth_provider: identifier for the remote auth provider
+            external_id: id on that system
+            user_id: complete mxid that it is mapped to
+        """
+        await self.db_pool.simple_delete(
+            table="user_external_ids",
+            keyvalues={
+                "auth_provider": auth_provider,
+                "external_id": external_id,
+                "user_id": user_id,
+            },
+            desc="remove_user_external_id",
+        )
+
     async def get_user_by_external_id(
         self, auth_provider: str, external_id: str
     ) -> Optional[str]:
@@ -1072,7 +1163,7 @@ def set_expiration_date_for_user_txn(self, txn, user_id, use_delta=False):
         expiration_ts = now_ms + self._account_validity_period
 
         if use_delta:
-            expiration_ts = self.rand.randrange(
+            expiration_ts = random.randrange(
                 expiration_ts - self._account_validity_startup_job_max_delta,
                 expiration_ts,
             )
@@ -1146,6 +1237,111 @@ async def update_access_token_last_validated(self, token_id: int) -> None:
             desc="update_access_token_last_validated",
         )
 
+    @cached()
+    async def mark_access_token_as_used(self, token_id: int) -> None:
+        """
+        Mark the access token as used, which invalidates the refresh token used
+        to obtain it.
+
+        Because get_user_by_access_token is cached, this function might be
+        called multiple times for the same token, effectively doing unnecessary
+        SQL updates. Because updating the `used` field only goes one way (from
+        False to True) it is safe to cache this function as well to avoid this
+        issue.
+
+        Args:
+            token_id: The ID of the access token to update.
+        Raises:
+            StoreError if there was a problem updating this.
+        """
+        await self.db_pool.simple_update_one(
+            "access_tokens",
+            {"id": token_id},
+            {"used": True},
+            desc="mark_access_token_as_used",
+        )
+
+    async def lookup_refresh_token(
+        self, token: str
+    ) -> Optional[RefreshTokenLookupResult]:
+        """Lookup a refresh token with hints about its validity."""
+
+        def _lookup_refresh_token_txn(txn) -> Optional[RefreshTokenLookupResult]:
+            txn.execute(
+                """
+                SELECT
+                    rt.id token_id,
+                    rt.user_id,
+                    rt.device_id,
+                    rt.next_token_id,
+                    (nrt.next_token_id IS NOT NULL) has_next_refresh_token_been_refreshed,
+                    at.used has_next_access_token_been_used
+                FROM refresh_tokens rt
+                LEFT JOIN refresh_tokens nrt ON rt.next_token_id = nrt.id
+                LEFT JOIN access_tokens at ON at.refresh_token_id = nrt.id
+                WHERE rt.token = ?
+            """,
+                (token,),
+            )
+            row = txn.fetchone()
+
+            if row is None:
+                return None
+
+            return RefreshTokenLookupResult(
+                token_id=row[0],
+                user_id=row[1],
+                device_id=row[2],
+                next_token_id=row[3],
+                has_next_refresh_token_been_refreshed=row[4],
+                # This column is nullable, ensure it's a boolean
+                has_next_access_token_been_used=(row[5] or False),
+            )
+
+        return await self.db_pool.runInteraction(
+            "lookup_refresh_token", _lookup_refresh_token_txn
+        )
+
+    async def replace_refresh_token(self, token_id: int, next_token_id: int) -> None:
+        """
+        Set the successor of a refresh token, removing the existing successor
+        if any.
+
+        Args:
+            token_id: ID of the refresh token to update.
+            next_token_id: ID of its successor.
+        """
+
+        def _replace_refresh_token_txn(txn) -> None:
+            # First check if there was an existing refresh token
+            old_next_token_id = self.db_pool.simple_select_one_onecol_txn(
+                txn,
+                "refresh_tokens",
+                {"id": token_id},
+                "next_token_id",
+                allow_none=True,
+            )
+
+            self.db_pool.simple_update_one_txn(
+                txn,
+                "refresh_tokens",
+                {"id": token_id},
+                {"next_token_id": next_token_id},
+            )
+
+            # Delete the old "next" token if it exists. This should cascade and
+            # delete the associated access_token
+            if old_next_token_id is not None:
+                self.db_pool.simple_delete_one_txn(
+                    txn,
+                    "refresh_tokens",
+                    {"id": old_next_token_id},
+                )
+
+        await self.db_pool.runInteraction(
+            "replace_refresh_token", _replace_refresh_token_txn
+        )
+
 
 class RegistrationBackgroundUpdateStore(RegistrationWorkerStore):
     def __init__(
@@ -1337,6 +1533,7 @@ def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer"
         self._ignore_unknown_session_error = hs.config.request_token_inhibit_3pid_errors
 
         self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id")
+        self._refresh_tokens_id_gen = IdGenerator(db_conn, "refresh_tokens", "id")
 
     async def add_access_token_to_user(
         self,
@@ -1345,14 +1542,18 @@ async def add_access_token_to_user(
         device_id: Optional[str],
         valid_until_ms: Optional[int],
         puppets_user_id: Optional[str] = None,
+        refresh_token_id: Optional[int] = None,
     ) -> int:
         """Adds an access token for the given user.
 
         Args:
             user_id: The user ID.
             token: The new access token to add.
-            device_id: ID of the device to associate with the access token
+            device_id: ID of the device to associate with the access token.
             valid_until_ms: when the token is valid until. None for no expiry.
+            puppets_user_id
+            refresh_token_id: ID of the refresh token generated alongside this
+                access token.
         Raises:
             StoreError if there was a problem adding this.
         Returns:
@@ -1371,12 +1572,47 @@ async def add_access_token_to_user(
                 "valid_until_ms": valid_until_ms,
                 "puppets_user_id": puppets_user_id,
                 "last_validated": now,
+                "refresh_token_id": refresh_token_id,
+                "used": False,
             },
             desc="add_access_token_to_user",
         )
 
         return next_id
 
+    async def add_refresh_token_to_user(
+        self,
+        user_id: str,
+        token: str,
+        device_id: Optional[str],
+    ) -> int:
+        """Adds a refresh token for the given user.
+
+        Args:
+            user_id: The user ID.
+            token: The new access token to add.
+            device_id: ID of the device to associate with the refresh token.
+        Raises:
+            StoreError if there was a problem adding this.
+        Returns:
+            The token ID
+        """
+        next_id = self._refresh_tokens_id_gen.get_next()
+
+        await self.db_pool.simple_insert(
+            "refresh_tokens",
+            {
+                "id": next_id,
+                "user_id": user_id,
+                "device_id": device_id,
+                "token": token,
+                "next_token_id": None,
+            },
+            desc="add_refresh_token_to_user",
+        )
+
+        return next_id
+
     def _set_device_for_access_token_txn(self, txn, token: str, device_id: str) -> str:
         old_device_id = self.db_pool.simple_select_one_onecol_txn(
             txn, "access_tokens", {"token": token}, "device_id"
@@ -1619,7 +1855,7 @@ async def user_delete_access_tokens(
         device_id: Optional[str] = None,
     ) -> List[Tuple[str, int, Optional[str]]]:
         """
-        Invalidate access tokens belonging to a user
+        Invalidate access and refresh tokens belonging to a user
 
         Args:
             user_id: ID of user the tokens belong to
@@ -1638,8 +1874,14 @@ def f(txn):
 
             items = keyvalues.items()
             where_clause = " AND ".join(k + " = ?" for k, _ in items)
-            values = [v for _, v in items]  # type: List[Union[str, int]]
+            values: List[Union[str, int]] = [v for _, v in items]
+            # Conveniently, refresh_tokens and access_tokens both use the user_id and device_id fields. Only caveat
+            # is the `except_token_id` param that is tricky to get right, so for now we're just using the same where
+            # clause and values before we handle that. This seems to be only used in the "set password" handler.
+            refresh_where_clause = where_clause
+            refresh_values = values.copy()
             if except_token_id:
+                # TODO: support that for refresh tokens
                 where_clause += " AND id != ?"
                 values.append(except_token_id)
 
@@ -1657,6 +1899,11 @@ def f(txn):
 
             txn.execute("DELETE FROM access_tokens WHERE %s" % where_clause, values)
 
+            txn.execute(
+                "DELETE FROM refresh_tokens WHERE %s" % refresh_where_clause,
+                refresh_values,
+            )
+
             return tokens_and_devices
 
         return await self.db_pool.runInteraction("user_delete_access_tokens", f)
@@ -1673,6 +1920,14 @@ def f(txn):
 
         await self.db_pool.runInteraction("delete_access_token", f)
 
+    async def delete_refresh_token(self, refresh_token: str) -> None:
+        def f(txn):
+            self.db_pool.simple_delete_one_txn(
+                txn, table="refresh_tokens", keyvalues={"token": refresh_token}
+            )
+
+        await self.db_pool.runInteraction("delete_refresh_token", f)
+
     async def add_user_pending_deactivation(self, user_id: str) -> None:
         """
         Adds a user to the table of users who need to be parted from all the rooms they're
diff --git a/synapse/storage/databases/main/rejections.py b/synapse/storage/databases/main/rejections.py
index 1e361aaa9a..167318b314 100644
--- a/synapse/storage/databases/main/rejections.py
+++ b/synapse/storage/databases/main/rejections.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py
index 5cd61547f7..2bbf6d6a95 100644
--- a/synapse/storage/databases/main/relations.py
+++ b/synapse/storage/databases/main/relations.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py
index 8db6f1396a..10ef734d8d 100644
--- a/synapse/storage/databases/main/room.py
+++ b/synapse/storage/databases/main/room.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
@@ -25,6 +24,7 @@
 from synapse.storage._base import SQLBaseStore, db_to_json
 from synapse.storage.database import DatabasePool, LoggingTransaction
 from synapse.storage.databases.main.search import SearchStore
+from synapse.storage.types import Cursor
 from synapse.types import JsonDict, ThirdPartyInstanceID
 from synapse.util import json_encoder
 from synapse.util.caches.descriptors import cached
@@ -72,6 +72,40 @@ def __init__(self, database: DatabasePool, db_conn, hs):
 
         self.config = hs.config
 
+    async def store_room(
+        self,
+        room_id: str,
+        room_creator_user_id: str,
+        is_public: bool,
+        room_version: RoomVersion,
+    ):
+        """Stores a room.
+
+        Args:
+            room_id: The desired room ID, can be None.
+            room_creator_user_id: The user ID of the room creator.
+            is_public: True to indicate that this room should appear in
+                public room lists.
+            room_version: The version of the room
+        Raises:
+            StoreError if the room could not be stored.
+        """
+        try:
+            await self.db_pool.simple_insert(
+                "rooms",
+                {
+                    "room_id": room_id,
+                    "creator": room_creator_user_id,
+                    "is_public": is_public,
+                    "room_version": room_version.identifier,
+                    "has_auth_chain_index": True,
+                },
+                desc="store_room",
+            )
+        except Exception as e:
+            logger.error("store_room with room_id=%s failed: %s", room_id, e)
+            raise StoreError(500, "Problem creating room.")
+
     async def get_room(self, room_id: str) -> dict:
         """Retrieve a room.
 
@@ -379,7 +413,7 @@ async def get_rooms_paginate(
         self,
         start: int,
         limit: int,
-        order_by: RoomSortOrder,
+        order_by: str,
         reverse_order: bool,
         search_term: Optional[str],
     ) -> Tuple[List[Dict[str, Any]], int]:
@@ -541,13 +575,11 @@ def _get_rooms_paginate_txn(txn):
         )
 
     @cached(max_entries=10000)
-    async def get_ratelimit_for_user(self, user_id):
-        """Check if there are any overrides for ratelimiting for the given
-        user
+    async def get_ratelimit_for_user(self, user_id: str) -> Optional[RatelimitOverride]:
+        """Check if there are any overrides for ratelimiting for the given user
 
         Args:
-            user_id (str)
-
+            user_id: user ID of the user
         Returns:
             RatelimitOverride if there is an override, else None. If the contents
             of RatelimitOverride are None or 0 then ratelimitng has been
@@ -569,6 +601,62 @@ async def get_ratelimit_for_user(self, user_id):
         else:
             return None
 
+    async def set_ratelimit_for_user(
+        self, user_id: str, messages_per_second: int, burst_count: int
+    ) -> None:
+        """Sets whether a user is set an overridden ratelimit.
+        Args:
+            user_id: user ID of the user
+            messages_per_second: The number of actions that can be performed in a second.
+            burst_count: How many actions that can be performed before being limited.
+        """
+
+        def set_ratelimit_txn(txn):
+            self.db_pool.simple_upsert_txn(
+                txn,
+                table="ratelimit_override",
+                keyvalues={"user_id": user_id},
+                values={
+                    "messages_per_second": messages_per_second,
+                    "burst_count": burst_count,
+                },
+            )
+
+            self._invalidate_cache_and_stream(
+                txn, self.get_ratelimit_for_user, (user_id,)
+            )
+
+        await self.db_pool.runInteraction("set_ratelimit", set_ratelimit_txn)
+
+    async def delete_ratelimit_for_user(self, user_id: str) -> None:
+        """Delete an overridden ratelimit for a user.
+        Args:
+            user_id: user ID of the user
+        """
+
+        def delete_ratelimit_txn(txn):
+            row = self.db_pool.simple_select_one_txn(
+                txn,
+                table="ratelimit_override",
+                keyvalues={"user_id": user_id},
+                retcols=["user_id"],
+                allow_none=True,
+            )
+
+            if not row:
+                return
+
+            # They are there, delete them.
+            self.db_pool.simple_delete_one_txn(
+                txn, "ratelimit_override", keyvalues={"user_id": user_id}
+            )
+
+            self._invalidate_cache_and_stream(
+                txn, self.get_ratelimit_for_user, (user_id,)
+            )
+
+        await self.db_pool.runInteraction("delete_ratelimit", delete_ratelimit_txn)
+
     @cached()
     async def get_retention_policy_for_room(self, room_id):
         """Get the retention policy for a given room.
@@ -736,14 +824,15 @@ async def quarantine_media_by_id(
         self,
         server_name: str,
         media_id: str,
-        quarantined_by: str,
+        quarantined_by: Optional[str],
     ) -> int:
-        """quarantines a single local or remote media id
+        """quarantines or unquarantines a single local or remote media id
 
         Args:
             server_name: The name of the server that holds this media
             media_id: The ID of the media to be quarantined
             quarantined_by: The user ID that initiated the quarantine request
+                If it is `None` media will be removed from quarantine
         """
         logger.info("Quarantining media: %s/%s", server_name, media_id)
         is_local = server_name == self.config.server_name
@@ -810,9 +899,9 @@ def _quarantine_media_txn(
         txn,
         local_mxcs: List[str],
         remote_mxcs: List[Tuple[str, str]],
-        quarantined_by: str,
+        quarantined_by: Optional[str],
     ) -> int:
-        """Quarantine local and remote media items
+        """Quarantine and unquarantine local and remote media items
 
         Args:
             txn (cursor)
@@ -820,18 +909,27 @@ def _quarantine_media_txn(
             remote_mxcs: A list of (remote server, media id) tuples representing
                 remote mxc URLs
             quarantined_by: The ID of the user who initiated the quarantine request
+                If it is `None` media will be removed from quarantine
         Returns:
             The total number of media items quarantined
         """
+
         # Update all the tables to set the quarantined_by flag
-        txn.executemany(
-            """
+        sql = """
             UPDATE local_media_repository
             SET quarantined_by = ?
-            WHERE media_id = ? AND safe_from_quarantine = ?
-        """,
-            ((quarantined_by, media_id, False) for media_id in local_mxcs),
-        )
+            WHERE media_id = ?
+        """
+
+        # set quarantine
+        if quarantined_by is not None:
+            sql += "AND safe_from_quarantine = ?"
+            rows = [(quarantined_by, media_id, False) for media_id in local_mxcs]
+        # remove from quarantine
+        else:
+            rows = [(quarantined_by, media_id) for media_id in local_mxcs]
+
+        txn.executemany(sql, rows)
         # Note that a rowcount of -1 can be used to indicate no rows were affected.
         total_media_quarantined = txn.rowcount if txn.rowcount > 0 else 0
 
@@ -847,55 +945,6 @@ def _quarantine_media_txn(
 
         return total_media_quarantined
 
-    async def get_all_new_public_rooms(
-        self, instance_name: str, last_id: int, current_id: int, limit: int
-    ) -> Tuple[List[Tuple[int, tuple]], int, bool]:
-        """Get updates for public rooms replication stream.
-
-        Args:
-            instance_name: The writer we want to fetch updates from. Unused
-                here since there is only ever one writer.
-            last_id: The token to fetch updates from. Exclusive.
-            current_id: The token to fetch updates up to. Inclusive.
-            limit: The requested limit for the number of rows to return. The
-                function may return more or fewer rows.
-
-        Returns:
-            A tuple consisting of: the updates, a token to use to fetch
-            subsequent updates, and whether we returned fewer rows than exists
-            between the requested tokens due to the limit.
-
-            The token returned can be used in a subsequent call to this
-            function to get further updatees.
-
-            The updates are a list of 2-tuples of stream ID and the row data
-        """
-        if last_id == current_id:
-            return [], current_id, False
-
-        def get_all_new_public_rooms(txn):
-            sql = """
-                SELECT stream_id, room_id, visibility, appservice_id, network_id
-                FROM public_room_list_stream
-                WHERE stream_id > ? AND stream_id <= ?
-                ORDER BY stream_id ASC
-                LIMIT ?
-            """
-
-            txn.execute(sql, (last_id, current_id, limit))
-            updates = [(row[0], row[1:]) for row in txn]
-            limited = False
-            upto_token = current_id
-            if len(updates) >= limit:
-                upto_token = updates[-1][0]
-                limited = True
-
-            return updates, upto_token, limited
-
-        return await self.db_pool.runInteraction(
-            "get_all_new_public_rooms", get_all_new_public_rooms
-        )
-
     async def get_rooms_for_retention_period_in_range(
         self, min_ms: Optional[int], max_ms: Optional[int], include_null: bool = False
     ) -> Dict[str, dict]:
@@ -980,10 +1029,22 @@ def get_rooms_for_retention_period_in_range_txn(txn):
         )
 
 
-class RoomBackgroundUpdateStore(SQLBaseStore):
+class _BackgroundUpdates:
     REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory"
     ADD_ROOMS_ROOM_VERSION_COLUMN = "add_rooms_room_version_column"
+    POPULATE_ROOM_DEPTH_MIN_DEPTH2 = "populate_room_depth_min_depth2"
+    REPLACE_ROOM_DEPTH_MIN_DEPTH = "replace_room_depth_min_depth"
+
+
+_REPLACE_ROOM_DEPTH_SQL_COMMANDS = (
+    "DROP TRIGGER populate_min_depth2_trigger ON room_depth",
+    "DROP FUNCTION populate_min_depth2()",
+    "ALTER TABLE room_depth DROP COLUMN min_depth",
+    "ALTER TABLE room_depth RENAME COLUMN min_depth2 TO min_depth",
+)
 
+
+class RoomBackgroundUpdateStore(SQLBaseStore):
     def __init__(self, database: DatabasePool, db_conn, hs):
         super().__init__(database, db_conn, hs)
 
@@ -995,15 +1056,25 @@ def __init__(self, database: DatabasePool, db_conn, hs):
         )
 
         self.db_pool.updates.register_background_update_handler(
-            self.REMOVE_TOMESTONED_ROOMS_BG_UPDATE,
+            _BackgroundUpdates.REMOVE_TOMESTONED_ROOMS_BG_UPDATE,
             self._remove_tombstoned_rooms_from_directory,
         )
 
         self.db_pool.updates.register_background_update_handler(
-            self.ADD_ROOMS_ROOM_VERSION_COLUMN,
+            _BackgroundUpdates.ADD_ROOMS_ROOM_VERSION_COLUMN,
             self._background_add_rooms_room_version_column,
         )
 
+        # BG updates to change the type of room_depth.min_depth
+        self.db_pool.updates.register_background_update_handler(
+            _BackgroundUpdates.POPULATE_ROOM_DEPTH_MIN_DEPTH2,
+            self._background_populate_room_depth_min_depth2,
+        )
+        self.db_pool.updates.register_background_update_handler(
+            _BackgroundUpdates.REPLACE_ROOM_DEPTH_MIN_DEPTH,
+            self._background_replace_room_depth_min_depth,
+        )
+
     async def _background_insert_retention(self, progress, batch_size):
         """Retrieves a list of all rooms within a range and inserts an entry for each of
         them into the room_retention table.
@@ -1122,7 +1193,9 @@ def _background_add_rooms_room_version_column_txn(txn: LoggingTransaction):
                 new_last_room_id = room_id
 
             self.db_pool.updates._background_update_progress_txn(
-                txn, self.ADD_ROOMS_ROOM_VERSION_COLUMN, {"room_id": new_last_room_id}
+                txn,
+                _BackgroundUpdates.ADD_ROOMS_ROOM_VERSION_COLUMN,
+                {"room_id": new_last_room_id},
             )
 
             return False
@@ -1134,7 +1207,7 @@ def _background_add_rooms_room_version_column_txn(txn: LoggingTransaction):
 
         if end:
             await self.db_pool.updates._end_background_update(
-                self.ADD_ROOMS_ROOM_VERSION_COLUMN
+                _BackgroundUpdates.ADD_ROOMS_ROOM_VERSION_COLUMN
             )
 
         return batch_size
@@ -1173,7 +1246,7 @@ def _get_rooms(txn):
 
         if not rooms:
             await self.db_pool.updates._end_background_update(
-                self.REMOVE_TOMESTONED_ROOMS_BG_UPDATE
+                _BackgroundUpdates.REMOVE_TOMESTONED_ROOMS_BG_UPDATE
             )
             return 0
 
@@ -1182,7 +1255,7 @@ def _get_rooms(txn):
             await self.set_room_is_public(room_id, False)
 
         await self.db_pool.updates._background_update_progress(
-            self.REMOVE_TOMESTONED_ROOMS_BG_UPDATE, {"room_id": rooms[-1]}
+            _BackgroundUpdates.REMOVE_TOMESTONED_ROOMS_BG_UPDATE, {"room_id": rooms[-1]}
         )
 
         return len(rooms)
@@ -1226,6 +1299,71 @@ async def has_auth_chain_index(self, room_id: str) -> bool:
 
         return max_ordering is None
 
+    async def _background_populate_room_depth_min_depth2(
+        self, progress: JsonDict, batch_size: int
+    ) -> int:
+        """Populate room_depth.min_depth2
+
+        This is to deal with the fact that min_depth was initially created as a
+        32-bit integer field.
+        """
+
+        def process(txn: Cursor) -> int:
+            last_room = progress.get("last_room", "")
+            txn.execute(
+                """
+                UPDATE room_depth SET min_depth2=min_depth
+                WHERE room_id IN (
+                   SELECT room_id FROM room_depth WHERE room_id > ?
+                   ORDER BY room_id LIMIT ?
+                )
+                RETURNING room_id;
+                """,
+                (last_room, batch_size),
+            )
+            row_count = txn.rowcount
+            if row_count == 0:
+                return 0
+            last_room = max(row[0] for row in txn)
+            logger.info("populated room_depth up to %s", last_room)
+
+            self.db_pool.updates._background_update_progress_txn(
+                txn,
+                _BackgroundUpdates.POPULATE_ROOM_DEPTH_MIN_DEPTH2,
+                {"last_room": last_room},
+            )
+            return row_count
+
+        result = await self.db_pool.runInteraction(
+            "_background_populate_min_depth2", process
+        )
+
+        if result != 0:
+            return result
+
+        await self.db_pool.updates._end_background_update(
+            _BackgroundUpdates.POPULATE_ROOM_DEPTH_MIN_DEPTH2
+        )
+        return 0
+
+    async def _background_replace_room_depth_min_depth(
+        self, progress: JsonDict, batch_size: int
+    ) -> int:
+        """Drop the old 'min_depth' column and rename 'min_depth2' into its place."""
+
+        def process(txn: Cursor) -> None:
+            for sql in _REPLACE_ROOM_DEPTH_SQL_COMMANDS:
+                logger.info("completing room_depth migration: %s", sql)
+                txn.execute(sql)
+
+        await self.db_pool.runInteraction("_background_replace_room_depth", process)
+
+        await self.db_pool.updates._end_background_update(
+            _BackgroundUpdates.REPLACE_ROOM_DEPTH_MIN_DEPTH,
+        )
+
+        return 0
+
 
 class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore):
     def __init__(self, database: DatabasePool, db_conn, hs):
@@ -1259,57 +1397,6 @@ async def upsert_room_on_join(self, room_id: str, room_version: RoomVersion):
             lock=False,
         )
 
-    async def store_room(
-        self,
-        room_id: str,
-        room_creator_user_id: str,
-        is_public: bool,
-        room_version: RoomVersion,
-    ):
-        """Stores a room.
-
-        Args:
-            room_id: The desired room ID, can be None.
-            room_creator_user_id: The user ID of the room creator.
-            is_public: True to indicate that this room should appear in
-                public room lists.
-            room_version: The version of the room
-        Raises:
-            StoreError if the room could not be stored.
-        """
-        try:
-
-            def store_room_txn(txn, next_id):
-                self.db_pool.simple_insert_txn(
-                    txn,
-                    "rooms",
-                    {
-                        "room_id": room_id,
-                        "creator": room_creator_user_id,
-                        "is_public": is_public,
-                        "room_version": room_version.identifier,
-                        "has_auth_chain_index": True,
-                    },
-                )
-                if is_public:
-                    self.db_pool.simple_insert_txn(
-                        txn,
-                        table="public_room_list_stream",
-                        values={
-                            "stream_id": next_id,
-                            "room_id": room_id,
-                            "visibility": is_public,
-                        },
-                    )
-
-            async with self._public_room_id_gen.get_next() as next_id:
-                await self.db_pool.runInteraction(
-                    "store_room_txn", store_room_txn, next_id
-                )
-        except Exception as e:
-            logger.error("store_room with room_id=%s failed: %s", room_id, e)
-            raise StoreError(500, "Problem creating room.")
-
     async def maybe_store_room_on_outlier_membership(
         self, room_id: str, room_version: RoomVersion
     ):
@@ -1338,49 +1425,14 @@ async def maybe_store_room_on_outlier_membership(
             lock=False,
         )
 
-    async def set_room_is_public(self, room_id, is_public):
-        def set_room_is_public_txn(txn, next_id):
-            self.db_pool.simple_update_one_txn(
-                txn,
-                table="rooms",
-                keyvalues={"room_id": room_id},
-                updatevalues={"is_public": is_public},
-            )
-
-            entries = self.db_pool.simple_select_list_txn(
-                txn,
-                table="public_room_list_stream",
-                keyvalues={
-                    "room_id": room_id,
-                    "appservice_id": None,
-                    "network_id": None,
-                },
-                retcols=("stream_id", "visibility"),
-            )
-
-            entries.sort(key=lambda r: r["stream_id"])
-
-            add_to_stream = True
-            if entries:
-                add_to_stream = bool(entries[-1]["visibility"]) != is_public
-
-            if add_to_stream:
-                self.db_pool.simple_insert_txn(
-                    txn,
-                    table="public_room_list_stream",
-                    values={
-                        "stream_id": next_id,
-                        "room_id": room_id,
-                        "visibility": is_public,
-                        "appservice_id": None,
-                        "network_id": None,
-                    },
-                )
+    async def set_room_is_public(self, room_id: str, is_public: bool) -> None:
+        await self.db_pool.simple_update_one(
+            table="rooms",
+            keyvalues={"room_id": room_id},
+            updatevalues={"is_public": is_public},
+            desc="set_room_is_public",
+        )
 
-        async with self._public_room_id_gen.get_next() as next_id:
-            await self.db_pool.runInteraction(
-                "set_room_is_public", set_room_is_public_txn, next_id
-            )
         self.hs.get_notifier().on_new_replication_data()
 
     async def set_room_is_public_appservice(
@@ -1401,68 +1453,33 @@ async def set_room_is_public_appservice(
                 list.
         """
 
-        def set_room_is_public_appservice_txn(txn, next_id):
-            if is_public:
-                try:
-                    self.db_pool.simple_insert_txn(
-                        txn,
-                        table="appservice_room_list",
-                        values={
-                            "appservice_id": appservice_id,
-                            "network_id": network_id,
-                            "room_id": room_id,
-                        },
-                    )
-                except self.database_engine.module.IntegrityError:
-                    # We've already inserted, nothing to do.
-                    return
-            else:
-                self.db_pool.simple_delete_txn(
-                    txn,
-                    table="appservice_room_list",
-                    keyvalues={
-                        "appservice_id": appservice_id,
-                        "network_id": network_id,
-                        "room_id": room_id,
-                    },
-                )
-
-            entries = self.db_pool.simple_select_list_txn(
-                txn,
-                table="public_room_list_stream",
+        if is_public:
+            await self.db_pool.simple_upsert(
+                table="appservice_room_list",
                 keyvalues={
+                    "appservice_id": appservice_id,
+                    "network_id": network_id,
                     "room_id": room_id,
+                },
+                values={},
+                insertion_values={
                     "appservice_id": appservice_id,
                     "network_id": network_id,
+                    "room_id": room_id,
                 },
-                retcols=("stream_id", "visibility"),
+                desc="set_room_is_public_appservice_true",
             )
-
-            entries.sort(key=lambda r: r["stream_id"])
-
-            add_to_stream = True
-            if entries:
-                add_to_stream = bool(entries[-1]["visibility"]) != is_public
-
-            if add_to_stream:
-                self.db_pool.simple_insert_txn(
-                    txn,
-                    table="public_room_list_stream",
-                    values={
-                        "stream_id": next_id,
-                        "room_id": room_id,
-                        "visibility": is_public,
-                        "appservice_id": appservice_id,
-                        "network_id": network_id,
-                    },
-                )
-
-        async with self._public_room_id_gen.get_next() as next_id:
-            await self.db_pool.runInteraction(
-                "set_room_is_public_appservice",
-                set_room_is_public_appservice_txn,
-                next_id,
+        else:
+            await self.db_pool.simple_delete(
+                table="appservice_room_list",
+                keyvalues={
+                    "appservice_id": appservice_id,
+                    "network_id": network_id,
+                    "room_id": room_id,
+                },
+                desc="set_room_is_public_appservice_false",
             )
+
         self.hs.get_notifier().on_new_replication_data()
 
     async def add_event_report(
@@ -1470,7 +1487,7 @@ async def add_event_report(
         room_id: str,
         event_id: str,
         user_id: str,
-        reason: str,
+        reason: Optional[str],
         content: JsonDict,
         received_ts: int,
     ) -> None:
@@ -1655,9 +1672,6 @@ def _get_event_reports_paginate_txn(txn):
             "get_event_reports_paginate", _get_event_reports_paginate_txn
         )
 
-    def get_current_public_room_stream_id(self):
-        return self._public_room_id_gen.get_current_token()
-
     async def block_room(self, room_id: str, user_id: str) -> None:
         """Marks the room as blocked. Can be called multiple times.
 
diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py
index a9216ca9ae..e8157ba3d4 100644
--- a/synapse/storage/databases/main/roommember.py
+++ b/synapse/storage/databases/main/roommember.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -14,7 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import TYPE_CHECKING, Dict, FrozenSet, Iterable, List, Optional, Set, Tuple
+from typing import (
+    TYPE_CHECKING,
+    Collection,
+    Dict,
+    FrozenSet,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
+
+import attr
 
 from synapse.api.constants import EventTypes, Membership
 from synapse.events import EventBase
@@ -34,7 +46,7 @@
     ProfileInfo,
     RoomsForUser,
 )
-from synapse.types import Collection, PersistedEventPosition, get_domain_from_id
+from synapse.types import PersistedEventPosition, StateMap, get_domain_from_id
 from synapse.util.async_helpers import Linearizer
 from synapse.util.caches import intern_string
 from synapse.util.caches.descriptors import _CacheContext, cached, cachedList
@@ -54,6 +66,10 @@ class RoomMemberWorkerStore(EventsWorkerStore):
     def __init__(self, database: DatabasePool, db_conn, hs):
         super().__init__(database, db_conn, hs)
 
+        # Used by `_get_joined_hosts` to ensure only one thing mutates the cache
+        # at a time. Keyed by room_id.
+        self._joined_host_linearizer = Linearizer("_JoinedHostsCache")
+
         # Is the current_state_events.membership up to date? Or is the
         # background update still running?
         self._current_state_events_membership_up_to_date = False
@@ -74,7 +90,7 @@ def __init__(self, database: DatabasePool, db_conn, hs):
                 60 * 1000,
             )
             self.hs.get_clock().call_later(
-                1000,
+                1,
                 self._count_known_servers,
             )
             LaterGauge(
@@ -174,6 +190,37 @@ def get_users_in_room_txn(self, txn, room_id: str) -> List[str]:
         txn.execute(sql, (room_id, Membership.JOIN))
         return [r[0] for r in txn]
 
+    @cached(max_entries=100000, iterable=True)
+    async def get_users_in_room_with_profiles(
+        self, room_id: str
+    ) -> Dict[str, ProfileInfo]:
+        """Get a mapping from user ID to profile information for all users in a given room.
+
+        Args:
+            room_id: The ID of the room to retrieve the users of.
+
+        Returns:
+            A mapping from user ID to ProfileInfo.
+        """
+
+        def _get_users_in_room_with_profiles(txn) -> Dict[str, ProfileInfo]:
+            sql = """
+                SELECT state_key, display_name, avatar_url FROM room_memberships as m
+                INNER JOIN current_state_events as c
+                ON m.event_id = c.event_id
+                AND m.room_id = c.room_id
+                AND m.user_id = c.state_key
+                WHERE c.type = 'm.room.member' AND c.room_id = ? AND m.membership = ?
+            """
+            txn.execute(sql, (room_id, Membership.JOIN))
+
+            return {r[0]: ProfileInfo(display_name=r[1], avatar_url=r[2]) for r in txn}
+
+        return await self.db_pool.runInteraction(
+            "get_users_in_room_with_profiles",
+            _get_users_in_room_with_profiles,
+        )
+
     @cached(max_entries=100000)
     async def get_room_summary(self, room_id: str) -> Dict[str, MemberSummary]:
         """Get the details of a room roughly suitable for use by the room
@@ -582,14 +629,12 @@ async def _get_joined_users_from_context(
         # We don't update the event cache hit ratio as it completely throws off
         # the hit ratio counts. After all, we don't populate the cache if we
         # miss it here
-        event_map = self._get_events_from_cache(
-            member_event_ids, allow_rejected=False, update_metrics=False
-        )
+        event_map = self._get_events_from_cache(member_event_ids, update_metrics=False)
 
         missing_member_event_ids = []
         for event_id in member_event_ids:
             ev_entry = event_map.get(event_id)
-            if ev_entry:
+            if ev_entry and not ev_entry.event.rejected_reason:
                 if ev_entry.event.membership == Membership.JOIN:
                     users_in_room[ev_entry.event.state_key] = ProfileInfo(
                         display_name=ev_entry.event.content.get("displayname", None),
@@ -602,7 +647,7 @@ async def _get_joined_users_from_context(
             event_to_memberships = await self._get_joined_profiles_from_event_ids(
                 missing_member_event_ids
             )
-            users_in_room.update((row for row in event_to_memberships.values() if row))
+            users_in_room.update(row for row in event_to_memberships.values() if row)
 
         if event is not None and event.type == EventTypes.Member:
             if event.membership == Membership.JOIN:
@@ -656,13 +701,22 @@ async def _get_joined_profiles_from_event_ids(self, event_ids: Iterable[str]):
 
     @cached(max_entries=10000)
     async def is_host_joined(self, room_id: str, host: str) -> bool:
+        return await self._check_host_room_membership(room_id, host, Membership.JOIN)
+
+    @cached(max_entries=10000)
+    async def is_host_invited(self, room_id: str, host: str) -> bool:
+        return await self._check_host_room_membership(room_id, host, Membership.INVITE)
+
+    async def _check_host_room_membership(
+        self, room_id: str, host: str, membership: str
+    ) -> bool:
         if "%" in host or "_" in host:
             raise Exception("Invalid host name")
 
         sql = """
             SELECT state_key FROM current_state_events AS c
             INNER JOIN room_memberships AS m USING (event_id)
-            WHERE m.membership = 'join'
+            WHERE m.membership = ?
                 AND type = 'm.room.member'
                 AND c.room_id = ?
                 AND state_key LIKE ?
@@ -675,7 +729,7 @@ async def is_host_joined(self, room_id: str, host: str) -> bool:
         like_clause = "%:" + host
 
         rows = await self.db_pool.execute(
-            "is_host_joined", None, sql, room_id, like_clause
+            "is_host_joined", None, sql, membership, room_id, like_clause
         )
 
         if not rows:
@@ -704,19 +758,82 @@ async def get_joined_hosts(self, room_id: str, state_entry):
 
     @cached(num_args=2, max_entries=10000, iterable=True)
     async def _get_joined_hosts(
-        self, room_id, state_group, current_state_ids, state_entry
-    ):
-        # We don't use `state_group`, its there so that we can cache based
-        # on it. However, its important that its never None, since two current_state's
-        # with a state_group of None are likely to be different.
+        self,
+        room_id: str,
+        state_group: int,
+        current_state_ids: StateMap[str],
+        state_entry: "_StateCacheEntry",
+    ) -> FrozenSet[str]:
+        # We don't use `state_group`, its there so that we can cache based on
+        # it. However, its important that its never None, since two
+        # current_state's with a state_group of None are likely to be different.
+        #
+        # The `state_group` must match the `state_entry.state_group` (if not None).
         assert state_group is not None
-
+        assert state_entry.state_group is None or state_entry.state_group == state_group
+
+        # We use a secondary cache of previous work to allow us to build up the
+        # joined hosts for the given state group based on previous state groups.
+        #
+        # We cache one object per room containing the results of the last state
+        # group we got joined hosts for. The idea is that generally
+        # `get_joined_hosts` is called with the "current" state group for the
+        # room, and so consecutive calls will be for consecutive state groups
+        # which point to the previous state group.
         cache = await self._get_joined_hosts_cache(room_id)
-        return await cache.get_destinations(state_entry)
+
+        # If the state group in the cache matches, we already have the data we need.
+        if state_entry.state_group == cache.state_group:
+            return frozenset(cache.hosts_to_joined_users)
+
+        # Since we'll mutate the cache we need to lock.
+        with (await self._joined_host_linearizer.queue(room_id)):
+            if state_entry.state_group == cache.state_group:
+                # Same state group, so nothing to do. We've already checked for
+                # this above, but the cache may have changed while waiting on
+                # the lock.
+                pass
+            elif state_entry.prev_group == cache.state_group:
+                # The cached work is for the previous state group, so we work out
+                # the delta.
+                for (typ, state_key), event_id in state_entry.delta_ids.items():
+                    if typ != EventTypes.Member:
+                        continue
+
+                    host = intern_string(get_domain_from_id(state_key))
+                    user_id = state_key
+                    known_joins = cache.hosts_to_joined_users.setdefault(host, set())
+
+                    event = await self.get_event(event_id)
+                    if event.membership == Membership.JOIN:
+                        known_joins.add(user_id)
+                    else:
+                        known_joins.discard(user_id)
+
+                        if not known_joins:
+                            cache.hosts_to_joined_users.pop(host, None)
+            else:
+                # The cache doesn't match the state group or prev state group,
+                # so we calculate the result from first principles.
+                joined_users = await self.get_joined_users_from_state(
+                    room_id, state_entry
+                )
+
+                cache.hosts_to_joined_users = {}
+                for user_id in joined_users:
+                    host = intern_string(get_domain_from_id(user_id))
+                    cache.hosts_to_joined_users.setdefault(host, set()).add(user_id)
+
+            if state_entry.state_group:
+                cache.state_group = state_entry.state_group
+            else:
+                cache.state_group = object()
+
+        return frozenset(cache.hosts_to_joined_users)
 
     @cached(max_entries=10000)
     def _get_joined_hosts_cache(self, room_id: str) -> "_JoinedHostsCache":
-        return _JoinedHostsCache(self, room_id)
+        return _JoinedHostsCache()
 
     @cached(num_args=2)
     async def did_forget(self, user_id: str, room_id: str) -> bool:
@@ -1026,71 +1143,18 @@ def f(txn):
         await self.db_pool.runInteraction("forget_membership", f)
 
 
+@attr.s(slots=True)
 class _JoinedHostsCache:
-    """Cache for joined hosts in a room that is optimised to handle updates
-    via state deltas.
-    """
-
-    def __init__(self, store, room_id):
-        self.store = store
-        self.room_id = room_id
-
-        self.hosts_to_joined_users = {}
-
-        self.state_group = object()
-
-        self.linearizer = Linearizer("_JoinedHostsCache")
-
-        self._len = 0
-
-    async def get_destinations(self, state_entry: "_StateCacheEntry") -> Set[str]:
-        """Get set of destinations for a state entry
-
-        Args:
-            state_entry
-
-        Returns:
-            The destinations as a set.
-        """
-        if state_entry.state_group == self.state_group:
-            return frozenset(self.hosts_to_joined_users)
-
-        with (await self.linearizer.queue(())):
-            if state_entry.state_group == self.state_group:
-                pass
-            elif state_entry.prev_group == self.state_group:
-                for (typ, state_key), event_id in state_entry.delta_ids.items():
-                    if typ != EventTypes.Member:
-                        continue
-
-                    host = intern_string(get_domain_from_id(state_key))
-                    user_id = state_key
-                    known_joins = self.hosts_to_joined_users.setdefault(host, set())
-
-                    event = await self.store.get_event(event_id)
-                    if event.membership == Membership.JOIN:
-                        known_joins.add(user_id)
-                    else:
-                        known_joins.discard(user_id)
+    """The cached data used by the `_get_joined_hosts_cache`."""
 
-                        if not known_joins:
-                            self.hosts_to_joined_users.pop(host, None)
-            else:
-                joined_users = await self.store.get_joined_users_from_state(
-                    self.room_id, state_entry
-                )
+    # Dict of host to the set of their users in the room at the state group.
+    hosts_to_joined_users = attr.ib(type=Dict[str, Set[str]], factory=dict)
 
-                self.hosts_to_joined_users = {}
-                for user_id in joined_users:
-                    host = intern_string(get_domain_from_id(user_id))
-                    self.hosts_to_joined_users.setdefault(host, set()).add(user_id)
-
-            if state_entry.state_group:
-                self.state_group = state_entry.state_group
-            else:
-                self.state_group = object()
-            self._len = sum(len(v) for v in self.hosts_to_joined_users.values())
-        return frozenset(self.hosts_to_joined_users)
+    # The state group `hosts_to_joined_users` is derived from. Will be an object
+    # if the instance is newly created or if the state is not based on a state
+    # group. (An object is used as a sentinel value to ensure that it never is
+    # equal to anything else).
+    state_group = attr.ib(type=Union[object, int], factory=object)
 
     def __len__(self):
-        return self._len
+        return sum(len(v) for v in self.hosts_to_joined_users.values())
diff --git a/synapse/storage/databases/main/schema/full_schemas/README.md b/synapse/storage/databases/main/schema/full_schemas/README.md
deleted file mode 100644
index c00f287190..0000000000
--- a/synapse/storage/databases/main/schema/full_schemas/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# Synapse Database Schemas
-
-These schemas are used as a basis to create brand new Synapse databases, on both
-SQLite3 and Postgres.
-
-## Building full schema dumps
-
-If you want to recreate these schemas, they need to be made from a database that
-has had all background updates run.
-
-To do so, use `scripts-dev/make_full_schema.sh`. This will produce new
-`full.sql.postgres ` and `full.sql.sqlite` files. 
-
-Ensure postgres is installed and your user has the ability to run bash commands
-such as `createdb`, then call
-
-    ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
-
-There are currently two folders with full-schema snapshots. `16` is a snapshot
-from 2015, for historical reference. The other contains the most recent full
-schema snapshot.
diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py
index f5e7d9ef98..6480d5a9f5 100644
--- a/synapse/storage/databases/main/search.py
+++ b/synapse/storage/databases/main/search.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,7 +15,7 @@
 import logging
 import re
 from collections import namedtuple
-from typing import List, Optional, Set
+from typing import Collection, List, Optional, Set
 
 from synapse.api.errors import SynapseError
 from synapse.events import EventBase
@@ -24,7 +23,6 @@
 from synapse.storage.database import DatabasePool
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.storage.engines import PostgresEngine, Sqlite3Engine
-from synapse.types import Collection
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/storage/databases/main/signatures.py b/synapse/storage/databases/main/signatures.py
index c8c67953e4..ab2159c2d3 100644
--- a/synapse/storage/databases/main/signatures.py
+++ b/synapse/storage/databases/main/signatures.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py
index a7f371732f..8e22da99ae 100644
--- a/synapse/storage/databases/main/state.py
+++ b/synapse/storage/databases/main/state.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -23,7 +22,7 @@
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion
 from synapse.events import EventBase
 from synapse.storage._base import SQLBaseStore
-from synapse.storage.database import DatabasePool
+from synapse.storage.database import DatabasePool, LoggingTransaction
 from synapse.storage.databases.main.events_worker import EventsWorkerStore
 from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
 from synapse.storage.state import StateFilter
@@ -59,15 +58,32 @@ def __init__(self, database: DatabasePool, db_conn, hs):
 
     async def get_room_version(self, room_id: str) -> RoomVersion:
         """Get the room_version of a given room
-
         Raises:
             NotFoundError: if the room is unknown
+            UnsupportedRoomVersionError: if the room uses an unknown room version.
+                Typically this happens if support for the room's version has been
+                removed from Synapse.
+        """
+        return await self.db_pool.runInteraction(
+            "get_room_version_txn",
+            self.get_room_version_txn,
+            room_id,
+        )
 
+    def get_room_version_txn(
+        self, txn: LoggingTransaction, room_id: str
+    ) -> RoomVersion:
+        """Get the room_version of a given room
+        Args:
+            txn: Transaction object
+            room_id: The room_id of the room you are trying to get the version for
+        Raises:
+            NotFoundError: if the room is unknown
             UnsupportedRoomVersionError: if the room uses an unknown room version.
                 Typically this happens if support for the room's version has been
                 removed from Synapse.
         """
-        room_version_id = await self.get_room_version_id(room_id)
+        room_version_id = self.get_room_version_id_txn(txn, room_id)
         v = KNOWN_ROOM_VERSIONS.get(room_version_id)
 
         if not v:
@@ -81,7 +97,20 @@ async def get_room_version(self, room_id: str) -> RoomVersion:
     @cached(max_entries=10000)
     async def get_room_version_id(self, room_id: str) -> str:
         """Get the room_version of a given room
+        Raises:
+            NotFoundError: if the room is unknown
+        """
+        return await self.db_pool.runInteraction(
+            "get_room_version_id_txn",
+            self.get_room_version_id_txn,
+            room_id,
+        )
 
+    def get_room_version_id_txn(self, txn: LoggingTransaction, room_id: str) -> str:
+        """Get the room_version of a given room
+        Args:
+            txn: Transaction object
+            room_id: The room_id of the room you are trying to get the version for
         Raises:
             NotFoundError: if the room is unknown
         """
@@ -89,24 +118,22 @@ async def get_room_version_id(self, room_id: str) -> str:
         # First we try looking up room version from the database, but for old
         # rooms we might not have added the room version to it yet so we fall
         # back to previous behaviour and look in current state events.
-
+        #
         # We really should have an entry in the rooms table for every room we
         # care about, but let's be a bit paranoid (at least while the background
         # update is happening) to avoid breaking existing rooms.
-        version = await self.db_pool.simple_select_one_onecol(
+        room_version = self.db_pool.simple_select_one_onecol_txn(
+            txn,
             table="rooms",
             keyvalues={"room_id": room_id},
             retcol="room_version",
-            desc="get_room_version",
             allow_none=True,
         )
 
-        if version is not None:
-            return version
+        if room_version is None:
+            raise NotFoundError("Could not room_version for %s" % (room_id,))
 
-        # Retrieve the room's create event
-        create_event = await self.get_create_event_for_room(room_id)
-        return create_event.content.get("room_version", "1")
+        return room_version
 
     async def get_room_predecessor(self, room_id: str) -> Optional[dict]:
         """Get the predecessor of an upgraded room if it exists.
@@ -190,7 +217,7 @@ def _get_current_state_ids_txn(txn):
 
     # FIXME: how should this be cached?
     async def get_filtered_current_state_ids(
-        self, room_id: str, state_filter: StateFilter = StateFilter.all()
+        self, room_id: str, state_filter: Optional[StateFilter] = None
     ) -> StateMap[str]:
         """Get the current state event of a given type for a room based on the
         current_state_events table.  This may not be as up-to-date as the result
@@ -205,7 +232,9 @@ async def get_filtered_current_state_ids(
             Map from type/state_key to event ID.
         """
 
-        where_clause, where_args = state_filter.make_sql_filter_clause()
+        where_clause, where_args = (
+            state_filter or StateFilter.all()
+        ).make_sql_filter_clause()
 
         if not where_clause:
             # We delegate to the cached version
diff --git a/synapse/storage/databases/main/state_deltas.py b/synapse/storage/databases/main/state_deltas.py
index 0dbb501f16..bff7d0404f 100644
--- a/synapse/storage/databases/main/state_deltas.py
+++ b/synapse/storage/databases/main/state_deltas.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/stats.py b/synapse/storage/databases/main/stats.py
index 38adecc78a..42edbcc057 100644
--- a/synapse/storage/databases/main/stats.py
+++ b/synapse/storage/databases/main/stats.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018, 2019 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
@@ -27,7 +26,6 @@
 from synapse.api.errors import StoreError
 from synapse.storage.database import DatabasePool
 from synapse.storage.databases.main.state_deltas import StateDeltasStore
-from synapse.storage.engines import PostgresEngine
 from synapse.types import JsonDict
 from synapse.util.caches.descriptors import cached
 
@@ -50,14 +48,6 @@
     "user": ("joined_rooms",),
 }
 
-# these fields are per-timeslice and so should be reset to 0 upon a new slice
-# You can draw these stats on a histogram.
-# Example: number of events sent locally during a time slice
-PER_SLICE_FIELDS = {
-    "room": ("total_events", "total_event_bytes"),
-    "user": ("invites_sent", "rooms_created", "total_events", "total_event_bytes"),
-}
-
 TYPE_TO_TABLE = {"room": ("room_stats", "room_id"), "user": ("user_stats", "user_id")}
 
 # these are the tables (& ID columns) which contain our actual subjects
@@ -67,18 +57,39 @@
 class UserSortOrder(Enum):
     """
     Enum to define the sorting method used when returning users
-    with get_users_media_usage_paginate
+    with get_users_paginate in __init__.py
+    and get_users_media_usage_paginate in stats.py
 
-    MEDIA_LENGTH = ordered by size of uploaded media. Smallest to largest.
-    MEDIA_COUNT = ordered by number of uploaded media. Smallest to largest.
+    When moves this to __init__.py gets `builtins.ImportError` with
+    `most likely due to a circular import`
+
+    MEDIA_LENGTH = ordered by size of uploaded media.
+    MEDIA_COUNT = ordered by number of uploaded media.
     USER_ID = ordered alphabetically by `user_id`.
+    NAME = ordered alphabetically by `user_id`. This is for compatibility reasons,
+    as the user_id is returned in the name field in the response in list users admin API.
     DISPLAYNAME = ordered alphabetically by `displayname`
+    GUEST = ordered by `is_guest`
+    ADMIN = ordered by `admin`
+    DEACTIVATED = ordered by `deactivated`
+    USER_TYPE = ordered alphabetically by `user_type`
+    AVATAR_URL = ordered alphabetically by `avatar_url`
+    SHADOW_BANNED = ordered by `shadow_banned`
+    CREATION_TS = ordered by `creation_ts`
     """
 
     MEDIA_LENGTH = "media_length"
     MEDIA_COUNT = "media_count"
     USER_ID = "user_id"
+    NAME = "name"
     DISPLAYNAME = "displayname"
+    GUEST = "is_guest"
+    ADMIN = "admin"
+    DEACTIVATED = "deactivated"
+    USER_TYPE = "user_type"
+    AVATAR_URL = "avatar_url"
+    SHADOW_BANNED = "shadow_banned"
+    CREATION_TS = "creation_ts"
 
 
 class StatsStore(StateDeltasStore):
@@ -88,7 +99,6 @@ def __init__(self, database: DatabasePool, db_conn, hs):
         self.server_name = hs.hostname
         self.clock = self.hs.get_clock()
         self.stats_enabled = hs.config.stats_enabled
-        self.stats_bucket_size = hs.config.stats_bucket_size
 
         self.stats_delta_processing_lock = DeferredLock()
 
@@ -104,22 +114,6 @@ def __init__(self, database: DatabasePool, db_conn, hs):
         self.db_pool.updates.register_noop_background_update("populate_stats_cleanup")
         self.db_pool.updates.register_noop_background_update("populate_stats_prepare")
 
-    def quantise_stats_time(self, ts):
-        """
-        Quantises a timestamp to be a multiple of the bucket size.
-
-        Args:
-            ts (int): the timestamp to quantise, in milliseconds since the Unix
-                Epoch
-
-        Returns:
-            int: a timestamp which
-              - is divisible by the bucket size;
-              - is no later than `ts`; and
-              - is the largest such timestamp.
-        """
-        return (ts // self.stats_bucket_size) * self.stats_bucket_size
-
     async def _populate_stats_process_users(self, progress, batch_size):
         """
         This is a background update which regenerates statistics for users.
@@ -270,56 +264,6 @@ async def update_room_state(self, room_id: str, fields: Dict[str, Any]) -> None:
             desc="update_room_state",
         )
 
-    async def get_statistics_for_subject(
-        self, stats_type: str, stats_id: str, start: str, size: int = 100
-    ) -> List[dict]:
-        """
-        Get statistics for a given subject.
-
-        Args:
-            stats_type: The type of subject
-            stats_id: The ID of the subject (e.g. room_id or user_id)
-            start: Pagination start. Number of entries, not timestamp.
-            size: How many entries to return.
-
-        Returns:
-            A list of dicts, where the dict has the keys of
-            ABSOLUTE_STATS_FIELDS[stats_type],  and "bucket_size" and "end_ts".
-        """
-        return await self.db_pool.runInteraction(
-            "get_statistics_for_subject",
-            self._get_statistics_for_subject_txn,
-            stats_type,
-            stats_id,
-            start,
-            size,
-        )
-
-    def _get_statistics_for_subject_txn(
-        self, txn, stats_type, stats_id, start, size=100
-    ):
-        """
-        Transaction-bound version of L{get_statistics_for_subject}.
-        """
-
-        table, id_col = TYPE_TO_TABLE[stats_type]
-        selected_columns = list(
-            ABSOLUTE_STATS_FIELDS[stats_type] + PER_SLICE_FIELDS[stats_type]
-        )
-
-        slice_list = self.db_pool.simple_select_list_paginate_txn(
-            txn,
-            table + "_historical",
-            "end_ts",
-            start,
-            size,
-            retcols=selected_columns + ["bucket_size", "end_ts"],
-            keyvalues={id_col: stats_id},
-            order_direction="DESC",
-        )
-
-        return slice_list
-
     @cached()
     async def get_earliest_token_for_stats(
         self, stats_type: str, id: str
@@ -433,14 +377,10 @@ def _update_stats_delta_txn(
 
         table, id_col = TYPE_TO_TABLE[stats_type]
 
-        quantised_ts = self.quantise_stats_time(int(ts))
-        end_ts = quantised_ts + self.stats_bucket_size
-
         # Lets be paranoid and check that all the given field names are known
         abs_field_names = ABSOLUTE_STATS_FIELDS[stats_type]
-        slice_field_names = PER_SLICE_FIELDS[stats_type]
         for field in chain(fields.keys(), absolute_field_overrides.keys()):
-            if field not in abs_field_names and field not in slice_field_names:
+            if field not in abs_field_names:
                 # guard against potential SQL injection dodginess
                 raise ValueError(
                     "%s is not a recognised field"
@@ -473,20 +413,6 @@ def _update_stats_delta_txn(
             additive_relatives=deltas_of_absolute_fields,
         )
 
-        per_slice_additive_relatives = {
-            key: fields.get(key, 0) for key in slice_field_names
-        }
-        self._upsert_copy_from_table_with_additive_relatives_txn(
-            txn=txn,
-            into_table=table + "_historical",
-            keyvalues={id_col: stats_id},
-            extra_dst_insvalues={"bucket_size": self.stats_bucket_size},
-            extra_dst_keyvalues={"end_ts": end_ts},
-            additive_relatives=per_slice_additive_relatives,
-            src_table=table + "_current",
-            copy_columns=abs_field_names,
-        )
-
     def _upsert_with_additive_relatives_txn(
         self, txn, table, keyvalues, absolutes, additive_relatives
     ):
@@ -510,7 +436,7 @@ def _upsert_with_additive_relatives_txn(
             ]
 
             relative_updates = [
-                "%(field)s = EXCLUDED.%(field)s + %(table)s.%(field)s"
+                "%(field)s = EXCLUDED.%(field)s + COALESCE(%(table)s.%(field)s, 0)"
                 % {"table": table, "field": field}
                 for field in additive_relatives.keys()
             ]
@@ -550,205 +476,13 @@ def _upsert_with_additive_relatives_txn(
                 self.db_pool.simple_insert_txn(txn, table, merged_dict)
             else:
                 for (key, val) in additive_relatives.items():
-                    current_row[key] += val
+                    if current_row[key] is None:
+                        current_row[key] = val
+                    else:
+                        current_row[key] += val
                 current_row.update(absolutes)
                 self.db_pool.simple_update_one_txn(txn, table, keyvalues, current_row)
 
-    def _upsert_copy_from_table_with_additive_relatives_txn(
-        self,
-        txn,
-        into_table,
-        keyvalues,
-        extra_dst_keyvalues,
-        extra_dst_insvalues,
-        additive_relatives,
-        src_table,
-        copy_columns,
-    ):
-        """Updates the historic stats table with latest updates.
-
-        This involves copying "absolute" fields from the `_current` table, and
-        adding relative fields to any existing values.
-
-        Args:
-             txn: Transaction
-             into_table (str): The destination table to UPSERT the row into
-             keyvalues (dict[str, any]): Row-identifying key values
-             extra_dst_keyvalues (dict[str, any]): Additional keyvalues
-                for `into_table`.
-             extra_dst_insvalues (dict[str, any]): Additional values to insert
-                on new row creation for `into_table`.
-             additive_relatives (dict[str, any]): Fields that will be added onto
-                if existing row present. (Must be disjoint from copy_columns.)
-             src_table (str): The source table to copy from
-             copy_columns (iterable[str]): The list of columns to copy
-        """
-        if self.database_engine.can_native_upsert:
-            ins_columns = chain(
-                keyvalues,
-                copy_columns,
-                additive_relatives,
-                extra_dst_keyvalues,
-                extra_dst_insvalues,
-            )
-            sel_exprs = chain(
-                keyvalues,
-                copy_columns,
-                (
-                    "?"
-                    for _ in chain(
-                        additive_relatives, extra_dst_keyvalues, extra_dst_insvalues
-                    )
-                ),
-            )
-            keyvalues_where = ("%s = ?" % f for f in keyvalues)
-
-            sets_cc = ("%s = EXCLUDED.%s" % (f, f) for f in copy_columns)
-            sets_ar = (
-                "%s = EXCLUDED.%s + %s.%s" % (f, f, into_table, f)
-                for f in additive_relatives
-            )
-
-            sql = """
-                INSERT INTO %(into_table)s (%(ins_columns)s)
-                SELECT %(sel_exprs)s
-                FROM %(src_table)s
-                WHERE %(keyvalues_where)s
-                ON CONFLICT (%(keyvalues)s)
-                DO UPDATE SET %(sets)s
-            """ % {
-                "into_table": into_table,
-                "ins_columns": ", ".join(ins_columns),
-                "sel_exprs": ", ".join(sel_exprs),
-                "keyvalues_where": " AND ".join(keyvalues_where),
-                "src_table": src_table,
-                "keyvalues": ", ".join(
-                    chain(keyvalues.keys(), extra_dst_keyvalues.keys())
-                ),
-                "sets": ", ".join(chain(sets_cc, sets_ar)),
-            }
-
-            qargs = list(
-                chain(
-                    additive_relatives.values(),
-                    extra_dst_keyvalues.values(),
-                    extra_dst_insvalues.values(),
-                    keyvalues.values(),
-                )
-            )
-            txn.execute(sql, qargs)
-        else:
-            self.database_engine.lock_table(txn, into_table)
-            src_row = self.db_pool.simple_select_one_txn(
-                txn, src_table, keyvalues, copy_columns
-            )
-            all_dest_keyvalues = {**keyvalues, **extra_dst_keyvalues}
-            dest_current_row = self.db_pool.simple_select_one_txn(
-                txn,
-                into_table,
-                keyvalues=all_dest_keyvalues,
-                retcols=list(chain(additive_relatives.keys(), copy_columns)),
-                allow_none=True,
-            )
-
-            if dest_current_row is None:
-                merged_dict = {
-                    **keyvalues,
-                    **extra_dst_keyvalues,
-                    **extra_dst_insvalues,
-                    **src_row,
-                    **additive_relatives,
-                }
-                self.db_pool.simple_insert_txn(txn, into_table, merged_dict)
-            else:
-                for (key, val) in additive_relatives.items():
-                    src_row[key] = dest_current_row[key] + val
-                self.db_pool.simple_update_txn(
-                    txn, into_table, all_dest_keyvalues, src_row
-                )
-
-    async def get_changes_room_total_events_and_bytes(
-        self, min_pos: int, max_pos: int
-    ) -> Tuple[Dict[str, Dict[str, int]], Dict[str, Dict[str, int]]]:
-        """Fetches the counts of events in the given range of stream IDs.
-
-        Args:
-            min_pos
-            max_pos
-
-        Returns:
-            Mapping of room ID to field changes.
-        """
-
-        return await self.db_pool.runInteraction(
-            "stats_incremental_total_events_and_bytes",
-            self.get_changes_room_total_events_and_bytes_txn,
-            min_pos,
-            max_pos,
-        )
-
-    def get_changes_room_total_events_and_bytes_txn(
-        self, txn, low_pos: int, high_pos: int
-    ) -> Tuple[Dict[str, Dict[str, int]], Dict[str, Dict[str, int]]]:
-        """Gets the total_events and total_event_bytes counts for rooms and
-        senders, in a range of stream_orderings (including backfilled events).
-
-        Args:
-            txn
-            low_pos: Low stream ordering
-            high_pos: High stream ordering
-
-        Returns:
-            The room and user deltas for total_events/total_event_bytes in the
-            format of `stats_id` -> fields
-        """
-
-        if low_pos >= high_pos:
-            # nothing to do here.
-            return {}, {}
-
-        if isinstance(self.database_engine, PostgresEngine):
-            new_bytes_expression = "OCTET_LENGTH(json)"
-        else:
-            new_bytes_expression = "LENGTH(CAST(json AS BLOB))"
-
-        sql = """
-            SELECT events.room_id, COUNT(*) AS new_events, SUM(%s) AS new_bytes
-            FROM events INNER JOIN event_json USING (event_id)
-            WHERE (? < stream_ordering AND stream_ordering <= ?)
-                OR (? <= stream_ordering AND stream_ordering <= ?)
-            GROUP BY events.room_id
-        """ % (
-            new_bytes_expression,
-        )
-
-        txn.execute(sql, (low_pos, high_pos, -high_pos, -low_pos))
-
-        room_deltas = {
-            room_id: {"total_events": new_events, "total_event_bytes": new_bytes}
-            for room_id, new_events, new_bytes in txn
-        }
-
-        sql = """
-            SELECT events.sender, COUNT(*) AS new_events, SUM(%s) AS new_bytes
-            FROM events INNER JOIN event_json USING (event_id)
-            WHERE (? < stream_ordering AND stream_ordering <= ?)
-                OR (? <= stream_ordering AND stream_ordering <= ?)
-            GROUP BY events.sender
-        """ % (
-            new_bytes_expression,
-        )
-
-        txn.execute(sql, (low_pos, high_pos, -high_pos, -low_pos))
-
-        user_deltas = {
-            user_id: {"total_events": new_events, "total_event_bytes": new_bytes}
-            for user_id, new_events, new_bytes in txn
-            if self.hs.is_mine_id(user_id)
-        }
-
-        return room_deltas, user_deltas
-
     async def _calculate_and_set_initial_state_for_room(
         self, room_id: str
     ) -> Tuple[dict, dict, int]:
@@ -875,6 +609,7 @@ def _fetch_current_state_stats(txn):
                 "invited_members": membership_counts.get(Membership.INVITE, 0),
                 "left_members": membership_counts.get(Membership.LEAVE, 0),
                 "banned_members": membership_counts.get(Membership.BAN, 0),
+                "knocked_members": membership_counts.get(Membership.KNOCK, 0),
                 "local_users_in_room": len(local_users_in_room),
             },
         )
@@ -914,10 +649,10 @@ async def get_users_media_usage_paginate(
         limit: int,
         from_ts: Optional[int] = None,
         until_ts: Optional[int] = None,
-        order_by: Optional[UserSortOrder] = UserSortOrder.USER_ID.value,
+        order_by: Optional[str] = UserSortOrder.USER_ID.value,
         direction: Optional[str] = "f",
         search_term: Optional[str] = None,
-    ) -> Tuple[List[JsonDict], Dict[str, int]]:
+    ) -> Tuple[List[JsonDict], int]:
         """Function to retrieve a paginated list of users and their uploaded local media
         (size and number). This will return a json list of users and the
         total number of users matching the filter criteria.
diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py
index 91f8abb67d..959f13de47 100644
--- a/synapse/storage/databases/main/stream.py
+++ b/synapse/storage/databases/main/stream.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018-2019 New Vector Ltd
@@ -38,7 +37,7 @@
 import abc
 import logging
 from collections import namedtuple
-from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Collection, Dict, List, Optional, Set, Tuple
 
 from twisted.internet import defer
 
@@ -54,7 +53,7 @@
 from synapse.storage.databases.main.events_worker import EventsWorkerStore
 from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
 from synapse.storage.util.id_generators import MultiWriterIdGenerator
-from synapse.types import Collection, PersistedEventPosition, RoomStreamToken
+from synapse.types import PersistedEventPosition, RoomStreamToken
 from synapse.util.caches.descriptors import cached
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
@@ -1086,9 +1085,7 @@ def _paginate_room_events_txn(
         # stream token (as returned by `RoomStreamToken.get_max_stream_pos`) and
         # then filtering the results.
         if from_token.topological is not None:
-            from_bound = (
-                from_token.as_historical_tuple()
-            )  # type: Tuple[Optional[int], int]
+            from_bound: Tuple[Optional[int], int] = from_token.as_historical_tuple()
         elif direction == "b":
             from_bound = (
                 None,
@@ -1100,7 +1097,7 @@ def _paginate_room_events_txn(
                 from_token.stream,
             )
 
-        to_bound = None  # type: Optional[Tuple[Optional[int], int]]
+        to_bound: Optional[Tuple[Optional[int], int]] = None
         if to_token:
             if to_token.topological is not None:
                 to_bound = to_token.as_historical_tuple()
diff --git a/synapse/storage/databases/main/tags.py b/synapse/storage/databases/main/tags.py
index 50067eabfc..f93ff0a545 100644
--- a/synapse/storage/databases/main/tags.py
+++ b/synapse/storage/databases/main/tags.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -43,7 +42,7 @@ async def get_tags_for_user(self, user_id: str) -> Dict[str, Dict[str, JsonDict]
             "room_tags", {"user_id": user_id}, ["room_id", "tag", "content"]
         )
 
-        tags_by_room = {}  # type: Dict[str, Dict[str, JsonDict]]
+        tags_by_room: Dict[str, Dict[str, JsonDict]] = {}
         for row in rows:
             room_tags = tags_by_room.setdefault(row["room_id"], {})
             room_tags[row["tag"]] = db_to_json(row["content"])
diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py
index b7072f1f5e..7728d5f102 100644
--- a/synapse/storage/databases/main/transactions.py
+++ b/synapse/storage/databases/main/transactions.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,13 +16,15 @@
 from collections import namedtuple
 from typing import Iterable, List, Optional, Tuple
 
+import attr
 from canonicaljson import encode_canonical_json
 
 from synapse.metrics.background_process_metrics import wrap_as_background_process
-from synapse.storage._base import SQLBaseStore, db_to_json
+from synapse.storage._base import db_to_json
 from synapse.storage.database import DatabasePool, LoggingTransaction
+from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
 from synapse.types import JsonDict
-from synapse.util.caches.expiringcache import ExpiringCache
+from synapse.util.caches.descriptors import cached
 
 db_binary_type = memoryview
 
@@ -39,10 +40,23 @@
     "_TransactionRow", ("response_code", "response_json")
 )
 
-SENTINEL = object()
 
+@attr.s(slots=True, frozen=True, auto_attribs=True)
+class DestinationRetryTimings:
+    """The current destination retry timing info for a remote server."""
 
-class TransactionWorkerStore(SQLBaseStore):
+    # The first time we tried and failed to reach the remote server, in ms.
+    failure_ts: int
+
+    # The last time we tried and failed to reach the remote server, in ms.
+    retry_last_ts: int
+
+    # How long since the last time we tried to reach the remote server before
+    # trying again, in ms.
+    retry_interval: int
+
+
+class TransactionWorkerStore(CacheInvalidationWorkerStore):
     def __init__(self, database: DatabasePool, db_conn, hs):
         super().__init__(database, db_conn, hs)
 
@@ -61,19 +75,6 @@ def _cleanup_transactions_txn(txn):
             "_cleanup_transactions", _cleanup_transactions_txn
         )
 
-
-class TransactionStore(TransactionWorkerStore):
-    """A collection of queries for handling PDUs."""
-
-    def __init__(self, database: DatabasePool, db_conn, hs):
-        super().__init__(database, db_conn, hs)
-
-        self._destination_retry_cache = ExpiringCache(
-            cache_name="get_destination_retry_timings",
-            clock=self._clock,
-            expiry_ms=5 * 60 * 1000,
-        )
-
     async def get_received_txn_response(
         self, transaction_id: str, origin: str
     ) -> Optional[Tuple[int, JsonDict]]:
@@ -133,20 +134,26 @@ async def set_received_txn_response(
             response_dict: The response, to be encoded into JSON.
         """
 
-        await self.db_pool.simple_insert(
+        await self.db_pool.simple_upsert(
             table="received_transactions",
-            values={
+            keyvalues={
                 "transaction_id": transaction_id,
                 "origin": origin,
+            },
+            values={},
+            insertion_values={
                 "response_code": code,
                 "response_json": db_binary_type(encode_canonical_json(response_dict)),
                 "ts": self._clock.time_msec(),
             },
-            or_ignore=True,
             desc="set_received_txn_response",
         )
 
-    async def get_destination_retry_timings(self, destination):
+    @cached(max_entries=10000)
+    async def get_destination_retry_timings(
+        self,
+        destination: str,
+    ) -> Optional[DestinationRetryTimings]:
         """Gets the current retry timings (if any) for a given destination.
 
         Args:
@@ -157,34 +164,29 @@ async def get_destination_retry_timings(self, destination):
             Otherwise a dict for the retry scheme
         """
 
-        result = self._destination_retry_cache.get(destination, SENTINEL)
-        if result is not SENTINEL:
-            return result
-
         result = await self.db_pool.runInteraction(
             "get_destination_retry_timings",
             self._get_destination_retry_timings,
             destination,
         )
 
-        # We don't hugely care about race conditions between getting and
-        # invalidating the cache, since we time out fairly quickly anyway.
-        self._destination_retry_cache[destination] = result
         return result
 
-    def _get_destination_retry_timings(self, txn, destination):
+    def _get_destination_retry_timings(
+        self, txn, destination: str
+    ) -> Optional[DestinationRetryTimings]:
         result = self.db_pool.simple_select_one_txn(
             txn,
             table="destinations",
             keyvalues={"destination": destination},
-            retcols=("destination", "failure_ts", "retry_last_ts", "retry_interval"),
+            retcols=("failure_ts", "retry_last_ts", "retry_interval"),
             allow_none=True,
         )
 
         # check we have a row and retry_last_ts is not null or zero
         # (retry_last_ts can't be negative)
         if result and result["retry_last_ts"]:
-            return result
+            return DestinationRetryTimings(**result)
         else:
             return None
 
@@ -205,7 +207,6 @@ async def set_destination_retry_timings(
             retry_interval: how long until next retry in ms
         """
 
-        self._destination_retry_cache.pop(destination, None)
         if self.database_engine.can_native_upsert:
             return await self.db_pool.runInteraction(
                 "set_destination_retry_timings",
@@ -253,6 +254,10 @@ def _set_destination_retry_timings_native(
 
         txn.execute(sql, (destination, failure_ts, retry_last_ts, retry_interval))
 
+        self._invalidate_cache_and_stream(
+            txn, self.get_destination_retry_timings, (destination,)
+        )
+
     def _set_destination_retry_timings_emulated(
         self, txn, destination, failure_ts, retry_last_ts, retry_interval
     ):
@@ -296,6 +301,10 @@ def _set_destination_retry_timings_emulated(
                 },
             )
 
+        self._invalidate_cache_and_stream(
+            txn, self.get_destination_retry_timings, (destination,)
+        )
+
     async def store_destination_rooms_entries(
         self,
         destinations: Iterable[str],
diff --git a/synapse/storage/databases/main/ui_auth.py b/synapse/storage/databases/main/ui_auth.py
index 5473ec1485..38bfdf5dad 100644
--- a/synapse/storage/databases/main/ui_auth.py
+++ b/synapse/storage/databases/main/ui_auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -225,12 +224,12 @@ def _set_ui_auth_session_data_txn(
         self, txn: LoggingTransaction, session_id: str, key: str, value: Any
     ):
         # Get the current value.
-        result = self.db_pool.simple_select_one_txn(
+        result: Dict[str, Any] = self.db_pool.simple_select_one_txn(  # type: ignore
             txn,
             table="ui_auth_sessions",
             keyvalues={"session_id": session_id},
             retcols=("serverdict",),
-        )  # type: Dict[str, Any]  # type: ignore
+        )
 
         # Update it and add it back to the database.
         serverdict = db_to_json(result["serverdict"])
diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py
index 1026f321e5..9d28d69ac7 100644
--- a/synapse/storage/databases/main/user_directory.py
+++ b/synapse/storage/databases/main/user_directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -143,8 +142,6 @@ async def _populate_user_directory_process_rooms(self, progress, batch_size):
             batch_size (int): Maximum number of state events to process
                 per cycle.
         """
-        state = self.hs.get_state_handler()
-
         # If we don't have progress filed, delete everything.
         if not progress:
             await self.delete_all_from_user_dir()
@@ -198,7 +195,7 @@ def _get_next_batch(txn):
                     room_id
                 )
 
-                users_with_profile = await state.get_current_users_in_room(room_id)
+                users_with_profile = await self.get_users_in_room_with_profiles(room_id)
                 user_ids = set(users_with_profile)
 
                 # Update each user in the user directory.
@@ -380,7 +377,7 @@ async def update_profile_in_user_dir(
             avatar_url = None
 
         def _update_profile_in_user_dir_txn(txn):
-            new_entry = self.db_pool.simple_upsert_txn(
+            self.db_pool.simple_upsert_txn(
                 txn,
                 table="user_directory",
                 keyvalues={"user_id": user_id},
@@ -391,8 +388,7 @@ def _update_profile_in_user_dir_txn(txn):
             if isinstance(self.database_engine, PostgresEngine):
                 # We weight the localpart most highly, then display name and finally
                 # server name
-                if self.database_engine.can_native_upsert:
-                    sql = """
+                sql = """
                         INSERT INTO user_directory_search(user_id, vector)
                         VALUES (?,
                             setweight(to_tsvector('simple', ?), 'A')
@@ -400,58 +396,15 @@ def _update_profile_in_user_dir_txn(txn):
                             || setweight(to_tsvector('simple', COALESCE(?, '')), 'B')
                         ) ON CONFLICT (user_id) DO UPDATE SET vector=EXCLUDED.vector
                     """
-                    txn.execute(
-                        sql,
-                        (
-                            user_id,
-                            get_localpart_from_id(user_id),
-                            get_domain_from_id(user_id),
-                            display_name,
-                        ),
-                    )
-                else:
-                    # TODO: Remove this code after we've bumped the minimum version
-                    # of postgres to always support upserts, so we can get rid of
-                    # `new_entry` usage
-                    if new_entry is True:
-                        sql = """
-                            INSERT INTO user_directory_search(user_id, vector)
-                            VALUES (?,
-                                setweight(to_tsvector('simple', ?), 'A')
-                                || setweight(to_tsvector('simple', ?), 'D')
-                                || setweight(to_tsvector('simple', COALESCE(?, '')), 'B')
-                            )
-                        """
-                        txn.execute(
-                            sql,
-                            (
-                                user_id,
-                                get_localpart_from_id(user_id),
-                                get_domain_from_id(user_id),
-                                display_name,
-                            ),
-                        )
-                    elif new_entry is False:
-                        sql = """
-                            UPDATE user_directory_search
-                            SET vector = setweight(to_tsvector('simple', ?), 'A')
-                                || setweight(to_tsvector('simple', ?), 'D')
-                                || setweight(to_tsvector('simple', COALESCE(?, '')), 'B')
-                            WHERE user_id = ?
-                        """
-                        txn.execute(
-                            sql,
-                            (
-                                get_localpart_from_id(user_id),
-                                get_domain_from_id(user_id),
-                                display_name,
-                                user_id,
-                            ),
-                        )
-                    else:
-                        raise RuntimeError(
-                            "upsert returned None when 'can_native_upsert' is False"
-                        )
+                txn.execute(
+                    sql,
+                    (
+                        user_id,
+                        get_localpart_from_id(user_id),
+                        get_domain_from_id(user_id),
+                        display_name,
+                    ),
+                )
             elif isinstance(self.database_engine, Sqlite3Engine):
                 value = "%s %s" % (user_id, display_name) if display_name else user_id
                 self.db_pool.simple_upsert_txn(
diff --git a/synapse/storage/databases/main/user_erasure_store.py b/synapse/storage/databases/main/user_erasure_store.py
index f9575b1f1f..1ecdd40c38 100644
--- a/synapse/storage/databases/main/user_erasure_store.py
+++ b/synapse/storage/databases/main/user_erasure_store.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Dict, Iterable
+
 from synapse.storage._base import SQLBaseStore
 from synapse.util.caches.descriptors import cached, cachedList
 
@@ -38,21 +39,16 @@ async def is_user_erased(self, user_id: str) -> bool:
         return bool(result)
 
     @cachedList(cached_method_name="is_user_erased", list_name="user_ids")
-    async def are_users_erased(self, user_ids):
+    async def are_users_erased(self, user_ids: Iterable[str]) -> Dict[str, bool]:
         """
         Checks which users in a list have requested erasure
 
         Args:
-            user_ids (iterable[str]): full user id to check
+            user_ids: full user ids to check
 
         Returns:
-            dict[str, bool]:
-                for each user, whether the user has requested erasure.
+            for each user, whether the user has requested erasure.
         """
-        # this serves the dual purpose of (a) making sure we can do len and
-        # iterate it multiple times, and (b) avoiding duplicates.
-        user_ids = tuple(set(user_ids))
-
         rows = await self.db_pool.simple_select_many_batch(
             table="erased_users",
             column="user_id",
diff --git a/synapse/storage/databases/state/__init__.py b/synapse/storage/databases/state/__init__.py
index c90d022899..e5100d6108 100644
--- a/synapse/storage/databases/state/__init__.py
+++ b/synapse/storage/databases/state/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py
index 1fd333b707..c2891cb07f 100644
--- a/synapse/storage/databases/state/bg_updates.py
+++ b/synapse/storage/databases/state/bg_updates.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +13,7 @@
 # limitations under the License.
 
 import logging
+from typing import Optional
 
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.database import DatabasePool
@@ -73,8 +73,10 @@ def _count_state_group_hops_txn(self, txn, state_group):
             return count
 
     def _get_state_groups_from_groups_txn(
-        self, txn, groups, state_filter=StateFilter.all()
+        self, txn, groups, state_filter: Optional[StateFilter] = None
     ):
+        state_filter = state_filter or StateFilter.all()
+
         results = {group: {} for group in groups}
 
         where_clause, where_args = state_filter.make_sql_filter_clause()
diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py
index 97ec65f757..f839c0c24f 100644
--- a/synapse/storage/databases/state/store.py
+++ b/synapse/storage/databases/state/store.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +14,7 @@
 
 import logging
 from collections import namedtuple
-from typing import Dict, Iterable, List, Set, Tuple
+from typing import Dict, Iterable, List, Optional, Set, Tuple
 
 from synapse.api.constants import EventTypes
 from synapse.storage._base import SQLBaseStore
@@ -210,7 +209,7 @@ def _get_state_for_group_using_cache(self, cache, group, state_filter):
         return state_filter.filter_state(state_dict_ids), not missing_types
 
     async def _get_state_for_groups(
-        self, groups: Iterable[int], state_filter: StateFilter = StateFilter.all()
+        self, groups: Iterable[int], state_filter: Optional[StateFilter] = None
     ) -> Dict[int, MutableStateMap[str]]:
         """Gets the state at each of a list of state groups, optionally
         filtering by type/state_key
@@ -223,6 +222,7 @@ async def _get_state_for_groups(
         Returns:
             Dict of state group to state map.
         """
+        state_filter = state_filter or StateFilter.all()
 
         member_filter, non_member_filter = state_filter.get_member_split()
 
@@ -372,18 +372,23 @@ def _insert_into_cache(
             )
 
     async def store_state_group(
-        self, event_id, room_id, prev_group, delta_ids, current_state_ids
+        self,
+        event_id: str,
+        room_id: str,
+        prev_group: Optional[int],
+        delta_ids: Optional[StateMap[str]],
+        current_state_ids: StateMap[str],
     ) -> int:
         """Store a new set of state, returning a newly assigned state group.
 
         Args:
-            event_id (str): The event ID for which the state was calculated
-            room_id (str)
-            prev_group (int|None): A previous state group for the room, optional.
-            delta_ids (dict|None): The delta between state at `prev_group` and
+            event_id: The event ID for which the state was calculated
+            room_id
+            prev_group: A previous state group for the room, optional.
+            delta_ids: The delta between state at `prev_group` and
                 `current_state_ids`, if `prev_group` was given. Same format as
                 `current_state_ids`.
-            current_state_ids (dict): The state to store. Map of (type, state_key)
+            current_state_ids: The state to store. Map of (type, state_key)
                 to event_id.
 
         Returns:
diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
index d15ccfacde..9abc02046e 100644
--- a/synapse/storage/engines/__init__.py
+++ b/synapse/storage/engines/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py
index cca839c70f..20cd63c330 100644
--- a/synapse/storage/engines/_base.py
+++ b/synapse/storage/engines/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -44,18 +43,16 @@ def can_native_upsert(self) -> bool:
 
     @property
     @abc.abstractmethod
-    def supports_tuple_comparison(self) -> bool:
+    def supports_using_any_list(self) -> bool:
         """
-        Do we support comparing tuples, i.e. `(a, b) > (c, d)`?
+        Do we support using `a = ANY(?)` and passing a list
         """
         ...
 
     @property
     @abc.abstractmethod
-    def supports_using_any_list(self) -> bool:
-        """
-        Do we support using `a = ANY(?)` and passing a list
-        """
+    def supports_returning(self) -> bool:
+        """Do we support the `RETURNING` clause in insert/update/delete?"""
         ...
 
     @abc.abstractmethod
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index 80a3558aec..30f948a0f7 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -47,8 +46,8 @@ def check_database(self, db_conn, allow_outdated_version: bool = False):
         self._version = db_conn.server_version
 
         # Are we on a supported PostgreSQL version?
-        if not allow_outdated_version and self._version < 90500:
-            raise RuntimeError("Synapse requires PostgreSQL 9.5+ or above.")
+        if not allow_outdated_version and self._version < 90600:
+            raise RuntimeError("Synapse requires PostgreSQL 9.6 or above.")
 
         with db_conn.cursor() as txn:
             txn.execute("SHOW SERVER_ENCODING")
@@ -130,15 +129,13 @@ def can_native_upsert(self):
         return True
 
     @property
-    def supports_tuple_comparison(self):
-        """
-        Do we support comparing tuples, i.e. `(a, b) > (c, d)`?
-        """
+    def supports_using_any_list(self):
+        """Do we support using `a = ANY(?)` and passing a list"""
         return True
 
     @property
-    def supports_using_any_list(self):
-        """Do we support using `a = ANY(?)` and passing a list"""
+    def supports_returning(self) -> bool:
+        """Do we support the `RETURNING` clause in insert/update/delete?"""
         return True
 
     def is_deadlock(self, error):
diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py
index b87e7798da..70d17d4f2c 100644
--- a/synapse/storage/engines/sqlite.py
+++ b/synapse/storage/engines/sqlite.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -56,24 +55,24 @@ def can_native_upsert(self):
         """
         return self.module.sqlite_version_info >= (3, 24, 0)
 
-    @property
-    def supports_tuple_comparison(self):
-        """
-        Do we support comparing tuples, i.e. `(a, b) > (c, d)`? This requires
-        SQLite 3.15+.
-        """
-        return self.module.sqlite_version_info >= (3, 15, 0)
-
     @property
     def supports_using_any_list(self):
         """Do we support using `a = ANY(?)` and passing a list"""
         return False
 
+    @property
+    def supports_returning(self) -> bool:
+        """Do we support the `RETURNING` clause in insert/update/delete?"""
+        return self.module.sqlite_version_info >= (3, 35, 0)
+
     def check_database(self, db_conn, allow_outdated_version: bool = False):
         if not allow_outdated_version:
             version = self.module.sqlite_version_info
-            if version < (3, 11, 0):
-                raise RuntimeError("Synapse requires sqlite 3.11 or above.")
+            # Synapse is untested against older SQLite versions, and we don't want
+            # to let users upgrade to a version of Synapse with broken support for their
+            # sqlite version, because it risks leaving them with a half-upgraded db.
+            if version < (3, 22, 0):
+                raise RuntimeError("Synapse requires sqlite 3.22 or above.")
 
     def check_new_database(self, txn):
         """Gets called when setting up a brand new database. This allows us to
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index c03871f393..540adb8781 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd.
 #
diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py
index 3a0d6fb32e..0e8270746d 100644
--- a/synapse/storage/persist_events.py
+++ b/synapse/storage/persist_events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018-2019 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -17,9 +16,24 @@
 
 import itertools
 import logging
-from collections import deque, namedtuple
-from typing import Dict, Iterable, List, Optional, Set, Tuple
+from collections import deque
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Collection,
+    Deque,
+    Dict,
+    Generic,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    TypeVar,
+)
 
+import attr
 from prometheus_client import Counter, Histogram
 
 from twisted.internet import defer
@@ -27,19 +41,19 @@
 from synapse.api.constants import EventTypes, Membership
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
+from synapse.logging import opentracing
 from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.databases import Databases
 from synapse.storage.databases.main.events import DeltaState
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.types import (
-    Collection,
     PersistedEventPosition,
     RoomStreamToken,
     StateMap,
     get_domain_from_id,
 )
-from synapse.util.async_helpers import ObservableDeferred
+from synapse.util.async_helpers import ObservableDeferred, yieldable_gather_results
 from synapse.util.metrics import Measure
 
 logger = logging.getLogger(__name__)
@@ -91,25 +105,53 @@
 )
 
 
-class _EventPeristenceQueue:
+@attr.s(auto_attribs=True, slots=True)
+class _EventPersistQueueItem:
+    events_and_contexts: List[Tuple[EventBase, EventContext]]
+    backfilled: bool
+    deferred: ObservableDeferred
+
+    parent_opentracing_span_contexts: List = attr.ib(factory=list)
+    """A list of opentracing spans waiting for this batch"""
+
+    opentracing_span_context: Any = None
+    """The opentracing span under which the persistence actually happened"""
+
+
+_PersistResult = TypeVar("_PersistResult")
+
+
+class _EventPeristenceQueue(Generic[_PersistResult]):
     """Queues up events so that they can be persisted in bulk with only one
     concurrent transaction per room.
     """
 
-    _EventPersistQueueItem = namedtuple(
-        "_EventPersistQueueItem", ("events_and_contexts", "backfilled", "deferred")
-    )
+    def __init__(
+        self,
+        per_item_callback: Callable[
+            [List[Tuple[EventBase, EventContext]], bool],
+            Awaitable[_PersistResult],
+        ],
+    ):
+        """Create a new event persistence queue
 
-    def __init__(self):
-        self._event_persist_queues = {}
-        self._currently_persisting_rooms = set()
+        The per_item_callback will be called for each item added via add_to_queue,
+        and its result will be returned via the Deferreds returned from add_to_queue.
+        """
+        self._event_persist_queues: Dict[str, Deque[_EventPersistQueueItem]] = {}
+        self._currently_persisting_rooms: Set[str] = set()
+        self._per_item_callback = per_item_callback
 
-    def add_to_queue(self, room_id, events_and_contexts, backfilled):
+    async def add_to_queue(
+        self,
+        room_id: str,
+        events_and_contexts: Iterable[Tuple[EventBase, EventContext]],
+        backfilled: bool,
+    ) -> _PersistResult:
         """Add events to the queue, with the given persist_event options.
 
-        NB: due to the normal usage pattern of this method, it does *not*
-        follow the synapse logcontext rules, and leaves the logcontext in
-        place whether or not the returned deferred is ready.
+        If we are not already processing events in this room, starts off a background
+        process to to so, calling the per_item_callback for each item.
 
         Args:
             room_id (str):
@@ -117,38 +159,56 @@ def add_to_queue(self, room_id, events_and_contexts, backfilled):
             backfilled (bool):
 
         Returns:
-            defer.Deferred: a deferred which will resolve once the events are
-            persisted. Runs its callbacks *without* a logcontext. The result
-            is the same as that returned by the callback passed to
-            `handle_queue`.
+            the result returned by the `_per_item_callback` passed to
+            `__init__`.
         """
         queue = self._event_persist_queues.setdefault(room_id, deque())
-        if queue:
-            # if the last item in the queue has the same `backfilled` setting,
-            # we can just add these new events to that item.
-            end_item = queue[-1]
-            if end_item.backfilled == backfilled:
-                end_item.events_and_contexts.extend(events_and_contexts)
-                return end_item.deferred.observe()
 
-        deferred = ObservableDeferred(defer.Deferred(), consumeErrors=True)
+        # if the last item in the queue has the same `backfilled` setting,
+        # we can just add these new events to that item.
+        if queue and queue[-1].backfilled == backfilled:
+            end_item = queue[-1]
+        else:
+            # need to make a new queue item
+            deferred: ObservableDeferred[_PersistResult] = ObservableDeferred(
+                defer.Deferred(), consumeErrors=True
+            )
 
-        queue.append(
-            self._EventPersistQueueItem(
-                events_and_contexts=events_and_contexts,
+            end_item = _EventPersistQueueItem(
+                events_and_contexts=[],
                 backfilled=backfilled,
                 deferred=deferred,
             )
-        )
+            queue.append(end_item)
+
+        # add our events to the queue item
+        end_item.events_and_contexts.extend(events_and_contexts)
+
+        # also add our active opentracing span to the item so that we get a link back
+        span = opentracing.active_span()
+        if span:
+            end_item.parent_opentracing_span_contexts.append(span.context)
+
+        # start a processor for the queue, if there isn't one already
+        self._handle_queue(room_id)
 
-        return deferred.observe()
+        # wait for the queue item to complete
+        res = await make_deferred_yieldable(end_item.deferred.observe())
 
-    def handle_queue(self, room_id, per_item_callback):
+        # add another opentracing span which links to the persist trace.
+        with opentracing.start_active_span_follows_from(
+            "persist_event_batch_complete", (end_item.opentracing_span_context,)
+        ):
+            pass
+
+        return res
+
+    def _handle_queue(self, room_id):
         """Attempts to handle the queue for a room if not already being handled.
 
-        The given callback will be invoked with for each item in the queue,
+        The queue's callback will be invoked with for each item in the queue,
         of type _EventPersistQueueItem. The per_item_callback will continuously
-        be called with new items, unless the queue becomnes empty. The return
+        be called with new items, unless the queue becomes empty. The return
         value of the function will be given to the deferreds waiting on the item,
         exceptions will be passed to the deferreds as well.
 
@@ -158,7 +218,6 @@ def handle_queue(self, room_id, per_item_callback):
         If another callback is currently handling the queue then it will not be
         invoked.
         """
-
         if room_id in self._currently_persisting_rooms:
             return
 
@@ -169,7 +228,17 @@ async def handle_queue_loop():
                 queue = self._get_drainining_queue(room_id)
                 for item in queue:
                     try:
-                        ret = await per_item_callback(item)
+                        with opentracing.start_active_span_follows_from(
+                            "persist_event_batch",
+                            item.parent_opentracing_span_contexts,
+                            inherit_force_tracing=True,
+                        ) as scope:
+                            if scope:
+                                item.opentracing_span_context = scope.span.context
+
+                            ret = await self._per_item_callback(
+                                item.events_and_contexts, item.backfilled
+                            )
                     except Exception:
                         with PreserveLoggingContext():
                             item.deferred.errback()
@@ -216,9 +285,10 @@ def __init__(self, hs, stores: Databases):
         self._clock = hs.get_clock()
         self._instance_name = hs.get_instance_name()
         self.is_mine_id = hs.is_mine_id
-        self._event_persist_queue = _EventPeristenceQueue()
+        self._event_persist_queue = _EventPeristenceQueue(self._persist_event_batch)
         self._state_resolution_handler = hs.get_state_resolution_handler()
 
+    @opentracing.trace
     async def persist_events(
         self,
         events_and_contexts: Iterable[Tuple[EventBase, EventContext]],
@@ -239,30 +309,25 @@ async def persist_events(
             matched the transcation ID; the existing event is returned in such
             a case.
         """
-        partitioned = {}  # type: Dict[str, List[Tuple[EventBase, EventContext]]]
+        partitioned: Dict[str, List[Tuple[EventBase, EventContext]]] = {}
         for event, ctx in events_and_contexts:
             partitioned.setdefault(event.room_id, []).append((event, ctx))
 
-        deferreds = []
-        for room_id, evs_ctxs in partitioned.items():
-            d = self._event_persist_queue.add_to_queue(
+        async def enqueue(item):
+            room_id, evs_ctxs = item
+            return await self._event_persist_queue.add_to_queue(
                 room_id, evs_ctxs, backfilled=backfilled
             )
-            deferreds.append(d)
 
-        for room_id in partitioned:
-            self._maybe_start_persisting(room_id)
+        ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
 
-        # Each deferred returns a map from event ID to existing event ID if the
-        # event was deduplicated. (The dict may also include other entries if
+        # Each call to add_to_queue returns a map from event ID to existing event ID if
+        # the event was deduplicated. (The dict may also include other entries if
         # the event was persisted in a batch with other events).
         #
-        # Since we use `defer.gatherResults` we need to merge the returned list
+        # Since we use `yieldable_gather_results` we need to merge the returned list
         # of dicts into one.
-        ret_vals = await make_deferred_yieldable(
-            defer.gatherResults(deferreds, consumeErrors=True)
-        )
-        replaced_events = {}
+        replaced_events: Dict[str, str] = {}
         for d in ret_vals:
             replaced_events.update(d)
 
@@ -279,6 +344,7 @@ async def persist_events(
             self.main_store.get_room_max_token(),
         )
 
+    @opentracing.trace
     async def persist_event(
         self, event: EventBase, context: EventContext, backfilled: bool = False
     ) -> Tuple[EventBase, PersistedEventPosition, RoomStreamToken]:
@@ -289,16 +355,12 @@ async def persist_event(
             event if it was deduplicated due to an existing event matching the
             transaction ID.
         """
-        deferred = self._event_persist_queue.add_to_queue(
-            event.room_id, [(event, context)], backfilled=backfilled
-        )
-
-        self._maybe_start_persisting(event.room_id)
-
-        # The deferred returns a map from event ID to existing event ID if the
+        # add_to_queue returns a map from event ID to existing event ID if the
         # event was deduplicated. (The dict may also include other entries if
         # the event was persisted in a batch with other events.)
-        replaced_events = await make_deferred_yieldable(deferred)
+        replaced_events = await self._event_persist_queue.add_to_queue(
+            event.room_id, [(event, context)], backfilled=backfilled
+        )
         replaced_event = replaced_events.get(event.event_id)
         if replaced_event:
             event = await self.main_store.get_event(replaced_event)
@@ -310,36 +372,21 @@ async def persist_event(
         pos = PersistedEventPosition(self._instance_name, event_stream_id)
         return event, pos, self.main_store.get_room_max_token()
 
-    def _maybe_start_persisting(self, room_id: str):
-        """Pokes the `_event_persist_queue` to start handling new items in the
-        queue, if not already in progress.
-
-        Causes the deferreds returned by `add_to_queue` to resolve with: a
-        dictionary of event ID to event ID we didn't persist as we already had
-        another event persisted with the same TXN ID.
-        """
-
-        async def persisting_queue(item):
-            with Measure(self._clock, "persist_events"):
-                return await self._persist_events(
-                    item.events_and_contexts, backfilled=item.backfilled
-                )
-
-        self._event_persist_queue.handle_queue(room_id, persisting_queue)
-
-    async def _persist_events(
+    async def _persist_event_batch(
         self,
         events_and_contexts: List[Tuple[EventBase, EventContext]],
         backfilled: bool = False,
     ) -> Dict[str, str]:
-        """Calculates the change to current state and forward extremities, and
+        """Callback for the _event_persist_queue
+
+        Calculates the change to current state and forward extremities, and
         persists the given events and with those updates.
 
         Returns:
             A dictionary of event ID to event ID we didn't persist as we already
             had another event persisted with the same TXN ID.
         """
-        replaced_events = {}  # type: Dict[str, str]
+        replaced_events: Dict[str, str] = {}
         if not events_and_contexts:
             return replaced_events
 
@@ -395,16 +442,14 @@ async def _persist_events(
             # Set of remote users which were in rooms the server has left. We
             # should check if we still share any rooms and if not we mark their
             # device lists as stale.
-            potentially_left_users = set()  # type: Set[str]
+            potentially_left_users: Set[str] = set()
 
             if not backfilled:
                 with Measure(self._clock, "_calculate_state_and_extrem"):
                     # Work out the new "current state" for each room.
                     # We do this by working out what the new extremities are and then
                     # calculating the state from that.
-                    events_by_room = (
-                        {}
-                    )  # type: Dict[str, List[Tuple[EventBase, EventContext]]]
+                    events_by_room: Dict[str, List[Tuple[EventBase, EventContext]]] = {}
                     for event, context in chunk:
                         events_by_room.setdefault(event.room_id, []).append(
                             (event, context)
@@ -577,9 +622,9 @@ async def _calculate_new_extremities(
         )
 
         # Remove any events which are prev_events of any existing events.
-        existing_prevs = await self.persist_events_store._get_events_which_are_prevs(
-            result
-        )  # type: Collection[str]
+        existing_prevs: Collection[
+            str
+        ] = await self.persist_events_store._get_events_which_are_prevs(result)
         result.difference_update(existing_prevs)
 
         # Finally handle the case where the new events have soft-failed prev
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 6c3c2da520..61392b9639 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014 - 2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
+# Copyright 2014 - 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,12 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import imp
+import importlib.util
 import logging
 import os
 import re
 from collections import Counter
-from typing import Generator, Iterable, List, Optional, TextIO, Tuple
+from typing import Collection, Generator, Iterable, List, Optional, TextIO, Tuple
 
 import attr
 from typing_extensions import Counter as CounterType
@@ -27,17 +25,13 @@
 from synapse.storage.database import LoggingDatabaseConnection
 from synapse.storage.engines import BaseDatabaseEngine
 from synapse.storage.engines.postgres import PostgresEngine
+from synapse.storage.schema import SCHEMA_COMPAT_VERSION, SCHEMA_VERSION
 from synapse.storage.types import Cursor
-from synapse.types import Collection
 
 logger = logging.getLogger(__name__)
 
 
-# Remember to update this number every time a change is made to database
-# schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 59
-
-dir_path = os.path.abspath(os.path.dirname(__file__))
+schema_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "schema")
 
 
 class PrepareDatabaseException(Exception):
@@ -64,6 +58,28 @@ class UpgradeDatabaseException(PrepareDatabaseException):
 )
 
 
+@attr.s
+class _SchemaState:
+    current_version: int = attr.ib()
+    """The current schema version of the database"""
+
+    compat_version: Optional[int] = attr.ib()
+    """The SCHEMA_VERSION of the oldest version of Synapse for this database
+
+    If this is None, we have an old version of the database without the necessary
+    table.
+    """
+
+    applied_deltas: Collection[str] = attr.ib(factory=tuple)
+    """Any delta files for `current_version` which have already been applied"""
+
+    upgraded: bool = attr.ib(default=False)
+    """Whether the current state was reached by applying deltas.
+
+    If False, we have run the full schema for `current_version`, and have applied no
+    deltas since. If True, we have run some deltas since the original creation."""
+
+
 def prepare_database(
     db_conn: LoggingDatabaseConnection,
     database_engine: BaseDatabaseEngine,
@@ -101,12 +117,11 @@ def prepare_database(
         version_info = _get_or_create_schema_state(cur, database_engine)
 
         if version_info:
-            user_version, delta_files, upgraded = version_info
             logger.info(
                 "%r: Existing schema is %i (+%i deltas)",
                 databases,
-                user_version,
-                len(delta_files),
+                version_info.current_version,
+                len(version_info.applied_deltas),
             )
 
             # config should only be None when we are preparing an in-memory SQLite db,
@@ -118,16 +133,18 @@ def prepare_database(
 
             # if it's a worker app, refuse to upgrade the database, to avoid multiple
             # workers doing it at once.
-            if config.worker_app is not None and user_version != SCHEMA_VERSION:
+            if (
+                config.worker_app is not None
+                and version_info.current_version != SCHEMA_VERSION
+            ):
                 raise UpgradeDatabaseException(
-                    OUTDATED_SCHEMA_ON_WORKER_ERROR % (SCHEMA_VERSION, user_version)
+                    OUTDATED_SCHEMA_ON_WORKER_ERROR
+                    % (SCHEMA_VERSION, version_info.current_version)
                 )
 
             _upgrade_existing_database(
                 cur,
-                user_version,
-                delta_files,
-                upgraded,
+                version_info,
                 database_engine,
                 config,
                 databases=databases,
@@ -169,7 +186,14 @@ def _setup_new_database(
 
     Example directory structure:
 
-        schema/
+    schema/
+        common/
+            delta/
+                ...
+            full_schemas/
+                11/
+                    foo.sql
+        main/
             delta/
                 ...
             full_schemas/
@@ -177,15 +201,14 @@ def _setup_new_database(
                     test.sql
                     ...
                 11/
-                    foo.sql
                     bar.sql
                 ...
 
     In the example foo.sql and bar.sql would be run, and then any delta files
     for versions strictly greater than 11.
 
-    Note: we apply the full schemas and deltas from the top level `schema/`
-    folder as well those in the data stores specified.
+    Note: we apply the full schemas and deltas from the `schema/common`
+    folder as well those in the databases specified.
 
     Args:
         cur: a database cursor
@@ -197,12 +220,12 @@ def _setup_new_database(
     # configured to our liking.
     database_engine.check_new_database(cur)
 
-    current_dir = os.path.join(dir_path, "schema", "full_schemas")
+    full_schemas_dir = os.path.join(schema_path, "common", "full_schemas")
 
     # First we find the highest full schema version we have
     valid_versions = []
 
-    for filename in os.listdir(current_dir):
+    for filename in os.listdir(full_schemas_dir):
         try:
             ver = int(filename)
         except ValueError:
@@ -220,22 +243,20 @@ def _setup_new_database(
 
     logger.debug("Initialising schema v%d", max_current_ver)
 
-    # Now lets find all the full schema files, both in the global schema and
-    # in data store schemas.
-    directories = [os.path.join(current_dir, str(max_current_ver))]
+    # Now let's find all the full schema files, both in the common schema and
+    # in database schemas.
+    directories = [os.path.join(full_schemas_dir, str(max_current_ver))]
     directories.extend(
         os.path.join(
-            dir_path,
-            "databases",
+            schema_path,
             database,
-            "schema",
             "full_schemas",
             str(max_current_ver),
         )
         for database in databases
     )
 
-    directory_entries = []  # type: List[_DirectoryListing]
+    directory_entries: List[_DirectoryListing] = []
     for directory in directories:
         directory_entries.extend(
             _DirectoryListing(file_name, os.path.join(directory, file_name))
@@ -262,9 +283,7 @@ def _setup_new_database(
 
     _upgrade_existing_database(
         cur,
-        current_version=max_current_ver,
-        applied_delta_files=[],
-        upgraded=False,
+        _SchemaState(current_version=max_current_ver, compat_version=None),
         database_engine=database_engine,
         config=None,
         databases=databases,
@@ -274,9 +293,7 @@ def _setup_new_database(
 
 def _upgrade_existing_database(
     cur: Cursor,
-    current_version: int,
-    applied_delta_files: List[str],
-    upgraded: bool,
+    current_schema_state: _SchemaState,
     database_engine: BaseDatabaseEngine,
     config: Optional[HomeServerConfig],
     databases: Collection[str],
@@ -322,12 +339,8 @@ def _upgrade_existing_database(
 
     Args:
         cur
-        current_version: The current version of the schema.
-        applied_delta_files: A list of deltas that have already been applied.
-        upgraded: Whether the current version was generated by having
-            applied deltas or from full schema file. If `True` the function
-            will never apply delta files for the given `current_version`, since
-            the current_version wasn't generated by applying those delta files.
+        current_schema_state: The current version of the schema, as
+            returned by _get_or_create_schema_state
         database_engine
         config:
             None if we are initialising a blank database, otherwise the application
@@ -338,13 +351,16 @@ def _upgrade_existing_database(
             upgrade portions of the delta scripts.
     """
     if is_empty:
-        assert not applied_delta_files
+        assert not current_schema_state.applied_deltas
     else:
         assert config
 
     is_worker = config and config.worker_app is not None
 
-    if current_version > SCHEMA_VERSION:
+    if (
+        current_schema_state.compat_version is not None
+        and current_schema_state.compat_version > SCHEMA_VERSION
+    ):
         raise ValueError(
             "Cannot use this database as it is too "
             + "new for the server to understand"
@@ -358,11 +374,26 @@ def _upgrade_existing_database(
         assert config is not None
         check_database_before_upgrade(cur, database_engine, config)
 
-    start_ver = current_version
-    if not upgraded:
+    # update schema_compat_version before we run any upgrades, so that if synapse
+    # gets downgraded again, it won't try to run against the upgraded database.
+    if (
+        current_schema_state.compat_version is None
+        or current_schema_state.compat_version < SCHEMA_COMPAT_VERSION
+    ):
+        cur.execute("DELETE FROM schema_compat_version")
+        cur.execute(
+            "INSERT INTO schema_compat_version(compat_version) VALUES (?)",
+            (SCHEMA_COMPAT_VERSION,),
+        )
+
+    start_ver = current_schema_state.current_version
+
+    # if we got to this schema version by running a full_schema rather than a series
+    # of deltas, we should not run the deltas for this version.
+    if not current_schema_state.upgraded:
         start_ver += 1
 
-    logger.debug("applied_delta_files: %s", applied_delta_files)
+    logger.debug("applied_delta_files: %s", current_schema_state.applied_deltas)
 
     if isinstance(database_engine, PostgresEngine):
         specific_engine_extension = ".postgres"
@@ -387,18 +418,16 @@ def _upgrade_existing_database(
         # directories for schema updates.
 
         # First we find the directories to search in
-        delta_dir = os.path.join(dir_path, "schema", "delta", str(v))
+        delta_dir = os.path.join(schema_path, "common", "delta", str(v))
         directories = [delta_dir]
         for database in databases:
-            directories.append(
-                os.path.join(dir_path, "databases", database, "schema", "delta", str(v))
-            )
+            directories.append(os.path.join(schema_path, database, "delta", str(v)))
 
         # Used to check if we have any duplicate file names
-        file_name_counter = Counter()  # type: CounterType[str]
+        file_name_counter: CounterType[str] = Counter()
 
         # Now find which directories have anything of interest.
-        directory_entries = []  # type: List[_DirectoryListing]
+        directory_entries: List[_DirectoryListing] = []
         for directory in directories:
             logger.debug("Looking for schema deltas in %s", directory)
             try:
@@ -440,7 +469,7 @@ def _upgrade_existing_database(
             absolute_path = entry.absolute_path
 
             logger.debug("Found file: %s (%s)", relative_path, absolute_path)
-            if relative_path in applied_delta_files:
+            if relative_path in current_schema_state.applied_deltas:
                 continue
 
             root_name, ext = os.path.splitext(file_name)
@@ -454,8 +483,13 @@ def _upgrade_existing_database(
                     )
 
                 module_name = "synapse.storage.v%d_%s" % (v, root_name)
-                with open(absolute_path) as python_file:
-                    module = imp.load_source(module_name, absolute_path, python_file)  # type: ignore
+
+                spec = importlib.util.spec_from_file_location(
+                    module_name, absolute_path
+                )
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)  # type: ignore
+
                 logger.info("Running script %s", relative_path)
                 module.run_create(cur, database_engine)  # type: ignore
                 if not is_empty:
@@ -605,7 +639,7 @@ def get_statements(f: Iterable[str]) -> Generator[str, None, None]:
 
 
 def executescript(txn: Cursor, schema_path: str) -> None:
-    with open(schema_path, "r") as f:
+    with open(schema_path) as f:
         execute_statements_from_stream(txn, f)
 
 
@@ -616,25 +650,39 @@ def execute_statements_from_stream(cur: Cursor, f: TextIO) -> None:
 
 def _get_or_create_schema_state(
     txn: Cursor, database_engine: BaseDatabaseEngine
-) -> Optional[Tuple[int, List[str], bool]]:
+) -> Optional[_SchemaState]:
     # Bluntly try creating the schema_version tables.
-    schema_path = os.path.join(dir_path, "schema", "schema_version.sql")
-    executescript(txn, schema_path)
+    sql_path = os.path.join(schema_path, "common", "schema_version.sql")
+    executescript(txn, sql_path)
 
     txn.execute("SELECT version, upgraded FROM schema_version")
     row = txn.fetchone()
 
+    if row is None:
+        # new database
+        return None
+
+    current_version = int(row[0])
+    upgraded = bool(row[1])
+
+    compat_version: Optional[int] = None
+    txn.execute("SELECT compat_version FROM schema_compat_version")
+    row = txn.fetchone()
     if row is not None:
-        current_version = int(row[0])
-        txn.execute(
-            "SELECT file FROM applied_schema_deltas WHERE version >= ?",
-            (current_version,),
-        )
-        applied_deltas = [d for d, in txn]
-        upgraded = bool(row[1])
-        return current_version, applied_deltas, upgraded
+        compat_version = int(row[0])
 
-    return None
+    txn.execute(
+        "SELECT file FROM applied_schema_deltas WHERE version >= ?",
+        (current_version,),
+    )
+    applied_deltas = tuple(d for d, in txn)
+
+    return _SchemaState(
+        current_version=current_version,
+        compat_version=compat_version,
+        applied_deltas=applied_deltas,
+        upgraded=upgraded,
+    )
 
 
 @attr.s(slots=True)
diff --git a/synapse/storage/purge_events.py b/synapse/storage/purge_events.py
index ad954990a7..30669beb7c 100644
--- a/synapse/storage/purge_events.py
+++ b/synapse/storage/purge_events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index f47cec0d86..2d5c21ef72 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/storage/relations.py b/synapse/storage/relations.py
index 2564f34b47..c552dbf04c 100644
--- a/synapse/storage/relations.py
+++ b/synapse/storage/relations.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index d2ff4da6b9..c34fbf21bc 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/synapse/storage/schema/README.md b/synapse/storage/schema/README.md
new file mode 100644
index 0000000000..4fc2061a3d
--- /dev/null
+++ b/synapse/storage/schema/README.md
@@ -0,0 +1,4 @@
+# Synapse Database Schemas
+
+This directory contains the schema files used to build Synapse databases. For more
+information, see https://matrix-org.github.io/synapse/develop/development/database_schema.html.
diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py
new file mode 100644
index 0000000000..a5bc0ee8a5
--- /dev/null
+++ b/synapse/storage/schema/__init__.py
@@ -0,0 +1,41 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SCHEMA_VERSION = 63
+"""Represents the expectations made by the codebase about the database schema
+
+This should be incremented whenever the codebase changes its requirements on the
+shape of the database schema (even if those requirements are backwards-compatible with
+older versions of Synapse).
+
+See https://matrix-org.github.io/synapse/develop/development/database_schema.html
+for more information on how this works.
+
+Changes in SCHEMA_VERSION = 61:
+    - The `user_stats_historical` and `room_stats_historical` tables are not written and
+      are not read (previously, they were written but not read).
+
+Changes in SCHEMA_VERSION = 63:
+    - The `public_room_list_stream` table is not written nor read to
+      (previously, it was written and read to, but not for any significant purpose).
+      https://github.com/matrix-org/synapse/pull/10565
+"""
+
+
+SCHEMA_COMPAT_VERSION = 59
+"""Limit on how far the synapse codebase can be rolled back without breaking db compat
+
+This value is stored in the database, and checked on startup. If the value in the
+database is greater than SCHEMA_VERSION, then Synapse will refuse to start.
+"""
diff --git a/synapse/storage/schema/delta/25/00background_updates.sql b/synapse/storage/schema/common/delta/25/00background_updates.sql
similarity index 100%
rename from synapse/storage/schema/delta/25/00background_updates.sql
rename to synapse/storage/schema/common/delta/25/00background_updates.sql
diff --git a/synapse/storage/schema/delta/35/00background_updates_add_col.sql b/synapse/storage/schema/common/delta/35/00background_updates_add_col.sql
similarity index 100%
rename from synapse/storage/schema/delta/35/00background_updates_add_col.sql
rename to synapse/storage/schema/common/delta/35/00background_updates_add_col.sql
diff --git a/synapse/storage/schema/delta/58/00background_update_ordering.sql b/synapse/storage/schema/common/delta/58/00background_update_ordering.sql
similarity index 100%
rename from synapse/storage/schema/delta/58/00background_update_ordering.sql
rename to synapse/storage/schema/common/delta/58/00background_update_ordering.sql
diff --git a/synapse/storage/schema/full_schemas/54/full.sql b/synapse/storage/schema/common/full_schemas/54/full.sql
similarity index 100%
rename from synapse/storage/schema/full_schemas/54/full.sql
rename to synapse/storage/schema/common/full_schemas/54/full.sql
diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/common/schema_version.sql
similarity index 81%
rename from synapse/storage/schema/schema_version.sql
rename to synapse/storage/schema/common/schema_version.sql
index 42e5cb6df5..f41fde5d2d 100644
--- a/synapse/storage/schema/schema_version.sql
+++ b/synapse/storage/schema/common/schema_version.sql
@@ -20,6 +20,13 @@ CREATE TABLE IF NOT EXISTS schema_version(
     CHECK (Lock='X')
 );
 
+CREATE TABLE IF NOT EXISTS schema_compat_version(
+    Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE,  -- Makes sure this table only has one row.
+    -- The SCHEMA_VERSION of the oldest synapse this database can be used with
+    compat_version INTEGER NOT NULL,
+    CHECK (Lock='X')
+);
+
 CREATE TABLE IF NOT EXISTS applied_schema_deltas(
     version INTEGER NOT NULL,
     file TEXT NOT NULL,
diff --git a/synapse/storage/databases/main/schema/delta/12/v12.sql b/synapse/storage/schema/main/delta/12/v12.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/12/v12.sql
rename to synapse/storage/schema/main/delta/12/v12.sql
diff --git a/synapse/storage/databases/main/schema/delta/13/v13.sql b/synapse/storage/schema/main/delta/13/v13.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/13/v13.sql
rename to synapse/storage/schema/main/delta/13/v13.sql
diff --git a/synapse/storage/databases/main/schema/delta/14/v14.sql b/synapse/storage/schema/main/delta/14/v14.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/14/v14.sql
rename to synapse/storage/schema/main/delta/14/v14.sql
diff --git a/synapse/storage/databases/main/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/main/delta/15/appservice_txns.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/15/appservice_txns.sql
rename to synapse/storage/schema/main/delta/15/appservice_txns.sql
diff --git a/synapse/storage/databases/main/schema/delta/15/presence_indices.sql b/synapse/storage/schema/main/delta/15/presence_indices.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/15/presence_indices.sql
rename to synapse/storage/schema/main/delta/15/presence_indices.sql
diff --git a/synapse/storage/databases/main/schema/delta/15/v15.sql b/synapse/storage/schema/main/delta/15/v15.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/15/v15.sql
rename to synapse/storage/schema/main/delta/15/v15.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/events_order_index.sql b/synapse/storage/schema/main/delta/16/events_order_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/events_order_index.sql
rename to synapse/storage/schema/main/delta/16/events_order_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/remote_media_cache_index.sql b/synapse/storage/schema/main/delta/16/remote_media_cache_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/remote_media_cache_index.sql
rename to synapse/storage/schema/main/delta/16/remote_media_cache_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/remove_duplicates.sql b/synapse/storage/schema/main/delta/16/remove_duplicates.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/remove_duplicates.sql
rename to synapse/storage/schema/main/delta/16/remove_duplicates.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/room_alias_index.sql b/synapse/storage/schema/main/delta/16/room_alias_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/room_alias_index.sql
rename to synapse/storage/schema/main/delta/16/room_alias_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/main/delta/16/unique_constraints.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/unique_constraints.sql
rename to synapse/storage/schema/main/delta/16/unique_constraints.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/users.sql b/synapse/storage/schema/main/delta/16/users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/users.sql
rename to synapse/storage/schema/main/delta/16/users.sql
diff --git a/synapse/storage/databases/main/schema/delta/17/drop_indexes.sql b/synapse/storage/schema/main/delta/17/drop_indexes.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/17/drop_indexes.sql
rename to synapse/storage/schema/main/delta/17/drop_indexes.sql
diff --git a/synapse/storage/databases/main/schema/delta/17/server_keys.sql b/synapse/storage/schema/main/delta/17/server_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/17/server_keys.sql
rename to synapse/storage/schema/main/delta/17/server_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/17/user_threepids.sql b/synapse/storage/schema/main/delta/17/user_threepids.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/17/user_threepids.sql
rename to synapse/storage/schema/main/delta/17/user_threepids.sql
diff --git a/synapse/storage/databases/main/schema/delta/18/server_keys_bigger_ints.sql b/synapse/storage/schema/main/delta/18/server_keys_bigger_ints.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/18/server_keys_bigger_ints.sql
rename to synapse/storage/schema/main/delta/18/server_keys_bigger_ints.sql
diff --git a/synapse/storage/databases/main/schema/delta/19/event_index.sql b/synapse/storage/schema/main/delta/19/event_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/19/event_index.sql
rename to synapse/storage/schema/main/delta/19/event_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/20/dummy.sql b/synapse/storage/schema/main/delta/20/dummy.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/20/dummy.sql
rename to synapse/storage/schema/main/delta/20/dummy.sql
diff --git a/synapse/storage/databases/main/schema/delta/20/pushers.py b/synapse/storage/schema/main/delta/20/pushers.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/20/pushers.py
rename to synapse/storage/schema/main/delta/20/pushers.py
diff --git a/synapse/storage/databases/main/schema/delta/21/end_to_end_keys.sql b/synapse/storage/schema/main/delta/21/end_to_end_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/21/end_to_end_keys.sql
rename to synapse/storage/schema/main/delta/21/end_to_end_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/21/receipts.sql b/synapse/storage/schema/main/delta/21/receipts.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/21/receipts.sql
rename to synapse/storage/schema/main/delta/21/receipts.sql
diff --git a/synapse/storage/databases/main/schema/delta/22/receipts_index.sql b/synapse/storage/schema/main/delta/22/receipts_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/22/receipts_index.sql
rename to synapse/storage/schema/main/delta/22/receipts_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/22/user_threepids_unique.sql b/synapse/storage/schema/main/delta/22/user_threepids_unique.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/22/user_threepids_unique.sql
rename to synapse/storage/schema/main/delta/22/user_threepids_unique.sql
diff --git a/synapse/storage/databases/main/schema/delta/24/stats_reporting.sql b/synapse/storage/schema/main/delta/24/stats_reporting.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/24/stats_reporting.sql
rename to synapse/storage/schema/main/delta/24/stats_reporting.sql
diff --git a/synapse/storage/databases/main/schema/delta/25/fts.py b/synapse/storage/schema/main/delta/25/fts.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/25/fts.py
rename to synapse/storage/schema/main/delta/25/fts.py
diff --git a/synapse/storage/databases/main/schema/delta/25/guest_access.sql b/synapse/storage/schema/main/delta/25/guest_access.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/25/guest_access.sql
rename to synapse/storage/schema/main/delta/25/guest_access.sql
diff --git a/synapse/storage/databases/main/schema/delta/25/history_visibility.sql b/synapse/storage/schema/main/delta/25/history_visibility.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/25/history_visibility.sql
rename to synapse/storage/schema/main/delta/25/history_visibility.sql
diff --git a/synapse/storage/databases/main/schema/delta/25/tags.sql b/synapse/storage/schema/main/delta/25/tags.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/25/tags.sql
rename to synapse/storage/schema/main/delta/25/tags.sql
diff --git a/synapse/storage/databases/main/schema/delta/26/account_data.sql b/synapse/storage/schema/main/delta/26/account_data.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/26/account_data.sql
rename to synapse/storage/schema/main/delta/26/account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/27/account_data.sql b/synapse/storage/schema/main/delta/27/account_data.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/27/account_data.sql
rename to synapse/storage/schema/main/delta/27/account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/27/forgotten_memberships.sql b/synapse/storage/schema/main/delta/27/forgotten_memberships.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/27/forgotten_memberships.sql
rename to synapse/storage/schema/main/delta/27/forgotten_memberships.sql
diff --git a/synapse/storage/databases/main/schema/delta/27/ts.py b/synapse/storage/schema/main/delta/27/ts.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/27/ts.py
rename to synapse/storage/schema/main/delta/27/ts.py
diff --git a/synapse/storage/databases/main/schema/delta/28/event_push_actions.sql b/synapse/storage/schema/main/delta/28/event_push_actions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/event_push_actions.sql
rename to synapse/storage/schema/main/delta/28/event_push_actions.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/events_room_stream.sql b/synapse/storage/schema/main/delta/28/events_room_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/events_room_stream.sql
rename to synapse/storage/schema/main/delta/28/events_room_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/public_roms_index.sql b/synapse/storage/schema/main/delta/28/public_roms_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/public_roms_index.sql
rename to synapse/storage/schema/main/delta/28/public_roms_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/receipts_user_id_index.sql b/synapse/storage/schema/main/delta/28/receipts_user_id_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/receipts_user_id_index.sql
rename to synapse/storage/schema/main/delta/28/receipts_user_id_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/upgrade_times.sql b/synapse/storage/schema/main/delta/28/upgrade_times.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/upgrade_times.sql
rename to synapse/storage/schema/main/delta/28/upgrade_times.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/users_is_guest.sql b/synapse/storage/schema/main/delta/28/users_is_guest.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/users_is_guest.sql
rename to synapse/storage/schema/main/delta/28/users_is_guest.sql
diff --git a/synapse/storage/databases/main/schema/delta/29/push_actions.sql b/synapse/storage/schema/main/delta/29/push_actions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/29/push_actions.sql
rename to synapse/storage/schema/main/delta/29/push_actions.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/alias_creator.sql b/synapse/storage/schema/main/delta/30/alias_creator.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/alias_creator.sql
rename to synapse/storage/schema/main/delta/30/alias_creator.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/as_users.py b/synapse/storage/schema/main/delta/30/as_users.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/as_users.py
rename to synapse/storage/schema/main/delta/30/as_users.py
diff --git a/synapse/storage/databases/main/schema/delta/30/deleted_pushers.sql b/synapse/storage/schema/main/delta/30/deleted_pushers.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/deleted_pushers.sql
rename to synapse/storage/schema/main/delta/30/deleted_pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/presence_stream.sql b/synapse/storage/schema/main/delta/30/presence_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/presence_stream.sql
rename to synapse/storage/schema/main/delta/30/presence_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/public_rooms.sql b/synapse/storage/schema/main/delta/30/public_rooms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/public_rooms.sql
rename to synapse/storage/schema/main/delta/30/public_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/push_rule_stream.sql b/synapse/storage/schema/main/delta/30/push_rule_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/push_rule_stream.sql
rename to synapse/storage/schema/main/delta/30/push_rule_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/threepid_guest_access_tokens.sql b/synapse/storage/schema/main/delta/30/threepid_guest_access_tokens.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/threepid_guest_access_tokens.sql
rename to synapse/storage/schema/main/delta/30/threepid_guest_access_tokens.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/invites.sql b/synapse/storage/schema/main/delta/31/invites.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/31/invites.sql
rename to synapse/storage/schema/main/delta/31/invites.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/local_media_repository_url_cache.sql b/synapse/storage/schema/main/delta/31/local_media_repository_url_cache.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/31/local_media_repository_url_cache.sql
rename to synapse/storage/schema/main/delta/31/local_media_repository_url_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/pushers.py b/synapse/storage/schema/main/delta/31/pushers.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/31/pushers.py
rename to synapse/storage/schema/main/delta/31/pushers.py
diff --git a/synapse/storage/databases/main/schema/delta/31/pushers_index.sql b/synapse/storage/schema/main/delta/31/pushers_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/31/pushers_index.sql
rename to synapse/storage/schema/main/delta/31/pushers_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/search_update.py b/synapse/storage/schema/main/delta/31/search_update.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/31/search_update.py
rename to synapse/storage/schema/main/delta/31/search_update.py
diff --git a/synapse/storage/databases/main/schema/delta/32/events.sql b/synapse/storage/schema/main/delta/32/events.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/32/events.sql
rename to synapse/storage/schema/main/delta/32/events.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/openid.sql b/synapse/storage/schema/main/delta/32/openid.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/32/openid.sql
rename to synapse/storage/schema/main/delta/32/openid.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/pusher_throttle.sql b/synapse/storage/schema/main/delta/32/pusher_throttle.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/32/pusher_throttle.sql
rename to synapse/storage/schema/main/delta/32/pusher_throttle.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/remove_indices.sql b/synapse/storage/schema/main/delta/32/remove_indices.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/32/remove_indices.sql
rename to synapse/storage/schema/main/delta/32/remove_indices.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/reports.sql b/synapse/storage/schema/main/delta/32/reports.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/32/reports.sql
rename to synapse/storage/schema/main/delta/32/reports.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/access_tokens_device_index.sql b/synapse/storage/schema/main/delta/33/access_tokens_device_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/access_tokens_device_index.sql
rename to synapse/storage/schema/main/delta/33/access_tokens_device_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/devices.sql b/synapse/storage/schema/main/delta/33/devices.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/devices.sql
rename to synapse/storage/schema/main/delta/33/devices.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys.sql b/synapse/storage/schema/main/delta/33/devices_for_e2e_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys.sql
rename to synapse/storage/schema/main/delta/33/devices_for_e2e_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys_clear_unknown_device.sql b/synapse/storage/schema/main/delta/33/devices_for_e2e_keys_clear_unknown_device.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys_clear_unknown_device.sql
rename to synapse/storage/schema/main/delta/33/devices_for_e2e_keys_clear_unknown_device.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/event_fields.py b/synapse/storage/schema/main/delta/33/event_fields.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/event_fields.py
rename to synapse/storage/schema/main/delta/33/event_fields.py
diff --git a/synapse/storage/databases/main/schema/delta/33/remote_media_ts.py b/synapse/storage/schema/main/delta/33/remote_media_ts.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/remote_media_ts.py
rename to synapse/storage/schema/main/delta/33/remote_media_ts.py
diff --git a/synapse/storage/databases/main/schema/delta/33/user_ips_index.sql b/synapse/storage/schema/main/delta/33/user_ips_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/user_ips_index.sql
rename to synapse/storage/schema/main/delta/33/user_ips_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/appservice_stream.sql b/synapse/storage/schema/main/delta/34/appservice_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/34/appservice_stream.sql
rename to synapse/storage/schema/main/delta/34/appservice_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/cache_stream.py b/synapse/storage/schema/main/delta/34/cache_stream.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/34/cache_stream.py
rename to synapse/storage/schema/main/delta/34/cache_stream.py
diff --git a/synapse/storage/databases/main/schema/delta/34/device_inbox.sql b/synapse/storage/schema/main/delta/34/device_inbox.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/34/device_inbox.sql
rename to synapse/storage/schema/main/delta/34/device_inbox.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/push_display_name_rename.sql b/synapse/storage/schema/main/delta/34/push_display_name_rename.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/34/push_display_name_rename.sql
rename to synapse/storage/schema/main/delta/34/push_display_name_rename.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/received_txn_purge.py b/synapse/storage/schema/main/delta/34/received_txn_purge.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/34/received_txn_purge.py
rename to synapse/storage/schema/main/delta/34/received_txn_purge.py
diff --git a/synapse/storage/databases/main/schema/delta/35/contains_url.sql b/synapse/storage/schema/main/delta/35/contains_url.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/contains_url.sql
rename to synapse/storage/schema/main/delta/35/contains_url.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/device_outbox.sql b/synapse/storage/schema/main/delta/35/device_outbox.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/device_outbox.sql
rename to synapse/storage/schema/main/delta/35/device_outbox.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/device_stream_id.sql b/synapse/storage/schema/main/delta/35/device_stream_id.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/device_stream_id.sql
rename to synapse/storage/schema/main/delta/35/device_stream_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/event_push_actions_index.sql b/synapse/storage/schema/main/delta/35/event_push_actions_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/event_push_actions_index.sql
rename to synapse/storage/schema/main/delta/35/event_push_actions_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/public_room_list_change_stream.sql b/synapse/storage/schema/main/delta/35/public_room_list_change_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/public_room_list_change_stream.sql
rename to synapse/storage/schema/main/delta/35/public_room_list_change_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/stream_order_to_extrem.sql b/synapse/storage/schema/main/delta/35/stream_order_to_extrem.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/stream_order_to_extrem.sql
rename to synapse/storage/schema/main/delta/35/stream_order_to_extrem.sql
diff --git a/synapse/storage/databases/main/schema/delta/36/readd_public_rooms.sql b/synapse/storage/schema/main/delta/36/readd_public_rooms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/36/readd_public_rooms.sql
rename to synapse/storage/schema/main/delta/36/readd_public_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/37/remove_auth_idx.py b/synapse/storage/schema/main/delta/37/remove_auth_idx.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/37/remove_auth_idx.py
rename to synapse/storage/schema/main/delta/37/remove_auth_idx.py
diff --git a/synapse/storage/databases/main/schema/delta/37/user_threepids.sql b/synapse/storage/schema/main/delta/37/user_threepids.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/37/user_threepids.sql
rename to synapse/storage/schema/main/delta/37/user_threepids.sql
diff --git a/synapse/storage/databases/main/schema/delta/38/postgres_fts_gist.sql b/synapse/storage/schema/main/delta/38/postgres_fts_gist.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/38/postgres_fts_gist.sql
rename to synapse/storage/schema/main/delta/38/postgres_fts_gist.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/appservice_room_list.sql b/synapse/storage/schema/main/delta/39/appservice_room_list.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/39/appservice_room_list.sql
rename to synapse/storage/schema/main/delta/39/appservice_room_list.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/device_federation_stream_idx.sql b/synapse/storage/schema/main/delta/39/device_federation_stream_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/39/device_federation_stream_idx.sql
rename to synapse/storage/schema/main/delta/39/device_federation_stream_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/event_push_index.sql b/synapse/storage/schema/main/delta/39/event_push_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/39/event_push_index.sql
rename to synapse/storage/schema/main/delta/39/event_push_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/federation_out_position.sql b/synapse/storage/schema/main/delta/39/federation_out_position.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/39/federation_out_position.sql
rename to synapse/storage/schema/main/delta/39/federation_out_position.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/membership_profile.sql b/synapse/storage/schema/main/delta/39/membership_profile.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/39/membership_profile.sql
rename to synapse/storage/schema/main/delta/39/membership_profile.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/current_state_idx.sql b/synapse/storage/schema/main/delta/40/current_state_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/40/current_state_idx.sql
rename to synapse/storage/schema/main/delta/40/current_state_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/device_inbox.sql b/synapse/storage/schema/main/delta/40/device_inbox.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/40/device_inbox.sql
rename to synapse/storage/schema/main/delta/40/device_inbox.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/device_list_streams.sql b/synapse/storage/schema/main/delta/40/device_list_streams.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/40/device_list_streams.sql
rename to synapse/storage/schema/main/delta/40/device_list_streams.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/event_push_summary.sql b/synapse/storage/schema/main/delta/40/event_push_summary.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/40/event_push_summary.sql
rename to synapse/storage/schema/main/delta/40/event_push_summary.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/pushers.sql b/synapse/storage/schema/main/delta/40/pushers.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/40/pushers.sql
rename to synapse/storage/schema/main/delta/40/pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/device_list_stream_idx.sql b/synapse/storage/schema/main/delta/41/device_list_stream_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/41/device_list_stream_idx.sql
rename to synapse/storage/schema/main/delta/41/device_list_stream_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/device_outbound_index.sql b/synapse/storage/schema/main/delta/41/device_outbound_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/41/device_outbound_index.sql
rename to synapse/storage/schema/main/delta/41/device_outbound_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/event_search_event_id_idx.sql b/synapse/storage/schema/main/delta/41/event_search_event_id_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/41/event_search_event_id_idx.sql
rename to synapse/storage/schema/main/delta/41/event_search_event_id_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/ratelimit.sql b/synapse/storage/schema/main/delta/41/ratelimit.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/41/ratelimit.sql
rename to synapse/storage/schema/main/delta/41/ratelimit.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/current_state_delta.sql b/synapse/storage/schema/main/delta/42/current_state_delta.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/42/current_state_delta.sql
rename to synapse/storage/schema/main/delta/42/current_state_delta.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/device_list_last_id.sql b/synapse/storage/schema/main/delta/42/device_list_last_id.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/42/device_list_last_id.sql
rename to synapse/storage/schema/main/delta/42/device_list_last_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/event_auth_state_only.sql b/synapse/storage/schema/main/delta/42/event_auth_state_only.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/42/event_auth_state_only.sql
rename to synapse/storage/schema/main/delta/42/event_auth_state_only.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/user_dir.py b/synapse/storage/schema/main/delta/42/user_dir.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/42/user_dir.py
rename to synapse/storage/schema/main/delta/42/user_dir.py
diff --git a/synapse/storage/databases/main/schema/delta/43/blocked_rooms.sql b/synapse/storage/schema/main/delta/43/blocked_rooms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/43/blocked_rooms.sql
rename to synapse/storage/schema/main/delta/43/blocked_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/43/quarantine_media.sql b/synapse/storage/schema/main/delta/43/quarantine_media.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/43/quarantine_media.sql
rename to synapse/storage/schema/main/delta/43/quarantine_media.sql
diff --git a/synapse/storage/databases/main/schema/delta/43/url_cache.sql b/synapse/storage/schema/main/delta/43/url_cache.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/43/url_cache.sql
rename to synapse/storage/schema/main/delta/43/url_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/43/user_share.sql b/synapse/storage/schema/main/delta/43/user_share.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/43/user_share.sql
rename to synapse/storage/schema/main/delta/43/user_share.sql
diff --git a/synapse/storage/databases/main/schema/delta/44/expire_url_cache.sql b/synapse/storage/schema/main/delta/44/expire_url_cache.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/44/expire_url_cache.sql
rename to synapse/storage/schema/main/delta/44/expire_url_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/45/group_server.sql b/synapse/storage/schema/main/delta/45/group_server.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/45/group_server.sql
rename to synapse/storage/schema/main/delta/45/group_server.sql
diff --git a/synapse/storage/databases/main/schema/delta/45/profile_cache.sql b/synapse/storage/schema/main/delta/45/profile_cache.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/45/profile_cache.sql
rename to synapse/storage/schema/main/delta/45/profile_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/drop_refresh_tokens.sql b/synapse/storage/schema/main/delta/46/drop_refresh_tokens.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/drop_refresh_tokens.sql
rename to synapse/storage/schema/main/delta/46/drop_refresh_tokens.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/drop_unique_deleted_pushers.sql b/synapse/storage/schema/main/delta/46/drop_unique_deleted_pushers.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/drop_unique_deleted_pushers.sql
rename to synapse/storage/schema/main/delta/46/drop_unique_deleted_pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/group_server.sql b/synapse/storage/schema/main/delta/46/group_server.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/group_server.sql
rename to synapse/storage/schema/main/delta/46/group_server.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/local_media_repository_url_idx.sql b/synapse/storage/schema/main/delta/46/local_media_repository_url_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/local_media_repository_url_idx.sql
rename to synapse/storage/schema/main/delta/46/local_media_repository_url_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/user_dir_null_room_ids.sql b/synapse/storage/schema/main/delta/46/user_dir_null_room_ids.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/user_dir_null_room_ids.sql
rename to synapse/storage/schema/main/delta/46/user_dir_null_room_ids.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/user_dir_typos.sql b/synapse/storage/schema/main/delta/46/user_dir_typos.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/user_dir_typos.sql
rename to synapse/storage/schema/main/delta/46/user_dir_typos.sql
diff --git a/synapse/storage/databases/main/schema/delta/47/last_access_media.sql b/synapse/storage/schema/main/delta/47/last_access_media.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/47/last_access_media.sql
rename to synapse/storage/schema/main/delta/47/last_access_media.sql
diff --git a/synapse/storage/databases/main/schema/delta/47/postgres_fts_gin.sql b/synapse/storage/schema/main/delta/47/postgres_fts_gin.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/47/postgres_fts_gin.sql
rename to synapse/storage/schema/main/delta/47/postgres_fts_gin.sql
diff --git a/synapse/storage/databases/main/schema/delta/47/push_actions_staging.sql b/synapse/storage/schema/main/delta/47/push_actions_staging.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/47/push_actions_staging.sql
rename to synapse/storage/schema/main/delta/47/push_actions_staging.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/add_user_consent.sql b/synapse/storage/schema/main/delta/48/add_user_consent.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/add_user_consent.sql
rename to synapse/storage/schema/main/delta/48/add_user_consent.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/add_user_ips_last_seen_index.sql b/synapse/storage/schema/main/delta/48/add_user_ips_last_seen_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/add_user_ips_last_seen_index.sql
rename to synapse/storage/schema/main/delta/48/add_user_ips_last_seen_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/deactivated_users.sql b/synapse/storage/schema/main/delta/48/deactivated_users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/deactivated_users.sql
rename to synapse/storage/schema/main/delta/48/deactivated_users.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/group_unique_indexes.py b/synapse/storage/schema/main/delta/48/group_unique_indexes.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/group_unique_indexes.py
rename to synapse/storage/schema/main/delta/48/group_unique_indexes.py
diff --git a/synapse/storage/databases/main/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/main/delta/48/groups_joinable.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/groups_joinable.sql
rename to synapse/storage/schema/main/delta/48/groups_joinable.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/profiles_batch.sql b/synapse/storage/schema/main/delta/48/profiles_batch.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/profiles_batch.sql
rename to synapse/storage/schema/main/delta/48/profiles_batch.sql
diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_consent_server_notice_sent.sql b/synapse/storage/schema/main/delta/49/add_user_consent_server_notice_sent.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/49/add_user_consent_server_notice_sent.sql
rename to synapse/storage/schema/main/delta/49/add_user_consent_server_notice_sent.sql
diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_daily_visits.sql b/synapse/storage/schema/main/delta/49/add_user_daily_visits.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/49/add_user_daily_visits.sql
rename to synapse/storage/schema/main/delta/49/add_user_daily_visits.sql
diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_ips_last_seen_only_index.sql b/synapse/storage/schema/main/delta/49/add_user_ips_last_seen_only_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/49/add_user_ips_last_seen_only_index.sql
rename to synapse/storage/schema/main/delta/49/add_user_ips_last_seen_only_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/50/add_creation_ts_users_index.sql b/synapse/storage/schema/main/delta/50/add_creation_ts_users_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/50/add_creation_ts_users_index.sql
rename to synapse/storage/schema/main/delta/50/add_creation_ts_users_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/50/erasure_store.sql b/synapse/storage/schema/main/delta/50/erasure_store.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/50/erasure_store.sql
rename to synapse/storage/schema/main/delta/50/erasure_store.sql
diff --git a/synapse/storage/databases/main/schema/delta/50/make_event_content_nullable.py b/synapse/storage/schema/main/delta/50/make_event_content_nullable.py
similarity index 99%
rename from synapse/storage/databases/main/schema/delta/50/make_event_content_nullable.py
rename to synapse/storage/schema/main/delta/50/make_event_content_nullable.py
index b1684a8441..acd6ad1e1f 100644
--- a/synapse/storage/databases/main/schema/delta/50/make_event_content_nullable.py
+++ b/synapse/storage/schema/main/delta/50/make_event_content_nullable.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/schema/delta/50/profiles_deactivated_users.sql b/synapse/storage/schema/main/delta/50/profiles_deactivated_users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/50/profiles_deactivated_users.sql
rename to synapse/storage/schema/main/delta/50/profiles_deactivated_users.sql
diff --git a/synapse/storage/databases/main/schema/delta/51/e2e_room_keys.sql b/synapse/storage/schema/main/delta/51/e2e_room_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/51/e2e_room_keys.sql
rename to synapse/storage/schema/main/delta/51/e2e_room_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/main/delta/51/monthly_active_users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/51/monthly_active_users.sql
rename to synapse/storage/schema/main/delta/51/monthly_active_users.sql
diff --git a/synapse/storage/databases/main/schema/delta/52/add_event_to_state_group_index.sql b/synapse/storage/schema/main/delta/52/add_event_to_state_group_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/52/add_event_to_state_group_index.sql
rename to synapse/storage/schema/main/delta/52/add_event_to_state_group_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/52/device_list_streams_unique_idx.sql b/synapse/storage/schema/main/delta/52/device_list_streams_unique_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/52/device_list_streams_unique_idx.sql
rename to synapse/storage/schema/main/delta/52/device_list_streams_unique_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/52/e2e_room_keys.sql b/synapse/storage/schema/main/delta/52/e2e_room_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/52/e2e_room_keys.sql
rename to synapse/storage/schema/main/delta/52/e2e_room_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/add_user_type_to_users.sql b/synapse/storage/schema/main/delta/53/add_user_type_to_users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/add_user_type_to_users.sql
rename to synapse/storage/schema/main/delta/53/add_user_type_to_users.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/drop_sent_transactions.sql b/synapse/storage/schema/main/delta/53/drop_sent_transactions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/drop_sent_transactions.sql
rename to synapse/storage/schema/main/delta/53/drop_sent_transactions.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/event_format_version.sql b/synapse/storage/schema/main/delta/53/event_format_version.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/event_format_version.sql
rename to synapse/storage/schema/main/delta/53/event_format_version.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql b/synapse/storage/schema/main/delta/53/user_dir_populate.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql
rename to synapse/storage/schema/main/delta/53/user_dir_populate.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_ips_index.sql b/synapse/storage/schema/main/delta/53/user_ips_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/user_ips_index.sql
rename to synapse/storage/schema/main/delta/53/user_ips_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_share.sql b/synapse/storage/schema/main/delta/53/user_share.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/user_share.sql
rename to synapse/storage/schema/main/delta/53/user_share.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_threepid_id.sql b/synapse/storage/schema/main/delta/53/user_threepid_id.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/user_threepid_id.sql
rename to synapse/storage/schema/main/delta/53/user_threepid_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/users_in_public_rooms.sql b/synapse/storage/schema/main/delta/53/users_in_public_rooms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/users_in_public_rooms.sql
rename to synapse/storage/schema/main/delta/53/users_in_public_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/account_validity_with_renewal.sql b/synapse/storage/schema/main/delta/54/account_validity_with_renewal.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/account_validity_with_renewal.sql
rename to synapse/storage/schema/main/delta/54/account_validity_with_renewal.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/add_validity_to_server_keys.sql b/synapse/storage/schema/main/delta/54/add_validity_to_server_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/add_validity_to_server_keys.sql
rename to synapse/storage/schema/main/delta/54/add_validity_to_server_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/delete_forward_extremities.sql b/synapse/storage/schema/main/delta/54/delete_forward_extremities.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/delete_forward_extremities.sql
rename to synapse/storage/schema/main/delta/54/delete_forward_extremities.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/drop_legacy_tables.sql b/synapse/storage/schema/main/delta/54/drop_legacy_tables.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/drop_legacy_tables.sql
rename to synapse/storage/schema/main/delta/54/drop_legacy_tables.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/drop_presence_list.sql b/synapse/storage/schema/main/delta/54/drop_presence_list.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/drop_presence_list.sql
rename to synapse/storage/schema/main/delta/54/drop_presence_list.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/relations.sql b/synapse/storage/schema/main/delta/54/relations.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/relations.sql
rename to synapse/storage/schema/main/delta/54/relations.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/stats.sql b/synapse/storage/schema/main/delta/54/stats.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/stats.sql
rename to synapse/storage/schema/main/delta/54/stats.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/stats2.sql b/synapse/storage/schema/main/delta/54/stats2.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/stats2.sql
rename to synapse/storage/schema/main/delta/54/stats2.sql
diff --git a/synapse/storage/databases/main/schema/delta/55/access_token_expiry.sql b/synapse/storage/schema/main/delta/55/access_token_expiry.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/55/access_token_expiry.sql
rename to synapse/storage/schema/main/delta/55/access_token_expiry.sql
diff --git a/synapse/storage/databases/main/schema/delta/55/profile_replication_status_index.sql b/synapse/storage/schema/main/delta/55/profile_replication_status_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/55/profile_replication_status_index.sql
rename to synapse/storage/schema/main/delta/55/profile_replication_status_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/55/track_threepid_validations.sql b/synapse/storage/schema/main/delta/55/track_threepid_validations.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/55/track_threepid_validations.sql
rename to synapse/storage/schema/main/delta/55/track_threepid_validations.sql
diff --git a/synapse/storage/databases/main/schema/delta/55/users_alter_deactivated.sql b/synapse/storage/schema/main/delta/55/users_alter_deactivated.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/55/users_alter_deactivated.sql
rename to synapse/storage/schema/main/delta/55/users_alter_deactivated.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/add_spans_to_device_lists.sql b/synapse/storage/schema/main/delta/56/add_spans_to_device_lists.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/add_spans_to_device_lists.sql
rename to synapse/storage/schema/main/delta/56/add_spans_to_device_lists.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/current_state_events_membership.sql b/synapse/storage/schema/main/delta/56/current_state_events_membership.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/current_state_events_membership.sql
rename to synapse/storage/schema/main/delta/56/current_state_events_membership.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/current_state_events_membership_mk2.sql b/synapse/storage/schema/main/delta/56/current_state_events_membership_mk2.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/current_state_events_membership_mk2.sql
rename to synapse/storage/schema/main/delta/56/current_state_events_membership_mk2.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/delete_keys_from_deleted_backups.sql b/synapse/storage/schema/main/delta/56/delete_keys_from_deleted_backups.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/delete_keys_from_deleted_backups.sql
rename to synapse/storage/schema/main/delta/56/delete_keys_from_deleted_backups.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/destinations_failure_ts.sql b/synapse/storage/schema/main/delta/56/destinations_failure_ts.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/destinations_failure_ts.sql
rename to synapse/storage/schema/main/delta/56/destinations_failure_ts.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/destinations_retry_interval_type.sql.postgres b/synapse/storage/schema/main/delta/56/destinations_retry_interval_type.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/destinations_retry_interval_type.sql.postgres
rename to synapse/storage/schema/main/delta/56/destinations_retry_interval_type.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/56/device_stream_id_insert.sql b/synapse/storage/schema/main/delta/56/device_stream_id_insert.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/device_stream_id_insert.sql
rename to synapse/storage/schema/main/delta/56/device_stream_id_insert.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/devices_last_seen.sql b/synapse/storage/schema/main/delta/56/devices_last_seen.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/devices_last_seen.sql
rename to synapse/storage/schema/main/delta/56/devices_last_seen.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/drop_unused_event_tables.sql b/synapse/storage/schema/main/delta/56/drop_unused_event_tables.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/drop_unused_event_tables.sql
rename to synapse/storage/schema/main/delta/56/drop_unused_event_tables.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/event_expiry.sql b/synapse/storage/schema/main/delta/56/event_expiry.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/event_expiry.sql
rename to synapse/storage/schema/main/delta/56/event_expiry.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/event_labels.sql b/synapse/storage/schema/main/delta/56/event_labels.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/event_labels.sql
rename to synapse/storage/schema/main/delta/56/event_labels.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/event_labels_background_update.sql b/synapse/storage/schema/main/delta/56/event_labels_background_update.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/event_labels_background_update.sql
rename to synapse/storage/schema/main/delta/56/event_labels_background_update.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/fix_room_keys_index.sql b/synapse/storage/schema/main/delta/56/fix_room_keys_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/fix_room_keys_index.sql
rename to synapse/storage/schema/main/delta/56/fix_room_keys_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/hidden_devices.sql b/synapse/storage/schema/main/delta/56/hidden_devices.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/hidden_devices.sql
rename to synapse/storage/schema/main/delta/56/hidden_devices.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/hidden_devices_fix.sql.sqlite b/synapse/storage/schema/main/delta/56/hidden_devices_fix.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/hidden_devices_fix.sql.sqlite
rename to synapse/storage/schema/main/delta/56/hidden_devices_fix.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/56/nuke_empty_communities_from_db.sql b/synapse/storage/schema/main/delta/56/nuke_empty_communities_from_db.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/nuke_empty_communities_from_db.sql
rename to synapse/storage/schema/main/delta/56/nuke_empty_communities_from_db.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/public_room_list_idx.sql b/synapse/storage/schema/main/delta/56/public_room_list_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/public_room_list_idx.sql
rename to synapse/storage/schema/main/delta/56/public_room_list_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor.sql b/synapse/storage/schema/main/delta/56/redaction_censor.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/redaction_censor.sql
rename to synapse/storage/schema/main/delta/56/redaction_censor.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor2.sql b/synapse/storage/schema/main/delta/56/redaction_censor2.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/redaction_censor2.sql
rename to synapse/storage/schema/main/delta/56/redaction_censor2.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor3_fix_update.sql.postgres b/synapse/storage/schema/main/delta/56/redaction_censor3_fix_update.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/redaction_censor3_fix_update.sql.postgres
rename to synapse/storage/schema/main/delta/56/redaction_censor3_fix_update.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor4.sql b/synapse/storage/schema/main/delta/56/redaction_censor4.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/redaction_censor4.sql
rename to synapse/storage/schema/main/delta/56/redaction_censor4.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/remove_tombstoned_rooms_from_directory.sql b/synapse/storage/schema/main/delta/56/remove_tombstoned_rooms_from_directory.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/remove_tombstoned_rooms_from_directory.sql
rename to synapse/storage/schema/main/delta/56/remove_tombstoned_rooms_from_directory.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/room_key_etag.sql b/synapse/storage/schema/main/delta/56/room_key_etag.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/room_key_etag.sql
rename to synapse/storage/schema/main/delta/56/room_key_etag.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/room_membership_idx.sql b/synapse/storage/schema/main/delta/56/room_membership_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/room_membership_idx.sql
rename to synapse/storage/schema/main/delta/56/room_membership_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/room_retention.sql b/synapse/storage/schema/main/delta/56/room_retention.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/room_retention.sql
rename to synapse/storage/schema/main/delta/56/room_retention.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/signing_keys.sql b/synapse/storage/schema/main/delta/56/signing_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/signing_keys.sql
rename to synapse/storage/schema/main/delta/56/signing_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/signing_keys_nonunique_signatures.sql b/synapse/storage/schema/main/delta/56/signing_keys_nonunique_signatures.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/signing_keys_nonunique_signatures.sql
rename to synapse/storage/schema/main/delta/56/signing_keys_nonunique_signatures.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/stats_separated.sql b/synapse/storage/schema/main/delta/56/stats_separated.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/stats_separated.sql
rename to synapse/storage/schema/main/delta/56/stats_separated.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/unique_user_filter_index.py b/synapse/storage/schema/main/delta/56/unique_user_filter_index.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/unique_user_filter_index.py
rename to synapse/storage/schema/main/delta/56/unique_user_filter_index.py
diff --git a/synapse/storage/databases/main/schema/delta/56/user_external_ids.sql b/synapse/storage/schema/main/delta/56/user_external_ids.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/user_external_ids.sql
rename to synapse/storage/schema/main/delta/56/user_external_ids.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/users_in_public_rooms_idx.sql b/synapse/storage/schema/main/delta/56/users_in_public_rooms_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/users_in_public_rooms_idx.sql
rename to synapse/storage/schema/main/delta/56/users_in_public_rooms_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/delete_old_current_state_events.sql b/synapse/storage/schema/main/delta/57/delete_old_current_state_events.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/delete_old_current_state_events.sql
rename to synapse/storage/schema/main/delta/57/delete_old_current_state_events.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/device_list_remote_cache_stale.sql b/synapse/storage/schema/main/delta/57/device_list_remote_cache_stale.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/device_list_remote_cache_stale.sql
rename to synapse/storage/schema/main/delta/57/device_list_remote_cache_stale.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/local_current_membership.py b/synapse/storage/schema/main/delta/57/local_current_membership.py
similarity index 99%
rename from synapse/storage/databases/main/schema/delta/57/local_current_membership.py
rename to synapse/storage/schema/main/delta/57/local_current_membership.py
index 44917f0a2e..66989222e6 100644
--- a/synapse/storage/databases/main/schema/delta/57/local_current_membership.py
+++ b/synapse/storage/schema/main/delta/57/local_current_membership.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/databases/main/schema/delta/57/remove_sent_outbound_pokes.sql b/synapse/storage/schema/main/delta/57/remove_sent_outbound_pokes.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/remove_sent_outbound_pokes.sql
rename to synapse/storage/schema/main/delta/57/remove_sent_outbound_pokes.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column.sql b/synapse/storage/schema/main/delta/57/rooms_version_column.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column.sql
rename to synapse/storage/schema/main/delta/57/rooms_version_column.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.postgres b/synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.postgres
rename to synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.sqlite b/synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.sqlite
rename to synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.postgres b/synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.postgres
rename to synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.sqlite b/synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.sqlite
rename to synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/58/02remove_dup_outbound_pokes.sql b/synapse/storage/schema/main/delta/58/02remove_dup_outbound_pokes.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/02remove_dup_outbound_pokes.sql
rename to synapse/storage/schema/main/delta/58/02remove_dup_outbound_pokes.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/03persist_ui_auth.sql b/synapse/storage/schema/main/delta/58/03persist_ui_auth.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/03persist_ui_auth.sql
rename to synapse/storage/schema/main/delta/58/03persist_ui_auth.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/05cache_instance.sql.postgres b/synapse/storage/schema/main/delta/58/05cache_instance.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/05cache_instance.sql.postgres
rename to synapse/storage/schema/main/delta/58/05cache_instance.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/06dlols_unique_idx.py b/synapse/storage/schema/main/delta/58/06dlols_unique_idx.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/06dlols_unique_idx.py
rename to synapse/storage/schema/main/delta/58/06dlols_unique_idx.py
diff --git a/synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.postgres b/synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.postgres
rename to synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite b/synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite
rename to synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/58/07persist_ui_auth_ips.sql b/synapse/storage/schema/main/delta/58/07persist_ui_auth_ips.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/07persist_ui_auth_ips.sql
rename to synapse/storage/schema/main/delta/58/07persist_ui_auth_ips.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.postgres b/synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.postgres
rename to synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.sqlite b/synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.sqlite
rename to synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/58/09shadow_ban.sql b/synapse/storage/schema/main/delta/58/09shadow_ban.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/09shadow_ban.sql
rename to synapse/storage/schema/main/delta/58/09shadow_ban.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/10_pushrules_enabled_delete_obsolete.sql b/synapse/storage/schema/main/delta/58/10_pushrules_enabled_delete_obsolete.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/10_pushrules_enabled_delete_obsolete.sql
rename to synapse/storage/schema/main/delta/58/10_pushrules_enabled_delete_obsolete.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/10drop_local_rejections_stream.sql b/synapse/storage/schema/main/delta/58/10drop_local_rejections_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/10drop_local_rejections_stream.sql
rename to synapse/storage/schema/main/delta/58/10drop_local_rejections_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/10federation_pos_instance_name.sql b/synapse/storage/schema/main/delta/58/10federation_pos_instance_name.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/10federation_pos_instance_name.sql
rename to synapse/storage/schema/main/delta/58/10federation_pos_instance_name.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/11dehydration.sql b/synapse/storage/schema/main/delta/58/11dehydration.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/11dehydration.sql
rename to synapse/storage/schema/main/delta/58/11dehydration.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/11fallback.sql b/synapse/storage/schema/main/delta/58/11fallback.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/11fallback.sql
rename to synapse/storage/schema/main/delta/58/11fallback.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/11user_id_seq.py b/synapse/storage/schema/main/delta/58/11user_id_seq.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/11user_id_seq.py
rename to synapse/storage/schema/main/delta/58/11user_id_seq.py
diff --git a/synapse/storage/databases/main/schema/delta/58/12room_stats.sql b/synapse/storage/schema/main/delta/58/12room_stats.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/12room_stats.sql
rename to synapse/storage/schema/main/delta/58/12room_stats.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/13remove_presence_allow_inbound.sql b/synapse/storage/schema/main/delta/58/13remove_presence_allow_inbound.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/13remove_presence_allow_inbound.sql
rename to synapse/storage/schema/main/delta/58/13remove_presence_allow_inbound.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql b/synapse/storage/schema/main/delta/58/14events_instance_name.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql
rename to synapse/storage/schema/main/delta/58/14events_instance_name.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql.postgres b/synapse/storage/schema/main/delta/58/14events_instance_name.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql.postgres
rename to synapse/storage/schema/main/delta/58/14events_instance_name.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/15_catchup_destination_rooms.sql b/synapse/storage/schema/main/delta/58/15_catchup_destination_rooms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/15_catchup_destination_rooms.sql
rename to synapse/storage/schema/main/delta/58/15_catchup_destination_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/15unread_count.sql b/synapse/storage/schema/main/delta/58/15unread_count.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/15unread_count.sql
rename to synapse/storage/schema/main/delta/58/15unread_count.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/16populate_stats_process_rooms_fix.sql b/synapse/storage/schema/main/delta/58/16populate_stats_process_rooms_fix.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/16populate_stats_process_rooms_fix.sql
rename to synapse/storage/schema/main/delta/58/16populate_stats_process_rooms_fix.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/17_catchup_last_successful.sql b/synapse/storage/schema/main/delta/58/17_catchup_last_successful.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/17_catchup_last_successful.sql
rename to synapse/storage/schema/main/delta/58/17_catchup_last_successful.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/18stream_positions.sql b/synapse/storage/schema/main/delta/58/18stream_positions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/18stream_positions.sql
rename to synapse/storage/schema/main/delta/58/18stream_positions.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/19account_validity_token_used_ts_ms.sql b/synapse/storage/schema/main/delta/58/19account_validity_token_used_ts_ms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/19account_validity_token_used_ts_ms.sql
rename to synapse/storage/schema/main/delta/58/19account_validity_token_used_ts_ms.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/19instance_map.sql.postgres b/synapse/storage/schema/main/delta/58/19instance_map.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/19instance_map.sql.postgres
rename to synapse/storage/schema/main/delta/58/19instance_map.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/19txn_id.sql b/synapse/storage/schema/main/delta/58/19txn_id.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/19txn_id.sql
rename to synapse/storage/schema/main/delta/58/19txn_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/20instance_name_event_tables.sql b/synapse/storage/schema/main/delta/58/20instance_name_event_tables.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/20instance_name_event_tables.sql
rename to synapse/storage/schema/main/delta/58/20instance_name_event_tables.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/20user_daily_visits.sql b/synapse/storage/schema/main/delta/58/20user_daily_visits.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/20user_daily_visits.sql
rename to synapse/storage/schema/main/delta/58/20user_daily_visits.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/21as_device_stream.sql b/synapse/storage/schema/main/delta/58/21as_device_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/21as_device_stream.sql
rename to synapse/storage/schema/main/delta/58/21as_device_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/21drop_device_max_stream_id.sql b/synapse/storage/schema/main/delta/58/21drop_device_max_stream_id.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/21drop_device_max_stream_id.sql
rename to synapse/storage/schema/main/delta/58/21drop_device_max_stream_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/22puppet_token.sql b/synapse/storage/schema/main/delta/58/22puppet_token.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/22puppet_token.sql
rename to synapse/storage/schema/main/delta/58/22puppet_token.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/22users_have_local_media.sql b/synapse/storage/schema/main/delta/58/22users_have_local_media.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/22users_have_local_media.sql
rename to synapse/storage/schema/main/delta/58/22users_have_local_media.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/23e2e_cross_signing_keys_idx.sql b/synapse/storage/schema/main/delta/58/23e2e_cross_signing_keys_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/23e2e_cross_signing_keys_idx.sql
rename to synapse/storage/schema/main/delta/58/23e2e_cross_signing_keys_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/24add_knock_members_to_stats.sql b/synapse/storage/schema/main/delta/58/24add_knock_members_to_stats.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/24add_knock_members_to_stats.sql
rename to synapse/storage/schema/main/delta/58/24add_knock_members_to_stats.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/24drop_event_json_index.sql b/synapse/storage/schema/main/delta/58/24drop_event_json_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/24drop_event_json_index.sql
rename to synapse/storage/schema/main/delta/58/24drop_event_json_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/25user_external_ids_user_id_idx.sql b/synapse/storage/schema/main/delta/58/25user_external_ids_user_id_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/25user_external_ids_user_id_idx.sql
rename to synapse/storage/schema/main/delta/58/25user_external_ids_user_id_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/26access_token_last_validated.sql b/synapse/storage/schema/main/delta/58/26access_token_last_validated.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/26access_token_last_validated.sql
rename to synapse/storage/schema/main/delta/58/26access_token_last_validated.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/27local_invites.sql b/synapse/storage/schema/main/delta/58/27local_invites.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/27local_invites.sql
rename to synapse/storage/schema/main/delta/58/27local_invites.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.postgres b/synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.postgres
rename to synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.sqlite b/synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.sqlite
rename to synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/59/01ignored_user.py b/synapse/storage/schema/main/delta/59/01ignored_user.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/01ignored_user.py
rename to synapse/storage/schema/main/delta/59/01ignored_user.py
diff --git a/synapse/storage/databases/main/schema/delta/59/02shard_send_to_device.sql b/synapse/storage/schema/main/delta/59/02shard_send_to_device.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/02shard_send_to_device.sql
rename to synapse/storage/schema/main/delta/59/02shard_send_to_device.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/03shard_send_to_device_sequence.sql.postgres b/synapse/storage/schema/main/delta/59/03shard_send_to_device_sequence.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/03shard_send_to_device_sequence.sql.postgres
rename to synapse/storage/schema/main/delta/59/03shard_send_to_device_sequence.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql b/synapse/storage/schema/main/delta/59/04_event_auth_chains.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql
rename to synapse/storage/schema/main/delta/59/04_event_auth_chains.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql.postgres b/synapse/storage/schema/main/delta/59/04_event_auth_chains.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql.postgres
rename to synapse/storage/schema/main/delta/59/04_event_auth_chains.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/04drop_account_data.sql b/synapse/storage/schema/main/delta/59/04drop_account_data.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/04drop_account_data.sql
rename to synapse/storage/schema/main/delta/59/04drop_account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/05cache_invalidation.sql b/synapse/storage/schema/main/delta/59/05cache_invalidation.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/05cache_invalidation.sql
rename to synapse/storage/schema/main/delta/59/05cache_invalidation.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/06chain_cover_index.sql b/synapse/storage/schema/main/delta/59/06chain_cover_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/06chain_cover_index.sql
rename to synapse/storage/schema/main/delta/59/06chain_cover_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql b/synapse/storage/schema/main/delta/59/06shard_account_data.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql
rename to synapse/storage/schema/main/delta/59/06shard_account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql.postgres b/synapse/storage/schema/main/delta/59/06shard_account_data.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql.postgres
rename to synapse/storage/schema/main/delta/59/06shard_account_data.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/07shard_account_data_fix.sql b/synapse/storage/schema/main/delta/59/07shard_account_data_fix.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/07shard_account_data_fix.sql
rename to synapse/storage/schema/main/delta/59/07shard_account_data_fix.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/08delete_pushers_for_deactivated_accounts.sql b/synapse/storage/schema/main/delta/59/08delete_pushers_for_deactivated_accounts.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/08delete_pushers_for_deactivated_accounts.sql
rename to synapse/storage/schema/main/delta/59/08delete_pushers_for_deactivated_accounts.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/08delete_stale_pushers.sql b/synapse/storage/schema/main/delta/59/08delete_stale_pushers.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/08delete_stale_pushers.sql
rename to synapse/storage/schema/main/delta/59/08delete_stale_pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/09rejected_events_metadata.sql b/synapse/storage/schema/main/delta/59/09rejected_events_metadata.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/09rejected_events_metadata.sql
rename to synapse/storage/schema/main/delta/59/09rejected_events_metadata.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/10delete_purged_chain_cover.sql b/synapse/storage/schema/main/delta/59/10delete_purged_chain_cover.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/10delete_purged_chain_cover.sql
rename to synapse/storage/schema/main/delta/59/10delete_purged_chain_cover.sql
diff --git a/synapse/storage/schema/main/delta/59/11drop_thumbnail_constraint.sql.postgres b/synapse/storage/schema/main/delta/59/11drop_thumbnail_constraint.sql.postgres
new file mode 100644
index 0000000000..54c1bca3b1
--- /dev/null
+++ b/synapse/storage/schema/main/delta/59/11drop_thumbnail_constraint.sql.postgres
@@ -0,0 +1,22 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- drop old constraints on remote_media_cache_thumbnails
+--
+-- This was originally part of 57.07, but it was done wrong, per
+-- https://github.com/matrix-org/synapse/issues/8649, so we do it again.
+INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES
+  (5911, 'media_repository_drop_index_wo_method_2', '{}', 'remote_media_repository_thumbnails_method_idx');
+
diff --git a/synapse/storage/schema/main/delta/59/12presence_stream_instance.sql b/synapse/storage/schema/main/delta/59/12presence_stream_instance.sql
new file mode 100644
index 0000000000..b6ba0bda1a
--- /dev/null
+++ b/synapse/storage/schema/main/delta/59/12presence_stream_instance.sql
@@ -0,0 +1,18 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Add a column to specify which instance wrote the row. Historic rows have
+-- `NULL`, which indicates that the master instance wrote them.
+ALTER TABLE presence_stream ADD COLUMN instance_name TEXT;
diff --git a/synapse/storage/schema/main/delta/59/12presence_stream_instance_seq.sql.postgres b/synapse/storage/schema/main/delta/59/12presence_stream_instance_seq.sql.postgres
new file mode 100644
index 0000000000..02b182adf9
--- /dev/null
+++ b/synapse/storage/schema/main/delta/59/12presence_stream_instance_seq.sql.postgres
@@ -0,0 +1,20 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE SEQUENCE IF NOT EXISTS presence_stream_sequence;
+
+SELECT setval('presence_stream_sequence', (
+    SELECT COALESCE(MAX(stream_id), 1) FROM presence_stream
+));
diff --git a/synapse/storage/schema/main/delta/59/13users_to_send_full_presence_to.sql b/synapse/storage/schema/main/delta/59/13users_to_send_full_presence_to.sql
new file mode 100644
index 0000000000..07b0f53ecf
--- /dev/null
+++ b/synapse/storage/schema/main/delta/59/13users_to_send_full_presence_to.sql
@@ -0,0 +1,34 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Add a table that keeps track of a list of users who should, upon their next
+-- sync request, receive presence for all currently online users that they are
+-- "interested" in.
+
+-- The motivation for a DB table over an in-memory list is so that this list
+-- can be added to and retrieved from by any worker. Specifically, we don't
+-- want to duplicate work across multiple sync workers.
+
+CREATE TABLE IF NOT EXISTS users_to_send_full_presence_to(
+    -- The user ID to send full presence to.
+    user_id TEXT PRIMARY KEY,
+    -- A presence stream ID token - the current presence stream token when the row was last upserted.
+    -- If a user calls /sync and this token is part of the update they're to receive, we also include
+    -- full user presence in the response.
+    -- This allows multiple devices for a user to receive full presence whenever they next call /sync.
+    presence_stream_id BIGINT,
+    FOREIGN KEY (user_id)
+        REFERENCES users (name)
+);
\ No newline at end of file
diff --git a/synapse/storage/schema/main/delta/59/14refresh_tokens.sql b/synapse/storage/schema/main/delta/59/14refresh_tokens.sql
new file mode 100644
index 0000000000..9a6bce1e3e
--- /dev/null
+++ b/synapse/storage/schema/main/delta/59/14refresh_tokens.sql
@@ -0,0 +1,34 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Holds MSC2918 refresh tokens
+CREATE TABLE refresh_tokens (
+  id BIGINT PRIMARY KEY,
+  user_id TEXT NOT NULL,
+  device_id TEXT NOT NULL,
+  token TEXT NOT NULL,
+  -- When consumed, a new refresh token is generated, which is tracked by
+  -- this foreign key
+  next_token_id BIGINT REFERENCES refresh_tokens (id) ON DELETE CASCADE,
+  UNIQUE(token)
+);
+
+-- Add a reference to the refresh token generated alongside each access token
+ALTER TABLE "access_tokens"
+  ADD COLUMN refresh_token_id BIGINT REFERENCES refresh_tokens (id) ON DELETE CASCADE;
+
+-- Add a flag whether the token was already used or not
+ALTER TABLE "access_tokens"
+  ADD COLUMN used BOOLEAN;
diff --git a/synapse/storage/schema/main/delta/59/15locks.sql b/synapse/storage/schema/main/delta/59/15locks.sql
new file mode 100644
index 0000000000..8b2999ff3e
--- /dev/null
+++ b/synapse/storage/schema/main/delta/59/15locks.sql
@@ -0,0 +1,37 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- A noddy implementation of a distributed lock across workers. While a worker
+-- has taken a lock out they should regularly update the `last_renewed_ts`
+-- column, a lock will be considered dropped if `last_renewed_ts` is from ages
+-- ago.
+CREATE TABLE worker_locks (
+    lock_name TEXT NOT NULL,
+    lock_key TEXT NOT NULL,
+    -- We write the instance name to ease manual debugging, we don't ever read
+    -- from it.
+    -- Note: instance names aren't guarenteed to be unique.
+    instance_name TEXT NOT NULL,
+    -- A random string generated each time an instance takes out a lock. Used by
+    -- the instance to tell whether the lock is still held by it (e.g. in the
+    -- case where the process stalls for a long time the lock may time out and
+    -- be taken out by another instance, at which point the original instance
+    -- can tell it no longer holds the lock as the tokens no longer match).
+    token TEXT NOT NULL,
+    last_renewed_ts BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX worker_locks_key ON worker_locks (lock_name, lock_key);
diff --git a/synapse/storage/schema/main/delta/59/16federation_inbound_staging.sql b/synapse/storage/schema/main/delta/59/16federation_inbound_staging.sql
new file mode 100644
index 0000000000..43bc5c025f
--- /dev/null
+++ b/synapse/storage/schema/main/delta/59/16federation_inbound_staging.sql
@@ -0,0 +1,32 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- A staging area for newly received events over federation.
+--
+-- Note we may store the same event multiple times if it comes from different
+-- servers; this is to handle the case if we get a redacted and non-redacted
+-- versions of the event.
+CREATE TABLE federation_inbound_events_staging (
+    origin TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    event_id TEXT NOT NULL,
+    received_ts BIGINT NOT NULL,
+    event_json TEXT NOT NULL,
+    internal_metadata TEXT NOT NULL
+);
+
+CREATE INDEX federation_inbound_events_staging_room ON federation_inbound_events_staging(room_id, received_ts);
+CREATE UNIQUE INDEX federation_inbound_events_staging_instance_event ON federation_inbound_events_staging(origin, event_id);
diff --git a/synapse/storage/schema/main/delta/60/01recreate_stream_ordering.sql.postgres b/synapse/storage/schema/main/delta/60/01recreate_stream_ordering.sql.postgres
new file mode 100644
index 0000000000..0edc9fe7a2
--- /dev/null
+++ b/synapse/storage/schema/main/delta/60/01recreate_stream_ordering.sql.postgres
@@ -0,0 +1,45 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- This migration handles the process of changing the type of `stream_ordering` to
+-- a BIGINT.
+--
+-- Note that this is only a problem on postgres as sqlite only has one "integer" type
+-- which can cope with values up to 2^63.
+
+-- First add a new column to contain the bigger stream_ordering
+ALTER TABLE events ADD COLUMN stream_ordering2 BIGINT;
+
+-- Create a rule which will populate it for new rows.
+CREATE OR REPLACE RULE "populate_stream_ordering2" AS
+    ON INSERT TO events
+    DO UPDATE events SET stream_ordering2=NEW.stream_ordering WHERE stream_ordering=NEW.stream_ordering;
+
+-- Start a bg process to populate it for old events
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (6001, 'populate_stream_ordering2', '{}');
+
+-- ... and some more to build indexes on it. These aren't really interdependent
+-- but the backround_updates manager can only handle a single dependency per update.
+INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES
+  (6001, 'index_stream_ordering2', '{}', 'populate_stream_ordering2'),
+  (6001, 'index_stream_ordering2_room_order', '{}', 'index_stream_ordering2'),
+  (6001, 'index_stream_ordering2_contains_url', '{}', 'index_stream_ordering2_room_order'),
+  (6001, 'index_stream_ordering2_room_stream', '{}', 'index_stream_ordering2_contains_url'),
+  (6001, 'index_stream_ordering2_ts', '{}', 'index_stream_ordering2_room_stream');
+
+-- ... and another to do the switcheroo
+INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES
+  (6001, 'replace_stream_ordering_column', '{}', 'index_stream_ordering2_ts');
diff --git a/synapse/storage/schema/main/delta/60/02change_stream_ordering_columns.sql.postgres b/synapse/storage/schema/main/delta/60/02change_stream_ordering_columns.sql.postgres
new file mode 100644
index 0000000000..630c24fd9e
--- /dev/null
+++ b/synapse/storage/schema/main/delta/60/02change_stream_ordering_columns.sql.postgres
@@ -0,0 +1,30 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- This migration is closely related to '01recreate_stream_ordering.sql.postgres'.
+--
+-- It updates the other tables which use an INTEGER to refer to a stream ordering.
+-- These tables are all small enough that a re-create is tractable.
+ALTER TABLE pushers ALTER COLUMN last_stream_ordering SET DATA TYPE BIGINT;
+ALTER TABLE federation_stream_position ALTER COLUMN stream_id SET DATA TYPE BIGINT;
+
+-- these aren't actually event stream orderings, but they are numbers where 2 billion
+-- is a bit limiting, application_services_state is tiny, and I don't want to ever have
+-- to do this again.
+ALTER TABLE application_services_state ALTER COLUMN last_txn SET DATA TYPE BIGINT;
+ALTER TABLE application_services_state ALTER COLUMN read_receipt_stream_id SET DATA TYPE BIGINT;
+ALTER TABLE application_services_state ALTER COLUMN presence_stream_id SET DATA TYPE BIGINT;
+
+
diff --git a/synapse/storage/schema/main/delta/61/01change_appservices_txns.sql.postgres b/synapse/storage/schema/main/delta/61/01change_appservices_txns.sql.postgres
new file mode 100644
index 0000000000..c8aec78e60
--- /dev/null
+++ b/synapse/storage/schema/main/delta/61/01change_appservices_txns.sql.postgres
@@ -0,0 +1,23 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- we use bigint elsewhere in the database for appservice txn ids (notably,
+-- application_services_state.last_txn), and generally we use bigints everywhere else
+-- we have monotonic counters, so let's bring this one in line.
+--
+-- assuming there aren't thousands of rows for decommisioned/non-functional ASes, this
+-- table should be pretty small, so safe to do a synchronous ALTER TABLE.
+
+ALTER TABLE application_services_txns ALTER COLUMN txn_id SET DATA TYPE BIGINT;
diff --git a/synapse/storage/schema/main/delta/61/01insertion_event_lookups.sql b/synapse/storage/schema/main/delta/61/01insertion_event_lookups.sql
new file mode 100644
index 0000000000..7d7bafc631
--- /dev/null
+++ b/synapse/storage/schema/main/delta/61/01insertion_event_lookups.sql
@@ -0,0 +1,49 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Add a table that keeps track of "insertion" events and
+-- their next_chunk_id's so we can navigate to the next chunk of history.
+CREATE TABLE IF NOT EXISTS insertion_events(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    next_chunk_id TEXT NOT NULL
+);
+CREATE UNIQUE INDEX IF NOT EXISTS insertion_events_event_id ON insertion_events(event_id);
+CREATE INDEX IF NOT EXISTS insertion_events_next_chunk_id ON insertion_events(next_chunk_id);
+
+-- Add a table that keeps track of all of the events we are inserting between.
+-- We use this when navigating the DAG and when we hit an event which matches
+-- `insertion_prev_event_id`, it should backfill from the "insertion" event and
+-- navigate the historical messages from there.
+CREATE TABLE IF NOT EXISTS insertion_event_edges(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    insertion_prev_event_id TEXT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS insertion_event_edges_event_id ON insertion_event_edges(event_id);
+CREATE INDEX IF NOT EXISTS insertion_event_edges_insertion_room_id ON insertion_event_edges(room_id);
+CREATE INDEX IF NOT EXISTS insertion_event_edges_insertion_prev_event_id ON insertion_event_edges(insertion_prev_event_id);
+
+-- Add a table that keeps track of how each chunk is labeled. The chunks are
+-- connected together based on an insertion events `next_chunk_id`.
+CREATE TABLE IF NOT EXISTS chunk_events(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    chunk_id TEXT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS chunk_events_event_id ON chunk_events(event_id);
+CREATE INDEX IF NOT EXISTS chunk_events_chunk_id ON chunk_events(chunk_id);
diff --git a/synapse/storage/schema/main/delta/61/02drop_redundant_room_depth_index.sql b/synapse/storage/schema/main/delta/61/02drop_redundant_room_depth_index.sql
new file mode 100644
index 0000000000..35ca7a40c0
--- /dev/null
+++ b/synapse/storage/schema/main/delta/61/02drop_redundant_room_depth_index.sql
@@ -0,0 +1,18 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- this index is redundant; there is another UNIQUE index on this table.
+DROP INDEX IF EXISTS room_depth_room;
+
diff --git a/synapse/storage/schema/main/delta/61/03recreate_min_depth.py b/synapse/storage/schema/main/delta/61/03recreate_min_depth.py
new file mode 100644
index 0000000000..f8d7db9f2e
--- /dev/null
+++ b/synapse/storage/schema/main/delta/61/03recreate_min_depth.py
@@ -0,0 +1,70 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This migration handles the process of changing the type of `room_depth.min_depth` to
+a BIGINT.
+"""
+from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
+from synapse.storage.types import Cursor
+
+
+def run_create(cur: Cursor, database_engine: BaseDatabaseEngine, *args, **kwargs):
+    if not isinstance(database_engine, PostgresEngine):
+        # this only applies to postgres - sqlite does not distinguish between big and
+        # little ints.
+        return
+
+    # First add a new column to contain the bigger min_depth
+    cur.execute("ALTER TABLE room_depth ADD COLUMN min_depth2 BIGINT")
+
+    # Create a trigger which will keep it populated.
+    cur.execute(
+        """
+        CREATE OR REPLACE FUNCTION populate_min_depth2() RETURNS trigger AS $BODY$
+            BEGIN
+                new.min_depth2 := new.min_depth;
+                RETURN NEW;
+            END;
+        $BODY$ LANGUAGE plpgsql
+        """
+    )
+
+    cur.execute(
+        """
+        CREATE TRIGGER populate_min_depth2_trigger BEFORE INSERT OR UPDATE ON room_depth
+        FOR EACH ROW
+        EXECUTE PROCEDURE populate_min_depth2()
+        """
+    )
+
+    # Start a bg process to populate it for old rooms
+    cur.execute(
+        """
+       INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+            (6103, 'populate_room_depth_min_depth2', '{}')
+       """
+    )
+
+    # and another to switch them over once it completes.
+    cur.execute(
+        """
+        INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES
+            (6103, 'replace_room_depth_min_depth', '{}', 'populate_room_depth2')
+        """
+    )
+
+
+def run_upgrade(cur: Cursor, database_engine: BaseDatabaseEngine, *args, **kwargs):
+    pass
diff --git a/synapse/storage/schema/main/delta/62/01insertion_event_extremities.sql b/synapse/storage/schema/main/delta/62/01insertion_event_extremities.sql
new file mode 100644
index 0000000000..b731ef284a
--- /dev/null
+++ b/synapse/storage/schema/main/delta/62/01insertion_event_extremities.sql
@@ -0,0 +1,24 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- Add a table that keeps track of which "insertion" events need to be backfilled
+CREATE TABLE IF NOT EXISTS insertion_event_extremities(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS insertion_event_extremities_event_id ON insertion_event_extremities(event_id);
+CREATE INDEX IF NOT EXISTS insertion_event_extremities_room_id ON insertion_event_extremities(room_id);
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/main/full_schemas/16/application_services.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/application_services.sql
rename to synapse/storage/schema/main/full_schemas/16/application_services.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/main/full_schemas/16/event_edges.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/event_edges.sql
rename to synapse/storage/schema/main/full_schemas/16/event_edges.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/main/full_schemas/16/event_signatures.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/event_signatures.sql
rename to synapse/storage/schema/main/full_schemas/16/event_signatures.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/im.sql b/synapse/storage/schema/main/full_schemas/16/im.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/im.sql
rename to synapse/storage/schema/main/full_schemas/16/im.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/keys.sql b/synapse/storage/schema/main/full_schemas/16/keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/keys.sql
rename to synapse/storage/schema/main/full_schemas/16/keys.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/main/full_schemas/16/media_repository.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/media_repository.sql
rename to synapse/storage/schema/main/full_schemas/16/media_repository.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/presence.sql b/synapse/storage/schema/main/full_schemas/16/presence.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/presence.sql
rename to synapse/storage/schema/main/full_schemas/16/presence.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/main/full_schemas/16/profiles.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/profiles.sql
rename to synapse/storage/schema/main/full_schemas/16/profiles.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/push.sql b/synapse/storage/schema/main/full_schemas/16/push.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/push.sql
rename to synapse/storage/schema/main/full_schemas/16/push.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/main/full_schemas/16/redactions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/redactions.sql
rename to synapse/storage/schema/main/full_schemas/16/redactions.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/main/full_schemas/16/room_aliases.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/room_aliases.sql
rename to synapse/storage/schema/main/full_schemas/16/room_aliases.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/state.sql b/synapse/storage/schema/main/full_schemas/16/state.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/state.sql
rename to synapse/storage/schema/main/full_schemas/16/state.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/main/full_schemas/16/transactions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/transactions.sql
rename to synapse/storage/schema/main/full_schemas/16/transactions.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/users.sql b/synapse/storage/schema/main/full_schemas/16/users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/users.sql
rename to synapse/storage/schema/main/full_schemas/16/users.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/54/full.sql.postgres b/synapse/storage/schema/main/full_schemas/54/full.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/54/full.sql.postgres
rename to synapse/storage/schema/main/full_schemas/54/full.sql.postgres
diff --git a/synapse/storage/databases/main/schema/full_schemas/54/full.sql.sqlite b/synapse/storage/schema/main/full_schemas/54/full.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/54/full.sql.sqlite
rename to synapse/storage/schema/main/full_schemas/54/full.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/full_schemas/54/stream_positions.sql b/synapse/storage/schema/main/full_schemas/54/stream_positions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/54/stream_positions.sql
rename to synapse/storage/schema/main/full_schemas/54/stream_positions.sql
diff --git a/synapse/storage/databases/state/schema/delta/23/drop_state_index.sql b/synapse/storage/schema/state/delta/23/drop_state_index.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/23/drop_state_index.sql
rename to synapse/storage/schema/state/delta/23/drop_state_index.sql
diff --git a/synapse/storage/databases/state/schema/delta/30/state_stream.sql b/synapse/storage/schema/state/delta/30/state_stream.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/30/state_stream.sql
rename to synapse/storage/schema/state/delta/30/state_stream.sql
diff --git a/synapse/storage/databases/state/schema/delta/32/remove_state_indices.sql b/synapse/storage/schema/state/delta/32/remove_state_indices.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/32/remove_state_indices.sql
rename to synapse/storage/schema/state/delta/32/remove_state_indices.sql
diff --git a/synapse/storage/databases/state/schema/delta/35/add_state_index.sql b/synapse/storage/schema/state/delta/35/add_state_index.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/35/add_state_index.sql
rename to synapse/storage/schema/state/delta/35/add_state_index.sql
diff --git a/synapse/storage/databases/state/schema/delta/35/state.sql b/synapse/storage/schema/state/delta/35/state.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/35/state.sql
rename to synapse/storage/schema/state/delta/35/state.sql
diff --git a/synapse/storage/databases/state/schema/delta/35/state_dedupe.sql b/synapse/storage/schema/state/delta/35/state_dedupe.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/35/state_dedupe.sql
rename to synapse/storage/schema/state/delta/35/state_dedupe.sql
diff --git a/synapse/storage/databases/state/schema/delta/47/state_group_seq.py b/synapse/storage/schema/state/delta/47/state_group_seq.py
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/47/state_group_seq.py
rename to synapse/storage/schema/state/delta/47/state_group_seq.py
diff --git a/synapse/storage/databases/state/schema/delta/56/state_group_room_idx.sql b/synapse/storage/schema/state/delta/56/state_group_room_idx.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/56/state_group_room_idx.sql
rename to synapse/storage/schema/state/delta/56/state_group_room_idx.sql
diff --git a/synapse/storage/schema/state/delta/61/02state_groups_state_n_distinct.sql.postgres b/synapse/storage/schema/state/delta/61/02state_groups_state_n_distinct.sql.postgres
new file mode 100644
index 0000000000..35a153da7b
--- /dev/null
+++ b/synapse/storage/schema/state/delta/61/02state_groups_state_n_distinct.sql.postgres
@@ -0,0 +1,34 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- By default the postgres statistics collector massively underestimates the
+-- number of distinct state groups are in the `state_groups_state`, which can
+-- cause postgres to use table scans for queries for multiple state groups.
+--
+-- To work around this we can manually tell postgres the number of distinct state
+-- groups there are by setting `n_distinct` (a negative value here is the number
+-- of distinct values divided by the number of rows, so -0.02 means on average
+-- there are 50 rows per distinct value). We don't need a particularly
+-- accurate number here, as a) we just want it to always use index scans and b)
+-- our estimate is going to be better than the one made by the statistics
+-- collector.
+
+ALTER TABLE state_groups_state ALTER COLUMN state_group SET (n_distinct = -0.02);
+
+-- Ideally we'd do an `ANALYZE state_groups_state (state_group)` here so that
+-- the above gets picked up immediately, but that can take a bit of time so we
+-- rely on the autovacuum eventually getting run and doing that in the
+-- background for us.
diff --git a/synapse/storage/databases/state/schema/full_schemas/54/full.sql b/synapse/storage/schema/state/full_schemas/54/full.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/full_schemas/54/full.sql
rename to synapse/storage/schema/state/full_schemas/54/full.sql
diff --git a/synapse/storage/databases/state/schema/full_schemas/54/sequence.sql.postgres b/synapse/storage/schema/state/full_schemas/54/sequence.sql.postgres
similarity index 100%
rename from synapse/storage/databases/state/schema/full_schemas/54/sequence.sql.postgres
rename to synapse/storage/schema/state/full_schemas/54/sequence.sql.postgres
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 2e277a21c4..e5400d681a 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -92,7 +91,7 @@ def from_types(types: Iterable[Tuple[str, Optional[str]]]) -> "StateFilter":
         Returns:
             The new state filter.
         """
-        type_dict = {}  # type: Dict[str, Optional[Set[str]]]
+        type_dict: Dict[str, Optional[Set[str]]] = {}
         for typ, s in types:
             if typ in type_dict:
                 if type_dict[typ] is None:
@@ -195,7 +194,7 @@ def make_sql_filter_clause(self) -> Tuple[str, List[str]]:
         """
 
         where_clause = ""
-        where_args = []  # type: List[str]
+        where_args: List[str] = []
 
         if self.is_full():
             return where_clause, where_args
@@ -449,7 +448,7 @@ def _get_state_groups_from_groups(
         return self.stores.state._get_state_groups_from_groups(groups, state_filter)
 
     async def get_state_for_events(
-        self, event_ids: Iterable[str], state_filter: StateFilter = StateFilter.all()
+        self, event_ids: Iterable[str], state_filter: Optional[StateFilter] = None
     ) -> Dict[str, StateMap[EventBase]]:
         """Given a list of event_ids and type tuples, return a list of state
         dicts for each event.
@@ -465,7 +464,7 @@ async def get_state_for_events(
 
         groups = set(event_to_groups.values())
         group_to_state = await self.stores.state._get_state_for_groups(
-            groups, state_filter
+            groups, state_filter or StateFilter.all()
         )
 
         state_event_map = await self.stores.main.get_events(
@@ -485,7 +484,7 @@ async def get_state_for_events(
         return {event: event_to_state[event] for event in event_ids}
 
     async def get_state_ids_for_events(
-        self, event_ids: Iterable[str], state_filter: StateFilter = StateFilter.all()
+        self, event_ids: Iterable[str], state_filter: Optional[StateFilter] = None
     ) -> Dict[str, StateMap[str]]:
         """
         Get the state dicts corresponding to a list of events, containing the event_ids
@@ -502,7 +501,7 @@ async def get_state_ids_for_events(
 
         groups = set(event_to_groups.values())
         group_to_state = await self.stores.state._get_state_for_groups(
-            groups, state_filter
+            groups, state_filter or StateFilter.all()
         )
 
         event_to_state = {
@@ -513,7 +512,7 @@ async def get_state_ids_for_events(
         return {event: event_to_state[event] for event in event_ids}
 
     async def get_state_for_event(
-        self, event_id: str, state_filter: StateFilter = StateFilter.all()
+        self, event_id: str, state_filter: Optional[StateFilter] = None
     ) -> StateMap[EventBase]:
         """
         Get the state dict corresponding to a particular event
@@ -525,11 +524,13 @@ async def get_state_for_event(
         Returns:
             A dict from (type, state_key) -> state_event
         """
-        state_map = await self.get_state_for_events([event_id], state_filter)
+        state_map = await self.get_state_for_events(
+            [event_id], state_filter or StateFilter.all()
+        )
         return state_map[event_id]
 
     async def get_state_ids_for_event(
-        self, event_id: str, state_filter: StateFilter = StateFilter.all()
+        self, event_id: str, state_filter: Optional[StateFilter] = None
     ) -> StateMap[str]:
         """
         Get the state dict corresponding to a particular event
@@ -539,13 +540,15 @@ async def get_state_ids_for_event(
             state_filter: The state filter used to fetch state from the database.
 
         Returns:
-            A dict from (type, state_key) -> state_event
+            A dict from (type, state_key) -> state_event_id
         """
-        state_map = await self.get_state_ids_for_events([event_id], state_filter)
+        state_map = await self.get_state_ids_for_events(
+            [event_id], state_filter or StateFilter.all()
+        )
         return state_map[event_id]
 
     def _get_state_for_groups(
-        self, groups: Iterable[int], state_filter: StateFilter = StateFilter.all()
+        self, groups: Iterable[int], state_filter: Optional[StateFilter] = None
     ) -> Awaitable[Dict[int, MutableStateMap[str]]]:
         """Gets the state at each of a list of state groups, optionally
         filtering by type/state_key
@@ -558,15 +561,17 @@ def _get_state_for_groups(
         Returns:
             Dict of state group to state map.
         """
-        return self.stores.state._get_state_for_groups(groups, state_filter)
+        return self.stores.state._get_state_for_groups(
+            groups, state_filter or StateFilter.all()
+        )
 
     async def store_state_group(
         self,
         event_id: str,
         room_id: str,
         prev_group: Optional[int],
-        delta_ids: Optional[dict],
-        current_state_ids: dict,
+        delta_ids: Optional[StateMap[str]],
+        current_state_ids: StateMap[str],
     ) -> int:
         """Store a new set of state, returning a newly assigned state group.
 
diff --git a/synapse/storage/types.py b/synapse/storage/types.py
index 17291c9d5e..57f4883bf4 100644
--- a/synapse/storage/types.py
+++ b/synapse/storage/types.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/util/__init__.py b/synapse/storage/util/__init__.py
index bfebb0f644..5e83dba2ed 100644
--- a/synapse/storage/util/__init__.py
+++ b/synapse/storage/util/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py
index d4643c4fdf..c768fdea56 100644
--- a/synapse/storage/util/id_generators.py
+++ b/synapse/storage/util/id_generators.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +16,7 @@
 import threading
 from collections import OrderedDict
 from contextlib import contextmanager
-from typing import Dict, List, Optional, Set, Tuple, Union
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
 
 import attr
 
@@ -91,7 +90,14 @@ class StreamIdGenerator:
             # ... persist event ...
     """
 
-    def __init__(self, db_conn, table, column, extra_tables=[], step=1):
+    def __init__(
+        self,
+        db_conn,
+        table,
+        column,
+        extra_tables: Iterable[Tuple[str, str]] = (),
+        step=1,
+    ):
         assert step != 0
         self._lock = threading.Lock()
         self._step = step
@@ -106,7 +112,7 @@ def __init__(self, db_conn, table, column, extra_tables=[], step=1):
         # insertion ordering will ensure its in the correct ordering.
         #
         # The key and values are the same, but we never look at the values.
-        self._unfinished_ids = OrderedDict()  # type: OrderedDict[int, int]
+        self._unfinished_ids: OrderedDict[int, int] = OrderedDict()
 
     def get_next(self):
         """
@@ -230,15 +236,15 @@ def __init__(
         # Note: If we are a negative stream then we still store all the IDs as
         # positive to make life easier for us, and simply negate the IDs when we
         # return them.
-        self._current_positions = {}  # type: Dict[str, int]
+        self._current_positions: Dict[str, int] = {}
 
         # Set of local IDs that we're still processing. The current position
         # should be less than the minimum of this set (if not empty).
-        self._unfinished_ids = set()  # type: Set[int]
+        self._unfinished_ids: Set[int] = set()
 
         # Set of local IDs that we've processed that are larger than the current
         # position, due to there being smaller unpersisted IDs.
-        self._finished_ids = set()  # type: Set[int]
+        self._finished_ids: Set[int] = set()
 
         # We track the max position where we know everything before has been
         # persisted. This is done by a) looking at the min across all instances
@@ -259,7 +265,7 @@ def __init__(
         self._persisted_upto_position = (
             min(self._current_positions.values()) if self._current_positions else 1
         )
-        self._known_persisted_positions = []  # type: List[int]
+        self._known_persisted_positions: List[int] = []
 
         self._sequence_gen = PostgresSequenceGenerator(sequence_name)
 
@@ -391,6 +397,11 @@ def get_next(self):
                 # ... persist event ...
         """
 
+        # If we have a list of instances that are allowed to write to this
+        # stream, make sure we're in it.
+        if self._writers and self._instance_name not in self._writers:
+            raise Exception("Tried to allocate stream ID on non-writer")
+
         return _MultiWriterCtxManager(self)
 
     def get_next_mult(self, n: int):
@@ -400,6 +411,11 @@ def get_next_mult(self, n: int):
                 # ... persist events ...
         """
 
+        # If we have a list of instances that are allowed to write to this
+        # stream, make sure we're in it.
+        if self._writers and self._instance_name not in self._writers:
+            raise Exception("Tried to allocate stream ID on non-writer")
+
         return _MultiWriterCtxManager(self, n)
 
     def get_next_txn(self, txn: LoggingTransaction):
@@ -410,6 +426,11 @@ def get_next_txn(self, txn: LoggingTransaction):
             # ... persist event ...
         """
 
+        # If we have a list of instances that are allowed to write to this
+        # stream, make sure we're in it.
+        if self._writers and self._instance_name not in self._writers:
+            raise Exception("Tried to allocate stream ID on non-writer")
+
         next_id = self._load_next_id_txn(txn)
 
         with self._lock:
@@ -444,7 +465,7 @@ def _mark_id_as_finished(self, next_id: int):
             self._unfinished_ids.discard(next_id)
             self._finished_ids.add(next_id)
 
-            new_cur = None  # type: Optional[int]
+            new_cur: Optional[int] = None
 
             if self._unfinished_ids:
                 # If there are unfinished IDs then the new position will be the
diff --git a/synapse/storage/util/sequence.py b/synapse/storage/util/sequence.py
index 36a67e7019..bb33e04fb1 100644
--- a/synapse/storage/util/sequence.py
+++ b/synapse/storage/util/sequence.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -209,10 +208,10 @@ def __init__(self, get_first_callback: GetFirstCallbackType):
                  get_next_id_txn; should return the curreent maximum id
         """
         # the callback. this is cleared after it is called, so that it can be GCed.
-        self._callback = get_first_callback  # type: Optional[GetFirstCallbackType]
+        self._callback: Optional[GetFirstCallbackType] = get_first_callback
 
         # The current max value, or None if we haven't looked in the DB yet.
-        self._current_max_id = None  # type: Optional[int]
+        self._current_max_id: Optional[int] = None
         self._lock = threading.Lock()
 
     def get_next_id_txn(self, txn: Cursor) -> int:
@@ -275,7 +274,7 @@ def build_sequence_generator(
             `check_consistency` details.
     """
     if isinstance(database_engine, PostgresEngine):
-        seq = PostgresSequenceGenerator(sequence_name)  # type: SequenceGenerator
+        seq: SequenceGenerator = PostgresSequenceGenerator(sequence_name)
     else:
         seq = LocalSequenceGenerator(get_first_callback)
 
diff --git a/synapse/streams/__init__.py b/synapse/streams/__init__.py
index bfebb0f644..5e83dba2ed 100644
--- a/synapse/streams/__init__.py
+++ b/synapse/streams/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/streams/config.py b/synapse/streams/config.py
index fdda21d165..cf4005984b 100644
--- a/synapse/streams/config.py
+++ b/synapse/streams/config.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -48,20 +47,22 @@ async def from_request(
     ) -> "PaginationConfig":
         direction = parse_string(request, "dir", default="f", allowed_values=["f", "b"])
 
-        from_tok = parse_string(request, "from")
-        to_tok = parse_string(request, "to")
+        from_tok_str = parse_string(request, "from")
+        to_tok_str = parse_string(request, "to")
 
         try:
-            if from_tok == "END":
+            from_tok = None
+            if from_tok_str == "END":
                 from_tok = None  # For backwards compat.
-            elif from_tok:
-                from_tok = await StreamToken.from_string(store, from_tok)
+            elif from_tok_str:
+                from_tok = await StreamToken.from_string(store, from_tok_str)
         except Exception:
             raise SynapseError(400, "'from' parameter is invalid")
 
         try:
-            if to_tok:
-                to_tok = await StreamToken.from_string(store, to_tok)
+            to_tok = None
+            if to_tok_str:
+                to_tok = await StreamToken.from_string(store, to_tok_str)
         except Exception:
             raise SynapseError(400, "'to' parameter is invalid")
 
diff --git a/synapse/streams/events.py b/synapse/streams/events.py
index 92fd5d489f..99b0aac2fb 100644
--- a/synapse/streams/events.py
+++ b/synapse/streams/events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,9 +32,9 @@ class EventSources:
     }
 
     def __init__(self, hs):
-        self.sources = {
+        self.sources: Dict[str, Any] = {
             name: cls(hs) for name, cls in EventSources.SOURCE_TYPES.items()
-        }  # type: Dict[str, Any]
+        }
         self.store = hs.get_datastore()
 
     def get_current_token(self) -> StreamToken:
diff --git a/synapse/types.py b/synapse/types.py
index 00655f6dd2..3e5f0e13e4 100644
--- a/synapse/types.py
+++ b/synapse/types.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
@@ -16,13 +15,11 @@
 import abc
 import re
 import string
-import sys
 from collections import namedtuple
 from typing import (
     TYPE_CHECKING,
     Any,
     Dict,
-    Iterable,
     Mapping,
     MutableMapping,
     Optional,
@@ -52,18 +49,6 @@
     from synapse.appservice.api import ApplicationService
     from synapse.storage.databases.main import DataStore
 
-# define a version of typing.Collection that works on python 3.5
-if sys.version_info[:3] >= (3, 6, 0):
-    from typing import Collection
-else:
-    from typing import Container, Sized
-
-    T_co = TypeVar("T_co", covariant=True)
-
-    class Collection(Iterable[T_co], Container[T_co], Sized):  # type: ignore
-        __slots__ = ()
-
-
 # Define a state map type from type/state_key to T (usually an event ID or
 # event)
 T = TypeVar("T")
@@ -198,14 +183,14 @@ def create_requester(
     )
 
 
-def get_domain_from_id(string):
+def get_domain_from_id(string: str) -> str:
     idx = string.find(":")
     if idx == -1:
         raise SynapseError(400, "Invalid ID: %r" % (string,))
     return string[idx + 1 :]
 
 
-def get_localpart_from_id(string):
+def get_localpart_from_id(string: str) -> str:
     idx = string.find(":")
     if idx == -1:
         raise SynapseError(400, "Invalid ID: %r" % (string,))
@@ -215,9 +200,8 @@ def get_localpart_from_id(string):
 DS = TypeVar("DS", bound="DomainSpecificString")
 
 
-class DomainSpecificString(
-    namedtuple("DomainSpecificString", ("localpart", "domain")), metaclass=abc.ABCMeta
-):
+@attr.s(slots=True, frozen=True, repr=False)
+class DomainSpecificString(metaclass=abc.ABCMeta):
     """Common base class among ID/name strings that have a local part and a
     domain name, prefixed with a sigil.
 
@@ -227,13 +211,10 @@ class DomainSpecificString(
         'domain' : The domain part of the name
     """
 
-    SIGIL = abc.abstractproperty()  # type: str  # type: ignore
+    SIGIL: str = abc.abstractproperty()  # type: ignore
 
-    # Deny iteration because it will bite you if you try to create a singleton
-    # set by:
-    #    users = set(user)
-    def __iter__(self):
-        raise ValueError("Attempted to iterate a %s" % (type(self).__name__,))
+    localpart = attr.ib(type=str)
+    domain = attr.ib(type=str)
 
     # Because this class is a namedtuple of strings and booleans, it is deeply
     # immutable.
@@ -288,30 +269,35 @@ def is_valid(cls: Type[DS], s: str) -> bool:
     __repr__ = to_string
 
 
+@attr.s(slots=True, frozen=True, repr=False)
 class UserID(DomainSpecificString):
     """Structure representing a user ID."""
 
     SIGIL = "@"
 
 
+@attr.s(slots=True, frozen=True, repr=False)
 class RoomAlias(DomainSpecificString):
     """Structure representing a room name."""
 
     SIGIL = "#"
 
 
+@attr.s(slots=True, frozen=True, repr=False)
 class RoomID(DomainSpecificString):
-    """Structure representing a room id. """
+    """Structure representing a room id."""
 
     SIGIL = "!"
 
 
+@attr.s(slots=True, frozen=True, repr=False)
 class EventID(DomainSpecificString):
-    """Structure representing an event id. """
+    """Structure representing an event id."""
 
     SIGIL = "$"
 
 
+@attr.s(slots=True, frozen=True, repr=False)
 class GroupID(DomainSpecificString):
     """Structure representing a group ID."""
 
@@ -319,7 +305,7 @@ class GroupID(DomainSpecificString):
 
     @classmethod
     def from_string(cls: Type[DS], s: str) -> DS:
-        group_id = super().from_string(s)  # type: DS # type: ignore
+        group_id: DS = super().from_string(s)  # type: ignore
 
         if not group_id.localpart:
             raise SynapseError(400, "Group ID cannot be empty", Codes.INVALID_PARAM)
@@ -432,7 +418,7 @@ def f2(m):
     return username.decode("ascii")
 
 
-@attr.s(frozen=True, slots=True, cmp=False)
+@attr.s(frozen=True, slots=True, order=False)
 class RoomStreamToken:
     """Tokens are positions between events. The token "s1" comes after event 1.
 
@@ -605,10 +591,10 @@ async def to_string(self, store: "DataStore") -> str:
             entries = []
             for name, pos in self.instance_map.items():
                 instance_id = await store.get_id_for_instance(name)
-                entries.append("{}.{}".format(instance_id, pos))
+                entries.append(f"{instance_id}.{pos}")
 
             encoded_map = "~".join(entries)
-            return "m{}~{}".format(self.stream, encoded_map)
+            return f"m{self.stream}~{encoded_map}"
         else:
             return "s%d" % (self.stream,)
 
@@ -628,7 +614,7 @@ class StreamToken:
     groups_key = attr.ib(type=int)
 
     _SEPARATOR = "_"
-    START = None  # type: StreamToken
+    START: "StreamToken"
 
     @classmethod
     async def from_string(cls, store: "DataStore", string: str) -> "StreamToken":
@@ -779,3 +765,32 @@ def get_verify_key_from_cross_signing_key(key_info):
     # and return that one key
     for key_id, key_data in keys.items():
         return (key_id, decode_verify_key_bytes(key_id, decode_base64(key_data)))
+
+
+@attr.s(auto_attribs=True, frozen=True, slots=True)
+class UserInfo:
+    """Holds information about a user. Result of get_userinfo_by_id.
+
+    Attributes:
+        user_id:  ID of the user.
+        appservice_id:  Application service ID that created this user.
+        consent_server_notice_sent:  Version of policy documents the user has been sent.
+        consent_version:  Version of policy documents the user has consented to.
+        creation_ts:  Creation timestamp of the user.
+        is_admin:  True if the user is an admin.
+        is_deactivated:  True if the user has been deactivated.
+        is_guest:  True if the user is a guest user.
+        is_shadow_banned:  True if the user has been shadow-banned.
+        user_type:  User type (None for normal user, 'support' and 'bot' other options).
+    """
+
+    user_id: UserID
+    appservice_id: Optional[int]
+    consent_server_notice_sent: Optional[str]
+    consent_version: Optional[str]
+    user_type: Optional[str]
+    creation_ts: int
+    is_admin: bool
+    is_deactivated: bool
+    is_guest: bool
+    is_shadow_banned: bool
diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index 517686f0a6..b69f562ca5 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,6 +15,7 @@
 import json
 import logging
 import re
+from typing import Pattern
 
 import attr
 from frozendict import frozendict
@@ -27,6 +27,9 @@
 logger = logging.getLogger(__name__)
 
 
+_WILDCARD_RUN = re.compile(r"([\?\*]+)")
+
+
 def _reject_invalid_json(val):
     """Do not allow Infinity, -Infinity, or NaN values in JSON."""
     raise ValueError("Invalid JSON value: '%s'" % val)
@@ -159,25 +162,54 @@ def log_failure(failure, msg, consumeErrors=True):
         return failure
 
 
-def glob_to_regex(glob):
+def glob_to_regex(glob: str, word_boundary: bool = False) -> Pattern:
     """Converts a glob to a compiled regex object.
 
-    The regex is anchored at the beginning and end of the string.
-
     Args:
-        glob (str)
+        glob: pattern to match
+        word_boundary: If True, the pattern will be allowed to match at word boundaries
+           anywhere in the string. Otherwise, the pattern is anchored at the start and
+           end of the string.
 
     Returns:
-        re.RegexObject
+        compiled regex pattern
     """
-    res = ""
-    for c in glob:
-        if c == "*":
-            res = res + ".*"
-        elif c == "?":
-            res = res + "."
+
+    # Patterns with wildcards must be simplified to avoid performance cliffs
+    # - The glob `?**?**?` is equivalent to the glob `???*`
+    # - The glob `???*` is equivalent to the regex `.{3,}`
+    chunks = []
+    for chunk in _WILDCARD_RUN.split(glob):
+        # No wildcards? re.escape()
+        if not _WILDCARD_RUN.match(chunk):
+            chunks.append(re.escape(chunk))
+            continue
+
+        # Wildcards? Simplify.
+        qmarks = chunk.count("?")
+        if "*" in chunk:
+            chunks.append(".{%d,}" % qmarks)
         else:
-            res = res + re.escape(c)
+            chunks.append(".{%d}" % qmarks)
+
+    res = "".join(chunks)
 
-    # \A anchors at start of string, \Z at end of string
-    return re.compile(r"\A" + res + r"\Z", re.IGNORECASE)
+    if word_boundary:
+        res = re_word_boundary(res)
+    else:
+        # \A anchors at start of string, \Z at end of string
+        res = r"\A" + res + r"\Z"
+
+    return re.compile(res, re.IGNORECASE)
+
+
+def re_word_boundary(r: str) -> str:
+    """
+    Adds word boundary characters to the start and end of an
+    expression to require that the match occur as a whole word,
+    but do so respecting the fact that strings starting or ending
+    with non-word characters will change word boundaries.
+    """
+    # we can't use \b as it chokes on unicode. however \W seems to be okay
+    # as shorthand for [^0-9A-Za-z_].
+    return r"(^|\W)%s(\W|$)" % (r,)
diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py
index c3b2d981ea..a3b65aee27 100644
--- a/synapse/util/async_helpers.py
+++ b/synapse/util/async_helpers.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -16,6 +15,7 @@
 
 import collections
 import inspect
+import itertools
 import logging
 from contextlib import contextmanager
 from typing import (
@@ -23,6 +23,7 @@
     Awaitable,
     Callable,
     Dict,
+    Generic,
     Hashable,
     Iterable,
     List,
@@ -39,6 +40,7 @@
 from twisted.internet.defer import CancelledError
 from twisted.internet.interfaces import IReactorTime
 from twisted.python import failure
+from twisted.python.failure import Failure
 
 from synapse.logging.context import (
     PreserveLoggingContext,
@@ -49,8 +51,10 @@
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
 
-class ObservableDeferred:
+
+class ObservableDeferred(Generic[_T]):
     """Wraps a deferred object so that we can add observer deferreds. These
     observer deferreds do not affect the callback chain of the original
     deferred.
@@ -68,7 +72,7 @@ class ObservableDeferred:
 
     __slots__ = ["_deferred", "_observers", "_result"]
 
-    def __init__(self, deferred: defer.Deferred, consumeErrors: bool = False):
+    def __init__(self, deferred: "defer.Deferred[_T]", consumeErrors: bool = False):
         object.__setattr__(self, "_deferred", deferred)
         object.__setattr__(self, "_result", None)
         object.__setattr__(self, "_observers", set())
@@ -113,7 +117,7 @@ def errback(f):
 
         deferred.addCallbacks(callback, errback)
 
-    def observe(self) -> defer.Deferred:
+    def observe(self) -> "defer.Deferred[_T]":
         """Observe the underlying deferred.
 
         This returns a brand new deferred that is resolved when the underlying
@@ -121,7 +125,7 @@ def observe(self) -> defer.Deferred:
         effect the underlying deferred.
         """
         if not self._result:
-            d = defer.Deferred()
+            d: "defer.Deferred[_T]" = defer.Deferred()
 
             def remove(r):
                 self._observers.discard(d)
@@ -135,7 +139,7 @@ def remove(r):
             success, res = self._result
             return defer.succeed(res) if success else defer.fail(res)
 
-    def observers(self) -> List[defer.Deferred]:
+    def observers(self) -> "List[defer.Deferred[_T]]":
         return self._observers
 
     def has_called(self) -> bool:
@@ -144,7 +148,7 @@ def has_called(self) -> bool:
     def has_succeeded(self) -> bool:
         return self._result is not None and self._result[0] is True
 
-    def get_result(self) -> Any:
+    def get_result(self) -> Union[_T, Failure]:
         return self._result[1]
 
     def __getattr__(self, name: str) -> Any:
@@ -161,8 +165,11 @@ def __repr__(self) -> str:
         )
 
 
+T = TypeVar("T")
+
+
 def concurrently_execute(
-    func: Callable, args: Iterable[Any], limit: int
+    func: Callable[[T], Any], args: Iterable[T], limit: int
 ) -> defer.Deferred:
     """Executes the function with each argument concurrently while limiting
     the number of concurrent executions.
@@ -174,20 +181,27 @@ def concurrently_execute(
         limit: Maximum number of conccurent executions.
 
     Returns:
-        Deferred[list]: Resolved when all function invocations have finished.
+        Deferred: Resolved when all function invocations have finished.
     """
     it = iter(args)
 
-    async def _concurrently_execute_inner():
+    async def _concurrently_execute_inner(value: T) -> None:
         try:
             while True:
-                await maybe_awaitable(func(next(it)))
+                await maybe_awaitable(func(value))
+                value = next(it)
         except StopIteration:
             pass
 
+    # We use `itertools.islice` to handle the case where the number of args is
+    # less than the limit, avoiding needlessly spawning unnecessary background
+    # tasks.
     return make_deferred_yieldable(
         defer.gatherResults(
-            [run_in_background(_concurrently_execute_inner) for _ in range(limit)],
+            [
+                run_in_background(_concurrently_execute_inner, value)
+                for value in itertools.islice(it, limit)
+            ],
             consumeErrors=True,
         )
     ).addErrback(unwrapFirstError)
@@ -247,7 +261,7 @@ def __init__(
             max_count: The maximum number of concurrent accesses
         """
         if name is None:
-            self.name = id(self)  # type: Union[str, int]
+            self.name: Union[str, int] = id(self)
         else:
             self.name = name
 
@@ -259,7 +273,7 @@ def __init__(
         self.max_count = max_count
 
         # key_to_defer is a map from the key to a _LinearizerEntry.
-        self.key_to_defer = {}  # type: Dict[Hashable, _LinearizerEntry]
+        self.key_to_defer: Dict[Hashable, _LinearizerEntry] = {}
 
     def is_queued(self, key: Hashable) -> bool:
         """Checks whether there is a process queued up waiting"""
@@ -399,13 +413,13 @@ class ReadWriteLock:
 
     def __init__(self):
         # Latest readers queued
-        self.key_to_current_readers = {}  # type: Dict[str, Set[defer.Deferred]]
+        self.key_to_current_readers: Dict[str, Set[defer.Deferred]] = {}
 
         # Latest writer queued
-        self.key_to_current_writer = {}  # type: Dict[str, defer.Deferred]
+        self.key_to_current_writer: Dict[str, defer.Deferred] = {}
 
     async def read(self, key: str) -> ContextManager:
-        new_defer = defer.Deferred()
+        new_defer: "defer.Deferred[None]" = defer.Deferred()
 
         curr_readers = self.key_to_current_readers.setdefault(key, set())
         curr_writer = self.key_to_current_writer.get(key, None)
@@ -428,7 +442,7 @@ def _ctx_manager():
         return _ctx_manager()
 
     async def write(self, key: str) -> ContextManager:
-        new_defer = defer.Deferred()
+        new_defer: "defer.Deferred[None]" = defer.Deferred()
 
         curr_readers = self.key_to_current_readers.get(key, set())
         curr_writer = self.key_to_current_writer.get(key, None)
@@ -461,10 +475,8 @@ def _ctx_manager():
 
 
 def timeout_deferred(
-    deferred: defer.Deferred,
-    timeout: float,
-    reactor: IReactorTime,
-) -> defer.Deferred:
+    deferred: "defer.Deferred[_T]", timeout: float, reactor: IReactorTime
+) -> "defer.Deferred[_T]":
     """The in built twisted `Deferred.addTimeout` fails to time out deferreds
     that have a canceller that throws exceptions. This method creates a new
     deferred that wraps and times out the given deferred, correctly handling
@@ -487,7 +499,7 @@ def timeout_deferred(
     Returns:
         A new Deferred, which will errback with defer.TimeoutError on timeout.
     """
-    new_d = defer.Deferred()
+    new_d: "defer.Deferred[_T]" = defer.Deferred()
 
     timed_out = [False]
 
diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py
new file mode 100644
index 0000000000..274cea7eb7
--- /dev/null
+++ b/synapse/util/batching_queue.py
@@ -0,0 +1,177 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import (
+    Awaitable,
+    Callable,
+    Dict,
+    Generic,
+    Hashable,
+    List,
+    Set,
+    Tuple,
+    TypeVar,
+)
+
+from prometheus_client import Gauge
+
+from twisted.internet import defer
+
+from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable
+from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.util import Clock
+
+logger = logging.getLogger(__name__)
+
+
+V = TypeVar("V")
+R = TypeVar("R")
+
+number_queued = Gauge(
+    "synapse_util_batching_queue_number_queued",
+    "The number of items waiting in the queue across all keys",
+    labelnames=("name",),
+)
+
+number_in_flight = Gauge(
+    "synapse_util_batching_queue_number_pending",
+    "The number of items across all keys either being processed or waiting in a queue",
+    labelnames=("name",),
+)
+
+number_of_keys = Gauge(
+    "synapse_util_batching_queue_number_of_keys",
+    "The number of distinct keys that have items queued",
+    labelnames=("name",),
+)
+
+
+class BatchingQueue(Generic[V, R]):
+    """A queue that batches up work, calling the provided processing function
+    with all pending work (for a given key).
+
+    The provided processing function will only be called once at a time for each
+    key. It will be called the next reactor tick after `add_to_queue` has been
+    called, and will keep being called until the queue has been drained (for the
+    given key).
+
+    If the processing function raises an exception then the exception is proxied
+    through to the callers waiting on that batch of work.
+
+    Note that the return value of `add_to_queue` will be the return value of the
+    processing function that processed the given item. This means that the
+    returned value will likely include data for other items that were in the
+    batch.
+
+    Args:
+        name: A name for the queue, used for logging contexts and metrics.
+            This must be unique, otherwise the metrics will be wrong.
+        clock: The clock to use to schedule work.
+        process_batch_callback: The callback to to be run to process a batch of
+            work.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        clock: Clock,
+        process_batch_callback: Callable[[List[V]], Awaitable[R]],
+    ):
+        self._name = name
+        self._clock = clock
+
+        # The set of keys currently being processed.
+        self._processing_keys: Set[Hashable] = set()
+
+        # The currently pending batch of values by key, with a Deferred to call
+        # with the result of the corresponding `_process_batch_callback` call.
+        self._next_values: Dict[Hashable, List[Tuple[V, defer.Deferred]]] = {}
+
+        # The function to call with batches of values.
+        self._process_batch_callback = process_batch_callback
+
+        number_queued.labels(self._name).set_function(
+            lambda: sum(len(q) for q in self._next_values.values())
+        )
+
+        number_of_keys.labels(self._name).set_function(lambda: len(self._next_values))
+
+        self._number_in_flight_metric: Gauge = number_in_flight.labels(self._name)
+
+    async def add_to_queue(self, value: V, key: Hashable = ()) -> R:
+        """Adds the value to the queue with the given key, returning the result
+        of the processing function for the batch that included the given value.
+
+        The optional `key` argument allows sharding the queue by some key. The
+        queues will then be processed in parallel, i.e. the process batch
+        function will be called in parallel with batched values from a single
+        key.
+        """
+
+        # First we create a defer and add it and the value to the list of
+        # pending items.
+        d = defer.Deferred()
+        self._next_values.setdefault(key, []).append((value, d))
+
+        # If we're not currently processing the key fire off a background
+        # process to start processing.
+        if key not in self._processing_keys:
+            run_as_background_process(self._name, self._process_queue, key)
+
+        with self._number_in_flight_metric.track_inprogress():
+            return await make_deferred_yieldable(d)
+
+    async def _process_queue(self, key: Hashable) -> None:
+        """A background task to repeatedly pull things off the queue for the
+        given key and call the `self._process_batch_callback` with the values.
+        """
+
+        if key in self._processing_keys:
+            return
+
+        try:
+            self._processing_keys.add(key)
+
+            while True:
+                # We purposefully wait a reactor tick to allow us to batch
+                # together requests that we're about to receive. A common
+                # pattern is to call `add_to_queue` multiple times at once, and
+                # deferring to the next reactor tick allows us to batch all of
+                # those up.
+                await self._clock.sleep(0)
+
+                next_values = self._next_values.pop(key, [])
+                if not next_values:
+                    # We've exhausted the queue.
+                    break
+
+                try:
+                    values = [value for value, _ in next_values]
+                    results = await self._process_batch_callback(values)
+
+                    with PreserveLoggingContext():
+                        for _, deferred in next_values:
+                            deferred.callback(results)
+
+                except Exception as e:
+                    with PreserveLoggingContext():
+                        for _, deferred in next_values:
+                            if deferred.called:
+                                continue
+
+                            deferred.errback(e)
+
+        finally:
+            self._processing_keys.discard(key)
diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py
index 48f64eeb38..9012034b7a 100644
--- a/synapse/util/caches/__init__.py
+++ b/synapse/util/caches/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 #
@@ -25,14 +24,24 @@
 
 logger = logging.getLogger(__name__)
 
-caches_by_name = {}  # type: Dict[str, Sized]
-collectors_by_name = {}  # type: Dict[str, CacheMetric]
+
+# Whether to track estimated memory usage of the LruCaches.
+TRACK_MEMORY_USAGE = False
+
+
+caches_by_name: Dict[str, Sized] = {}
+collectors_by_name: Dict[str, "CacheMetric"] = {}
 
 cache_size = Gauge("synapse_util_caches_cache:size", "", ["name"])
 cache_hits = Gauge("synapse_util_caches_cache:hits", "", ["name"])
 cache_evicted = Gauge("synapse_util_caches_cache:evicted_size", "", ["name"])
 cache_total = Gauge("synapse_util_caches_cache:total", "", ["name"])
 cache_max_size = Gauge("synapse_util_caches_cache_max_size", "", ["name"])
+cache_memory_usage = Gauge(
+    "synapse_util_caches_cache_size_bytes",
+    "Estimated memory usage of the caches",
+    ["name"],
+)
 
 response_cache_size = Gauge("synapse_util_caches_response_cache:size", "", ["name"])
 response_cache_hits = Gauge("synapse_util_caches_response_cache:hits", "", ["name"])
@@ -53,6 +62,7 @@ class CacheMetric:
     hits = attr.ib(default=0)
     misses = attr.ib(default=0)
     evicted_size = attr.ib(default=0)
+    memory_usage = attr.ib(default=None)
 
     def inc_hits(self):
         self.hits += 1
@@ -63,6 +73,19 @@ def inc_misses(self):
     def inc_evictions(self, size=1):
         self.evicted_size += size
 
+    def inc_memory_usage(self, memory: int):
+        if self.memory_usage is None:
+            self.memory_usage = 0
+
+        self.memory_usage += memory
+
+    def dec_memory_usage(self, memory: int):
+        self.memory_usage -= memory
+
+    def clear_memory_usage(self):
+        if self.memory_usage is not None:
+            self.memory_usage = 0
+
     def describe(self):
         return []
 
@@ -82,6 +105,13 @@ def collect(self):
                 cache_total.labels(self._cache_name).set(self.hits + self.misses)
                 if getattr(self._cache, "max_size", None):
                     cache_max_size.labels(self._cache_name).set(self._cache.max_size)
+
+                if TRACK_MEMORY_USAGE:
+                    # self.memory_usage can be None if nothing has been inserted
+                    # into the cache yet.
+                    cache_memory_usage.labels(self._cache_name).set(
+                        self.memory_usage or 0
+                    )
             if self._collect_callback:
                 self._collect_callback()
         except Exception as e:
diff --git a/synapse/util/caches/cached_call.py b/synapse/util/caches/cached_call.py
index 3ee0f2317a..e58dd91eda 100644
--- a/synapse/util/caches/cached_call.py
+++ b/synapse/util/caches/cached_call.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import enum
 from typing import Awaitable, Callable, Generic, Optional, TypeVar, Union
 
 from twisted.internet.defer import Deferred
@@ -23,6 +22,10 @@
 TV = TypeVar("TV")
 
 
+class _Sentinel(enum.Enum):
+    sentinel = object()
+
+
 class CachedCall(Generic[TV]):
     """A wrapper for asynchronous calls whose results should be shared
 
@@ -64,9 +67,9 @@ def __init__(self, f: Callable[[], Awaitable[TV]]):
             f: The underlying function. Only one call to this function will be alive
                 at once (per instance of CachedCall)
         """
-        self._callable = f  # type: Optional[Callable[[], Awaitable[TV]]]
-        self._deferred = None  # type: Optional[Deferred]
-        self._result = None  # type: Union[None, Failure, TV]
+        self._callable: Optional[Callable[[], Awaitable[TV]]] = f
+        self._deferred: Optional[Deferred] = None
+        self._result: Union[_Sentinel, TV, Failure] = _Sentinel.sentinel
 
     async def get(self) -> TV:
         """Kick off the call if necessary, and return the result"""
@@ -79,8 +82,9 @@ async def get(self) -> TV:
             self._callable = None
 
             # once the deferred completes, store the result. We cannot simply leave the
-            # result in the deferred, since if it's a Failure, GCing the deferred
-            # would then log a critical error about unhandled Failures.
+            # result in the deferred, since `awaiting` a deferred destroys its result.
+            # (Also, if it's a Failure, GCing the deferred would log a critical error
+            # about unhandled Failures)
             def got_result(r):
                 self._result = r
 
@@ -93,13 +97,15 @@ def got_result(r):
         #    and any eventual exception may not be reported.
 
         # we can now await the deferred, and once it completes, return the result.
-        await make_deferred_yieldable(self._deferred)
+        if isinstance(self._result, _Sentinel):
+            await make_deferred_yieldable(self._deferred)
+            assert not isinstance(self._result, _Sentinel)
+
+        if isinstance(self._result, Failure):
+            self._result.raiseException()
+            raise AssertionError("unexpected return from Failure.raiseException")
 
-        # I *think* this is the easiest way to correctly raise a Failure without having
-        # to gut-wrench into the implementation of Deferred.
-        d = Deferred()
-        d.callback(self._result)
-        return await d
+        return self._result
 
 
 class RetryOnExceptionCachedCall(Generic[TV]):
diff --git a/synapse/util/caches/deferred_cache.py b/synapse/util/caches/deferred_cache.py
index 1adc92eb90..b6456392cd 100644
--- a/synapse/util/caches/deferred_cache.py
+++ b/synapse/util/caches/deferred_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
@@ -71,7 +70,6 @@ def __init__(
         self,
         name: str,
         max_entries: int = 1000,
-        keylen: int = 1,
         tree: bool = False,
         iterable: bool = False,
         apply_cache_factor_from_config: bool = True,
@@ -91,26 +89,25 @@ def __init__(
         cache_type = TreeCache if tree else dict
 
         # _pending_deferred_cache maps from the key value to a `CacheEntry` object.
-        self._pending_deferred_cache = (
-            cache_type()
-        )  # type: MutableMapping[KT, CacheEntry]
+        self._pending_deferred_cache: Union[
+            TreeCache, "MutableMapping[KT, CacheEntry]"
+        ] = cache_type()
 
         def metrics_cb():
             cache_pending_metric.labels(name).set(len(self._pending_deferred_cache))
 
         # cache is used for completed results and maps to the result itself, rather than
         # a Deferred.
-        self.cache = LruCache(
+        self.cache: LruCache[KT, VT] = LruCache(
             max_size=max_entries,
-            keylen=keylen,
             cache_name=name,
             cache_type=cache_type,
             size_callback=(lambda d: len(d) or 1) if iterable else None,
             metrics_collection_callback=metrics_cb,
             apply_cache_factor_from_config=apply_cache_factor_from_config,
-        )  # type: LruCache[KT, VT]
+        )
 
-        self.thread = None  # type: Optional[threading.Thread]
+        self.thread: Optional[threading.Thread] = None
 
     @property
     def max_entries(self):
@@ -178,7 +175,7 @@ def get_immediate(
     def set(
         self,
         key: KT,
-        value: defer.Deferred,
+        value: "defer.Deferred[VT]",
         callback: Optional[Callable[[], None]] = None,
     ) -> defer.Deferred:
         """Adds a new entry to the cache (or updates an existing one).
@@ -226,7 +223,7 @@ def set(
         if value.called:
             result = value.result
             if not isinstance(result, failure.Failure):
-                self.cache.set(key, result, callbacks)
+                self.cache.set(key, cast(VT, result), callbacks)
             return value
 
         # otherwise, we'll add an entry to the _pending_deferred_cache for now,
@@ -283,13 +280,24 @@ def eb(_fail):
         # we return a new Deferred which will be called before any subsequent observers.
         return observable.observe()
 
-    def prefill(self, key: KT, value: VT, callback: Callable[[], None] = None):
+    def prefill(
+        self, key: KT, value: VT, callback: Optional[Callable[[], None]] = None
+    ):
         callbacks = [callback] if callback else []
         self.cache.set(key, value, callbacks=callbacks)
 
     def invalidate(self, key):
+        """Delete a key, or tree of entries
+
+        If the cache is backed by a regular dict, then "key" must be of
+        the right type for this cache
+
+        If the cache is backed by a TreeCache, then "key" must be a tuple, but
+        may be of lower cardinality than the TreeCache - in which case the whole
+        subtree is deleted.
+        """
         self.check_thread()
-        self.cache.pop(key, None)
+        self.cache.del_multi(key)
 
         # if we have a pending lookup for this key, remove it from the
         # _pending_deferred_cache, which will (a) stop it being returned
@@ -300,20 +308,10 @@ def invalidate(self, key):
         # run the invalidation callbacks now, rather than waiting for the
         # deferred to resolve.
         if entry:
-            entry.invalidate()
-
-    def invalidate_many(self, key: KT):
-        self.check_thread()
-        if not isinstance(key, tuple):
-            raise TypeError("The cache key must be a tuple not %r" % (type(key),))
-        key = cast(KT, key)
-        self.cache.del_multi(key)
-
-        # if we have a pending lookup for this key, remove it from the
-        # _pending_deferred_cache, as above
-        entry_dict = self._pending_deferred_cache.pop(key, None)
-        if entry_dict is not None:
-            for entry in iterate_tree_cache_entry(entry_dict):
+            # _pending_deferred_cache.pop should either return a CacheEntry, or, in the
+            # case of a TreeCache, a dict of keys to cache entries. Either way calling
+            # iterate_tree_cache_entry on it will do the right thing.
+            for entry in iterate_tree_cache_entry(entry):
                 entry.invalidate()
 
     def invalidate_all(self):
diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index 4e84379914..1ca31e41ac 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -47,18 +46,17 @@
 
 
 class _CachedFunction(Generic[F]):
-    invalidate = None  # type: Any
-    invalidate_all = None  # type: Any
-    invalidate_many = None  # type: Any
-    prefill = None  # type: Any
-    cache = None  # type: Any
-    num_args = None  # type: Any
+    invalidate: Any = None
+    invalidate_all: Any = None
+    prefill: Any = None
+    cache: Any = None
+    num_args: Any = None
 
-    __name__ = None  # type: str
+    __name__: str
 
     # Note: This function signature is actually fiddled with by the synapse mypy
     # plugin to a) make it a bound method, and b) remove any `cache_context` arg.
-    __call__ = None  # type: F
+    __call__: F
 
 
 class _CacheDescriptorBase:
@@ -117,8 +115,8 @@ def __init__(self, orig: Callable[..., Any], num_args, cache_context=False):
 
 
 class _LruCachedFunction(Generic[F]):
-    cache = None  # type: LruCache[CacheKey, Any]
-    __call__ = None  # type: F
+    cache: LruCache[CacheKey, Any]
+    __call__: F
 
 
 def lru_cache(
@@ -182,10 +180,10 @@ def __init__(
         self.max_entries = max_entries
 
     def __get__(self, obj, owner):
-        cache = LruCache(
+        cache: LruCache[CacheKey, Any] = LruCache(
             cache_name=self.orig.__name__,
             max_size=self.max_entries,
-        )  # type: LruCache[CacheKey, Any]
+        )
 
         get_cache_key = self.cache_key_builder
         sentinel = LruCacheDescriptor._Sentinel.sentinel
@@ -263,18 +261,22 @@ def __init__(
     ):
         super().__init__(orig, num_args=num_args, cache_context=cache_context)
 
+        if tree and self.num_args < 2:
+            raise RuntimeError(
+                "tree=True is nonsensical for cached functions with a single parameter"
+            )
+
         self.max_entries = max_entries
         self.tree = tree
         self.iterable = iterable
 
     def __get__(self, obj, owner):
-        cache = DeferredCache(
+        cache: DeferredCache[CacheKey, Any] = DeferredCache(
             name=self.orig.__name__,
             max_entries=self.max_entries,
-            keylen=self.num_args,
             tree=self.tree,
             iterable=self.iterable,
-        )  # type: DeferredCache[CacheKey, Any]
+        )
 
         get_cache_key = self.cache_key_builder
 
@@ -304,11 +306,11 @@ def _wrapped(*args, **kwargs):
         wrapped = cast(_CachedFunction, _wrapped)
 
         if self.num_args == 1:
+            assert not self.tree
             wrapped.invalidate = lambda key: cache.invalidate(key[0])
             wrapped.prefill = lambda key, val: cache.prefill(key[0], val)
         else:
             wrapped.invalidate = cache.invalidate
-            wrapped.invalidate_many = cache.invalidate_many
             wrapped.prefill = cache.prefill
 
         wrapped.invalidate_all = cache.invalidate_all
@@ -323,8 +325,8 @@ def _wrapped(*args, **kwargs):
 class DeferredCacheListDescriptor(_CacheDescriptorBase):
     """Wraps an existing cache to support bulk fetching of keys.
 
-    Given a list of keys it looks in the cache to find any hits, then passes
-    the list of missing keys to the wrapped function.
+    Given an iterable of keys it looks in the cache to find any hits, then passes
+    the tuple of missing keys to the wrapped function.
 
     Once wrapped, the function returns a Deferred which resolves to the list
     of results.
@@ -357,7 +359,7 @@ def __init__(self, orig, cached_method_name, list_name, num_args=None):
 
     def __get__(self, obj, objtype=None):
         cached_method = getattr(obj, self.cached_method_name)
-        cache = cached_method.cache  # type: DeferredCache[CacheKey, Any]
+        cache: DeferredCache[CacheKey, Any] = cached_method.cache
         num_args = cached_method.num_args
 
         @functools.wraps(self.orig)
@@ -411,7 +413,7 @@ def arg_to_cache_key(arg):
                 # relevant result for that key.
                 deferreds_map = {}
                 for arg in missing:
-                    deferred = defer.Deferred()
+                    deferred: "defer.Deferred[Any]" = defer.Deferred()
                     deferreds_map[arg] = deferred
                     key = arg_to_cache_key(arg)
                     cache.set(key, deferred, callback=invalidate_callback)
@@ -438,7 +440,9 @@ def errback(f):
                     return f
 
                 args_to_call = dict(arg_dict)
-                args_to_call[self.list_name] = list(missing)
+                # copy the missing set before sending it to the callee, to guard against
+                # modification.
+                args_to_call[self.list_name] = tuple(missing)
 
                 cached_defers.append(
                     defer.maybeDeferred(
@@ -468,15 +472,15 @@ class _CacheContext:
 
     Cache = Union[DeferredCache, LruCache]
 
-    _cache_context_objects = (
-        WeakValueDictionary()
-    )  # type: WeakValueDictionary[Tuple[_CacheContext.Cache, CacheKey], _CacheContext]
+    _cache_context_objects: """WeakValueDictionary[
+        Tuple["_CacheContext.Cache", CacheKey], "_CacheContext"
+    ]""" = WeakValueDictionary()
 
     def __init__(self, cache: "_CacheContext.Cache", cache_key: CacheKey) -> None:
         self._cache = cache
         self._cache_key = cache_key
 
-    def invalidate(self):  # type: () -> None
+    def invalidate(self) -> None:
         """Invalidates the cache entry referred to by the context."""
         self._cache.invalidate(self._cache_key)
 
@@ -523,14 +527,14 @@ def cachedList(
 
     Used to do batch lookups for an already created cache. A single argument
     is specified as a list that is iterated through to lookup keys in the
-    original cache. A new list consisting of the keys that weren't in the cache
-    get passed to the original function, the result of which is stored in the
+    original cache. A new tuple consisting of the (deduplicated) keys that weren't in
+    the cache gets passed to the original function, the result of which is stored in the
     cache.
 
     Args:
         cached_method_name: The name of the single-item lookup method.
             This is only used to find the cache to use.
-        list_name: The name of the argument that is the list to use to
+        list_name: The name of the argument that is the iterable to use to
             do batch lookups in the cache.
         num_args: Number of arguments to use as the key in the cache
             (including list_name). Defaults to all named parameters.
diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py
index b3b413b02c..3f852edd7f 100644
--- a/synapse/util/caches/dictionary_cache.py
+++ b/synapse/util/caches/dictionary_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -63,13 +62,13 @@ class DictionaryCache(Generic[KT, DKT]):
     """
 
     def __init__(self, name: str, max_entries: int = 1000):
-        self.cache = LruCache(
+        self.cache: LruCache[KT, DictionaryEntry] = LruCache(
             max_size=max_entries, cache_name=name, size_callback=len
-        )  # type: LruCache[KT, DictionaryEntry]
+        )
 
         self.name = name
         self.sequence = 0
-        self.thread = None  # type: Optional[threading.Thread]
+        self.thread: Optional[threading.Thread] = None
 
     def check_thread(self) -> None:
         expected_thread = self.thread
diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index e15f7ee698..bde16b8577 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,40 +14,50 @@
 
 import logging
 from collections import OrderedDict
+from typing import Any, Generic, Optional, TypeVar, Union, overload
+
+import attr
+from typing_extensions import Literal
 
 from synapse.config import cache as cache_config
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.util import Clock
 from synapse.util.caches import register_cache
 
 logger = logging.getLogger(__name__)
 
 
-SENTINEL = object()
+SENTINEL: Any = object()
+
 
+T = TypeVar("T")
+KT = TypeVar("KT")
+VT = TypeVar("VT")
 
-class ExpiringCache:
+
+class ExpiringCache(Generic[KT, VT]):
     def __init__(
         self,
-        cache_name,
-        clock,
-        max_len=0,
-        expiry_ms=0,
-        reset_expiry_on_get=False,
-        iterable=False,
+        cache_name: str,
+        clock: Clock,
+        max_len: int = 0,
+        expiry_ms: int = 0,
+        reset_expiry_on_get: bool = False,
+        iterable: bool = False,
     ):
         """
         Args:
-            cache_name (str): Name of this cache, used for logging.
-            clock (Clock)
-            max_len (int): Max size of dict. If the dict grows larger than this
+            cache_name: Name of this cache, used for logging.
+            clock
+            max_len: Max size of dict. If the dict grows larger than this
                 then the oldest items get automatically evicted. Default is 0,
                 which indicates there is no max limit.
-            expiry_ms (int): How long before an item is evicted from the cache
+            expiry_ms: How long before an item is evicted from the cache
                 in milliseconds. Default is 0, indicating items never get
                 evicted based on time.
-            reset_expiry_on_get (bool): If true, will reset the expiry time for
+            reset_expiry_on_get: If true, will reset the expiry time for
                 an item on access. Defaults to False.
-            iterable (bool): If true, the size is calculated by summing the
+            iterable: If true, the size is calculated by summing the
                 sizes of all entries, rather than the number of entries.
         """
         self._cache_name = cache_name
@@ -62,7 +71,7 @@ def __init__(
         self._expiry_ms = expiry_ms
         self._reset_expiry_on_get = reset_expiry_on_get
 
-        self._cache = OrderedDict()
+        self._cache: OrderedDict[KT, _CacheEntry] = OrderedDict()
 
         self.iterable = iterable
 
@@ -79,12 +88,12 @@ def f():
 
         self._clock.looping_call(f, self._expiry_ms / 2)
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: KT, value: VT) -> None:
         now = self._clock.time_msec()
         self._cache[key] = _CacheEntry(now, value)
         self.evict()
 
-    def evict(self):
+    def evict(self) -> None:
         # Evict if there are now too many items
         while self._max_size and len(self) > self._max_size:
             _key, value = self._cache.popitem(last=False)
@@ -93,7 +102,7 @@ def evict(self):
             else:
                 self.metrics.inc_evictions()
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: KT) -> VT:
         try:
             entry = self._cache[key]
             self.metrics.inc_hits()
@@ -106,7 +115,7 @@ def __getitem__(self, key):
 
         return entry.value
 
-    def pop(self, key, default=SENTINEL):
+    def pop(self, key: KT, default: T = SENTINEL) -> Union[VT, T]:
         """Removes and returns the value with the given key from the cache.
 
         If the key isn't in the cache then `default` will be returned if
@@ -115,29 +124,40 @@ def pop(self, key, default=SENTINEL):
         Identical functionality to `dict.pop(..)`.
         """
 
-        value = self._cache.pop(key, default)
+        value = self._cache.pop(key, SENTINEL)
+        # The key was not found.
         if value is SENTINEL:
-            raise KeyError(key)
+            if default is SENTINEL:
+                raise KeyError(key)
+            return default
 
-        return value
+        return value.value
 
-    def __contains__(self, key):
+    def __contains__(self, key: KT) -> bool:
         return key in self._cache
 
-    def get(self, key, default=None):
+    @overload
+    def get(self, key: KT, default: Literal[None] = None) -> Optional[VT]:
+        ...
+
+    @overload
+    def get(self, key: KT, default: T) -> Union[VT, T]:
+        ...
+
+    def get(self, key: KT, default: Optional[T] = None) -> Union[VT, Optional[T]]:
         try:
             return self[key]
         except KeyError:
             return default
 
-    def setdefault(self, key, value):
+    def setdefault(self, key: KT, value: VT) -> VT:
         try:
             return self[key]
         except KeyError:
             self[key] = value
             return value
 
-    def _prune_cache(self):
+    def _prune_cache(self) -> None:
         if not self._expiry_ms:
             # zero expiry time means don't expire. This should never get called
             # since we have this check in start too.
@@ -166,7 +186,7 @@ def _prune_cache(self):
             len(self),
         )
 
-    def __len__(self):
+    def __len__(self) -> int:
         if self.iterable:
             return sum(len(entry.value) for entry in self._cache.values())
         else:
@@ -190,9 +210,7 @@ def set_cache_factor(self, factor: float) -> bool:
         return False
 
 
+@attr.s(slots=True)
 class _CacheEntry:
-    __slots__ = ["time", "value"]
-
-    def __init__(self, time, value):
-        self.time = time
-        self.value = value
+    time = attr.ib(type=int)
+    value = attr.ib()
diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py
index 60bb6ff642..5c65d187b6 100644
--- a/synapse/util/caches/lrucache.py
+++ b/synapse/util/caches/lrucache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,13 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import threading
+import weakref
 from functools import wraps
 from typing import (
+    TYPE_CHECKING,
     Any,
     Callable,
+    Collection,
     Generic,
     Iterable,
+    List,
     Optional,
     Type,
     TypeVar,
@@ -30,9 +34,45 @@
 
 from typing_extensions import Literal
 
+from twisted.internet import reactor
+
 from synapse.config import cache as cache_config
+from synapse.metrics.background_process_metrics import wrap_as_background_process
+from synapse.util import Clock, caches
 from synapse.util.caches import CacheMetric, register_cache
-from synapse.util.caches.treecache import TreeCache
+from synapse.util.caches.treecache import TreeCache, iterate_tree_cache_entry
+from synapse.util.linked_list import ListNode
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+try:
+    from pympler.asizeof import Asizer
+
+    def _get_size_of(val: Any, *, recurse=True) -> int:
+        """Get an estimate of the size in bytes of the object.
+
+        Args:
+            val: The object to size.
+            recurse: If true will include referenced values in the size,
+                otherwise only sizes the given object.
+        """
+        # Ignore singleton values when calculating memory usage.
+        if val in ((), None, ""):
+            return 0
+
+        sizer = Asizer()
+        sizer.exclude_refs((), None, "")
+        return sizer.asizeof(val, limit=100 if recurse else 0)
+
+
+except ImportError:
+
+    def _get_size_of(val: Any, *, recurse=True) -> int:
+        return 0
+
 
 # Function type: the type used for invalidation callbacks
 FT = TypeVar("FT", bound=Callable[..., Any])
@@ -50,26 +90,217 @@ def enumerate_leaves(node, depth):
         yield node
     else:
         for n in node.values():
-            for m in enumerate_leaves(n, depth - 1):
-                yield m
+            yield from enumerate_leaves(n, depth - 1)
+
+
+P = TypeVar("P")
+
+
+class _TimedListNode(ListNode[P]):
+    """A `ListNode` that tracks last access time."""
+
+    __slots__ = ["last_access_ts_secs"]
+
+    def update_last_access(self, clock: Clock):
+        self.last_access_ts_secs = int(clock.time())
+
+
+# Whether to insert new cache entries to the global list. We only add to it if
+# time based eviction is enabled.
+USE_GLOBAL_LIST = False
+
+# A linked list of all cache entries, allowing efficient time based eviction.
+GLOBAL_ROOT = ListNode["_Node"].create_root_node()
+
+
+@wrap_as_background_process("LruCache._expire_old_entries")
+async def _expire_old_entries(clock: Clock, expiry_seconds: int):
+    """Walks the global cache list to find cache entries that haven't been
+    accessed in the given number of seconds.
+    """
+
+    now = int(clock.time())
+    node = GLOBAL_ROOT.prev_node
+    assert node is not None
+
+    i = 0
+
+    logger.debug("Searching for stale caches")
+
+    while node is not GLOBAL_ROOT:
+        # Only the root node isn't a `_TimedListNode`.
+        assert isinstance(node, _TimedListNode)
+
+        if node.last_access_ts_secs > now - expiry_seconds:
+            break
+
+        cache_entry = node.get_cache_entry()
+        next_node = node.prev_node
+
+        # The node should always have a reference to a cache entry and a valid
+        # `prev_node`, as we only drop them when we remove the node from the
+        # list.
+        assert next_node is not None
+        assert cache_entry is not None
+        cache_entry.drop_from_cache()
+
+        # If we do lots of work at once we yield to allow other stuff to happen.
+        if (i + 1) % 10000 == 0:
+            logger.debug("Waiting during drop")
+            await clock.sleep(0)
+            logger.debug("Waking during drop")
+
+        node = next_node
+
+        # If we've yielded then our current node may have been evicted, so we
+        # need to check that its still valid.
+        if node.prev_node is None:
+            break
+
+        i += 1
+
+    logger.info("Dropped %d items from caches", i)
+
+
+def setup_expire_lru_cache_entries(hs: "HomeServer"):
+    """Start a background job that expires all cache entries if they have not
+    been accessed for the given number of seconds.
+    """
+    if not hs.config.caches.expiry_time_msec:
+        return
+
+    logger.info(
+        "Expiring LRU caches after %d seconds", hs.config.caches.expiry_time_msec / 1000
+    )
+
+    global USE_GLOBAL_LIST
+    USE_GLOBAL_LIST = True
+
+    clock = hs.get_clock()
+    clock.looping_call(
+        _expire_old_entries, 30 * 1000, clock, hs.config.caches.expiry_time_msec / 1000
+    )
 
 
 class _Node:
-    __slots__ = ["prev_node", "next_node", "key", "value", "callbacks"]
+    __slots__ = [
+        "_list_node",
+        "_global_list_node",
+        "_cache",
+        "key",
+        "value",
+        "callbacks",
+        "memory",
+    ]
+
+    def __init__(
+        self,
+        root: "ListNode[_Node]",
+        key,
+        value,
+        cache: "weakref.ReferenceType[LruCache]",
+        clock: Clock,
+        callbacks: Collection[Callable[[], None]] = (),
+    ):
+        self._list_node = ListNode.insert_after(self, root)
+        self._global_list_node = None
+        if USE_GLOBAL_LIST:
+            self._global_list_node = _TimedListNode.insert_after(self, GLOBAL_ROOT)
+            self._global_list_node.update_last_access(clock)
+
+        # We store a weak reference to the cache object so that this _Node can
+        # remove itself from the cache. If the cache is dropped we ensure we
+        # remove our entries in the lists.
+        self._cache = cache
 
-    def __init__(self, prev_node, next_node, key, value, callbacks=set()):
-        self.prev_node = prev_node
-        self.next_node = next_node
         self.key = key
         self.value = value
-        self.callbacks = callbacks
+
+        # Set of callbacks to run when the node gets deleted. We store as a list
+        # rather than a set to keep memory usage down (and since we expect few
+        # entries per node, the performance of checking for duplication in a
+        # list vs using a set is negligible).
+        #
+        # Note that we store this as an optional list to keep the memory
+        # footprint down. Storing `None` is free as its a singleton, while empty
+        # lists are 56 bytes (and empty sets are 216 bytes, if we did the naive
+        # thing and used sets).
+        self.callbacks: Optional[List[Callable[[], None]]] = None
+
+        self.add_callbacks(callbacks)
+
+        self.memory = 0
+        if caches.TRACK_MEMORY_USAGE:
+            self.memory = (
+                _get_size_of(key)
+                + _get_size_of(value)
+                + _get_size_of(self._list_node, recurse=False)
+                + _get_size_of(self.callbacks, recurse=False)
+                + _get_size_of(self, recurse=False)
+            )
+            self.memory += _get_size_of(self.memory, recurse=False)
+
+            if self._global_list_node:
+                self.memory += _get_size_of(self._global_list_node, recurse=False)
+                self.memory += _get_size_of(self._global_list_node.last_access_ts_secs)
+
+    def add_callbacks(self, callbacks: Collection[Callable[[], None]]) -> None:
+        """Add to stored list of callbacks, removing duplicates."""
+
+        if not callbacks:
+            return
+
+        if not self.callbacks:
+            self.callbacks = []
+
+        for callback in callbacks:
+            if callback not in self.callbacks:
+                self.callbacks.append(callback)
+
+    def run_and_clear_callbacks(self) -> None:
+        """Run all callbacks and clear the stored list of callbacks. Used when
+        the node is being deleted.
+        """
+
+        if not self.callbacks:
+            return
+
+        for callback in self.callbacks:
+            callback()
+
+        self.callbacks = None
+
+    def drop_from_cache(self) -> None:
+        """Drop this node from the cache.
+
+        Ensures that the entry gets removed from the cache and that we get
+        removed from all lists.
+        """
+        cache = self._cache()
+        if not cache or not cache.pop(self.key, None):
+            # `cache.pop` should call `drop_from_lists()`, unless this Node had
+            # already been removed from the cache.
+            self.drop_from_lists()
+
+    def drop_from_lists(self) -> None:
+        """Remove this node from the cache lists."""
+        self._list_node.remove_from_list()
+
+        if self._global_list_node:
+            self._global_list_node.remove_from_list()
+
+    def move_to_front(self, clock: Clock, cache_list_root: ListNode) -> None:
+        """Moves this node to the front of all the lists its in."""
+        self._list_node.move_after(cache_list_root)
+        if self._global_list_node:
+            self._global_list_node.move_after(GLOBAL_ROOT)
+            self._global_list_node.update_last_access(clock)
 
 
 class LruCache(Generic[KT, VT]):
     """
     Least-recently-used cache, supporting prometheus metrics and invalidation callbacks.
 
-    Supports del_multi only if cache_type=TreeCache
     If cache_type=TreeCache, all keys must be tuples.
     """
 
@@ -77,11 +308,11 @@ def __init__(
         self,
         max_size: int,
         cache_name: Optional[str] = None,
-        keylen: int = 1,
         cache_type: Type[Union[dict, TreeCache]] = dict,
         size_callback: Optional[Callable] = None,
         metrics_collection_callback: Optional[Callable[[], None]] = None,
         apply_cache_factor_from_config: bool = True,
+        clock: Optional[Clock] = None,
     ):
         """
         Args:
@@ -90,9 +321,6 @@ def __init__(
             cache_name: The name of this cache, for the prometheus metrics. If unset,
                 no metrics will be reported on this cache.
 
-            keylen: The length of the tuple used as the cache key. Ignored unless
-                cache_type is `TreeCache`.
-
             cache_type (type):
                 type of underlying cache to be used. Typically one of dict
                 or TreeCache.
@@ -110,6 +338,13 @@ def __init__(
             apply_cache_factor_from_config (bool): If true, `max_size` will be
                 multiplied by a cache factor derived from the homeserver config
         """
+        # Default `clock` to something sensible. Note that we rename it to
+        # `real_clock` so that mypy doesn't think its still `Optional`.
+        if clock is None:
+            real_clock = Clock(reactor)
+        else:
+            real_clock = clock
+
         cache = cache_type()
         self.cache = cache  # Used for introspection.
         self.apply_cache_factor_from_config = apply_cache_factor_from_config
@@ -126,32 +361,46 @@ def __init__(
 
         # register_cache might call our "set_cache_factor" callback; there's nothing to
         # do yet when we get resized.
-        self._on_resize = None  # type: Optional[Callable[[],None]]
+        self._on_resize: Optional[Callable[[], None]] = None
 
         if cache_name is not None:
-            metrics = register_cache(
+            metrics: Optional[CacheMetric] = register_cache(
                 "lru_cache",
                 cache_name,
                 self,
                 collect_callback=metrics_collection_callback,
-            )  # type: Optional[CacheMetric]
+            )
         else:
             metrics = None
 
         # this is exposed for access from outside this class
         self.metrics = metrics
 
-        list_root = _Node(None, None, None, None)
-        list_root.next_node = list_root
-        list_root.prev_node = list_root
+        # We create a single weakref to self here so that we don't need to keep
+        # creating more each time we create a `_Node`.
+        weak_ref_to_self = weakref.ref(self)
+
+        list_root = ListNode[_Node].create_root_node()
 
         lock = threading.Lock()
 
         def evict():
             while cache_len() > self.max_size:
+                # Get the last node in the list (i.e. the oldest node).
                 todelete = list_root.prev_node
-                evicted_len = delete_node(todelete)
-                cache.pop(todelete.key, None)
+
+                # The list root should always have a valid `prev_node` if the
+                # cache is not empty.
+                assert todelete is not None
+
+                # The node should always have a reference to a cache entry, as
+                # we only drop the cache entry when we remove the node from the
+                # list.
+                node = todelete.get_cache_entry()
+                assert node is not None
+
+                evicted_len = delete_node(node)
+                cache.pop(node.key, None)
                 if metrics:
                     metrics.inc_evictions(evicted_len)
 
@@ -176,50 +425,39 @@ def cache_len():
 
         self.len = synchronized(cache_len)
 
-        def add_node(key, value, callbacks=set()):
-            prev_node = list_root
-            next_node = prev_node.next_node
-            node = _Node(prev_node, next_node, key, value, callbacks)
-            prev_node.next_node = node
-            next_node.prev_node = node
+        def add_node(key, value, callbacks: Collection[Callable[[], None]] = ()):
+            node = _Node(list_root, key, value, weak_ref_to_self, real_clock, callbacks)
             cache[key] = node
 
             if size_callback:
                 cached_cache_len[0] += size_callback(node.value)
 
-        def move_node_to_front(node):
-            prev_node = node.prev_node
-            next_node = node.next_node
-            prev_node.next_node = next_node
-            next_node.prev_node = prev_node
-            prev_node = list_root
-            next_node = prev_node.next_node
-            node.prev_node = prev_node
-            node.next_node = next_node
-            prev_node.next_node = node
-            next_node.prev_node = node
-
-        def delete_node(node):
-            prev_node = node.prev_node
-            next_node = node.next_node
-            prev_node.next_node = next_node
-            next_node.prev_node = prev_node
+            if caches.TRACK_MEMORY_USAGE and metrics:
+                metrics.inc_memory_usage(node.memory)
+
+        def move_node_to_front(node: _Node):
+            node.move_to_front(real_clock, list_root)
+
+        def delete_node(node: _Node) -> int:
+            node.drop_from_lists()
 
             deleted_len = 1
             if size_callback:
                 deleted_len = size_callback(node.value)
                 cached_cache_len[0] -= deleted_len
 
-            for cb in node.callbacks:
-                cb()
-            node.callbacks.clear()
+            node.run_and_clear_callbacks()
+
+            if caches.TRACK_MEMORY_USAGE and metrics:
+                metrics.dec_memory_usage(node.memory)
+
             return deleted_len
 
         @overload
         def cache_get(
             key: KT,
             default: Literal[None] = None,
-            callbacks: Iterable[Callable[[], None]] = ...,
+            callbacks: Collection[Callable[[], None]] = ...,
             update_metrics: bool = ...,
         ) -> Optional[VT]:
             ...
@@ -228,7 +466,7 @@ def cache_get(
         def cache_get(
             key: KT,
             default: T,
-            callbacks: Iterable[Callable[[], None]] = ...,
+            callbacks: Collection[Callable[[], None]] = ...,
             update_metrics: bool = ...,
         ) -> Union[T, VT]:
             ...
@@ -237,13 +475,13 @@ def cache_get(
         def cache_get(
             key: KT,
             default: Optional[T] = None,
-            callbacks: Iterable[Callable[[], None]] = [],
+            callbacks: Collection[Callable[[], None]] = (),
             update_metrics: bool = True,
         ):
             node = cache.get(key, None)
             if node is not None:
                 move_node_to_front(node)
-                node.callbacks.update(callbacks)
+                node.add_callbacks(callbacks)
                 if update_metrics and metrics:
                     metrics.inc_hits()
                 return node.value
@@ -253,16 +491,14 @@ def cache_get(
                 return default
 
         @synchronized
-        def cache_set(key: KT, value: VT, callbacks: Iterable[Callable[[], None]] = []):
+        def cache_set(key: KT, value: VT, callbacks: Iterable[Callable[[], None]] = ()):
             node = cache.get(key, None)
             if node is not None:
                 # We sometimes store large objects, e.g. dicts, which cause
                 # the inequality check to take a long time. So let's only do
                 # the check if we have some callbacks to call.
-                if node.callbacks and value != node.value:
-                    for cb in node.callbacks:
-                        cb()
-                    node.callbacks.clear()
+                if value != node.value:
+                    node.run_and_clear_callbacks()
 
                 # We don't bother to protect this by value != node.value as
                 # generally size_callback will be cheap compared with equality
@@ -272,7 +508,7 @@ def cache_set(key: KT, value: VT, callbacks: Iterable[Callable[[], None]] = []):
                     cached_cache_len[0] -= size_callback(node.value)
                     cached_cache_len[0] += size_callback(value)
 
-                node.callbacks.update(callbacks)
+                node.add_callbacks(callbacks)
 
                 move_node_to_front(node)
                 node.value = value
@@ -311,26 +547,39 @@ def cache_pop(key: KT, default: Optional[T] = None):
 
         @synchronized
         def cache_del_multi(key: KT) -> None:
+            """Delete an entry, or tree of entries
+
+            If the LruCache is backed by a regular dict, then "key" must be of
+            the right type for this cache
+
+            If the LruCache is backed by a TreeCache, then "key" must be a tuple, but
+            may be of lower cardinality than the TreeCache - in which case the whole
+            subtree is deleted.
             """
-            This will only work if constructed with cache_type=TreeCache
-            """
-            popped = cache.pop(key)
+            popped = cache.pop(key, None)
             if popped is None:
                 return
-            for leaf in enumerate_leaves(popped, keylen - len(cast(tuple, key))):
+            # for each deleted node, we now need to remove it from the linked list
+            # and run its callbacks.
+            for leaf in iterate_tree_cache_entry(popped):
                 delete_node(leaf)
 
         @synchronized
         def cache_clear() -> None:
-            list_root.next_node = list_root
-            list_root.prev_node = list_root
             for node in cache.values():
-                for cb in node.callbacks:
-                    cb()
+                node.run_and_clear_callbacks()
+                node.drop_from_lists()
+
+            assert list_root.next_node == list_root
+            assert list_root.prev_node == list_root
+
             cache.clear()
             if size_callback:
                 cached_cache_len[0] = 0
 
+            if caches.TRACK_MEMORY_USAGE and metrics:
+                metrics.clear_memory_usage()
+
         @synchronized
         def cache_contains(key: KT) -> bool:
             return key in cache
@@ -344,11 +593,10 @@ def cache_contains(key: KT) -> bool:
         self.set = cache_set
         self.setdefault = cache_set_default
         self.pop = cache_pop
+        self.del_multi = cache_del_multi
         # `invalidate` is exposed for consistency with DeferredCache, so that it can be
         # invalidated by the cache invalidation replication stream.
-        self.invalidate = cache_pop
-        if cache_type is TreeCache:
-            self.del_multi = cache_del_multi
+        self.invalidate = cache_del_multi
         self.len = synchronized(cache_len)
         self.contains = cache_contains
         self.clear = cache_clear
@@ -394,3 +642,11 @@ def set_cache_factor(self, factor: float) -> bool:
                 self._on_resize()
             return True
         return False
+
+    def __del__(self) -> None:
+        # We're about to be deleted, so we make sure to clear up all the nodes
+        # and run callbacks, etc.
+        #
+        # This happens e.g. in the sync code where we have an expiring cache of
+        # lru caches.
+        self.clear()
diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py
index 46ea8e0964..ed7204336f 100644
--- a/synapse/util/caches/response_cache.py
+++ b/synapse/util/caches/response_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import Any, Callable, Dict, Generic, Optional, TypeVar
+from typing import Any, Awaitable, Callable, Dict, Generic, Optional, TypeVar
+
+import attr
 
 from twisted.internet import defer
 
@@ -24,10 +25,36 @@
 
 logger = logging.getLogger(__name__)
 
-T = TypeVar("T")
+# the type of the key in the cache
+KV = TypeVar("KV")
+
+# the type of the result from the operation
+RV = TypeVar("RV")
+
 
+@attr.s(auto_attribs=True)
+class ResponseCacheContext(Generic[KV]):
+    """Information about a missed ResponseCache hit
 
-class ResponseCache(Generic[T]):
+    This object can be passed into the callback for additional feedback
+    """
+
+    cache_key: KV
+    """The cache key that caused the cache miss
+
+    This should be considered read-only.
+
+    TODO: in attrs 20.1, make it frozen with an on_setattr.
+    """
+
+    should_cache: bool = True
+    """Whether the result should be cached once the request completes.
+
+    This can be modified by the callback if it decides its result should not be cached.
+    """
+
+
+class ResponseCache(Generic[KV]):
     """
     This caches a deferred response. Until the deferred completes it will be
     returned from the cache. This means that if the client retries the request
@@ -36,8 +63,10 @@ class ResponseCache(Generic[T]):
     """
 
     def __init__(self, clock: Clock, name: str, timeout_ms: float = 0):
-        # Requests that haven't finished yet.
-        self.pending_result_cache = {}  # type: Dict[T, ObservableDeferred]
+        # This is poorly-named: it includes both complete and incomplete results.
+        # We keep complete results rather than switching to absolute values because
+        # that makes it easier to cache Failure results.
+        self.pending_result_cache: Dict[KV, ObservableDeferred] = {}
 
         self.clock = clock
         self.timeout_sec = timeout_ms / 1000.0
@@ -51,16 +80,13 @@ def size(self) -> int:
     def __len__(self) -> int:
         return self.size()
 
-    def get(self, key: T) -> Optional[defer.Deferred]:
+    def get(self, key: KV) -> Optional[defer.Deferred]:
         """Look up the given key.
 
-        Can return either a new Deferred (which also doesn't follow the synapse
-        logcontext rules), or, if the request has completed, the actual
-        result. You will probably want to make_deferred_yieldable the result.
+        Returns a new Deferred (which also doesn't follow the synapse
+        logcontext rules). You will probably want to make_deferred_yieldable the result.
 
-        If there is no entry for the key, returns None. It is worth noting that
-        this means there is no way to distinguish a completed result of None
-        from an absent cache entry.
+        If there is no entry for the key, returns None.
 
         Args:
             key: key to get/set in the cache
@@ -77,42 +103,56 @@ def get(self, key: T) -> Optional[defer.Deferred]:
             self._metrics.inc_misses()
             return None
 
-    def set(self, key: T, deferred: defer.Deferred) -> defer.Deferred:
+    def _set(
+        self, context: ResponseCacheContext[KV], deferred: defer.Deferred
+    ) -> defer.Deferred:
         """Set the entry for the given key to the given deferred.
 
         *deferred* should run its callbacks in the sentinel logcontext (ie,
         you should wrap normal synapse deferreds with
         synapse.logging.context.run_in_background).
 
-        Can return either a new Deferred (which also doesn't follow the synapse
-        logcontext rules), or, if *deferred* was already complete, the actual
-        result. You will probably want to make_deferred_yieldable the result.
+        Returns a new Deferred (which also doesn't follow the synapse logcontext rules).
+        You will probably want to make_deferred_yieldable the result.
 
         Args:
-            key: key to get/set in the cache
+            context: Information about the cache miss
             deferred: The deferred which resolves to the result.
 
         Returns:
             A new deferred which resolves to the actual result.
         """
         result = ObservableDeferred(deferred, consumeErrors=True)
+        key = context.cache_key
         self.pending_result_cache[key] = result
 
-        def remove(r):
-            if self.timeout_sec:
+        def on_complete(r):
+            # if this cache has a non-zero timeout, and the callback has not cleared
+            # the should_cache bit, we leave it in the cache for now and schedule
+            # its removal later.
+            if self.timeout_sec and context.should_cache:
                 self.clock.call_later(
                     self.timeout_sec, self.pending_result_cache.pop, key, None
                 )
             else:
+                # otherwise, remove the result immediately.
                 self.pending_result_cache.pop(key, None)
             return r
 
-        result.addBoth(remove)
+        # make sure we do this *after* adding the entry to pending_result_cache,
+        # in case the result is already complete (in which case flipping the order would
+        # leave us with a stuck entry in the cache).
+        result.addBoth(on_complete)
         return result.observe()
 
-    def wrap(
-        self, key: T, callback: "Callable[..., Any]", *args: Any, **kwargs: Any
-    ) -> defer.Deferred:
+    async def wrap(
+        self,
+        key: KV,
+        callback: Callable[..., Awaitable[RV]],
+        *args: Any,
+        cache_context: bool = False,
+        **kwargs: Any,
+    ) -> RV:
         """Wrap together a *get* and *set* call, taking care of logcontexts
 
         First looks up the key in the cache, and if it is present makes it
@@ -141,22 +181,28 @@ async def handle_request(request):
 
             *args: positional parameters to pass to the callback, if it is used
 
+            cache_context: if set, the callback will be given a `cache_context` kw arg,
+                which will be a ResponseCacheContext object.
+
             **kwargs: named parameters to pass to the callback, if it is used
 
         Returns:
-            Deferred which resolves to the result
+            The result of the callback (from the cache, or otherwise)
         """
         result = self.get(key)
         if not result:
             logger.debug(
                 "[%s]: no cached result for [%s], calculating new one", self._name, key
             )
+            context = ResponseCacheContext(cache_key=key)
+            if cache_context:
+                kwargs["cache_context"] = context
             d = run_in_background(callback, *args, **kwargs)
-            result = self.set(key, d)
+            result = self._set(context, d)
         elif not isinstance(result, defer.Deferred) or result.called:
             logger.info("[%s]: using completed cached result for [%s]", self._name, key)
         else:
             logger.info(
                 "[%s]: using incomplete cached result for [%s]", self._name, key
             )
-        return make_deferred_yieldable(result)
+        return await make_deferred_yieldable(result)
diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py
index 644e9e778a..3a41a8baa6 100644
--- a/synapse/util/caches/stream_change_cache.py
+++ b/synapse/util/caches/stream_change_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,11 +14,10 @@
 
 import logging
 import math
-from typing import Dict, FrozenSet, List, Mapping, Optional, Set, Union
+from typing import Collection, Dict, FrozenSet, List, Mapping, Optional, Set, Union
 
 from sortedcontainers import SortedDict
 
-from synapse.types import Collection
 from synapse.util import caches
 
 logger = logging.getLogger(__name__)
@@ -47,10 +45,10 @@ def __init__(
     ):
         self._original_max_size = max_size
         self._max_size = math.floor(max_size)
-        self._entity_to_key = {}  # type: Dict[EntityType, int]
+        self._entity_to_key: Dict[EntityType, int] = {}
 
         # map from stream id to the a set of entities which changed at that stream id.
-        self._cache = SortedDict()  # type: SortedDict[int, Set[EntityType]]
+        self._cache: SortedDict[int, Set[EntityType]] = SortedDict()
 
         # the earliest stream_pos for which we can reliably answer
         # get_all_entities_changed. In other words, one less than the earliest
@@ -157,7 +155,7 @@ def get_all_entities_changed(self, stream_pos: int) -> Optional[List[EntityType]
         if stream_pos < self._earliest_known_stream_pos:
             return None
 
-        changed_entities = []  # type: List[EntityType]
+        changed_entities: List[EntityType] = []
 
         for k in self._cache.islice(start=self._cache.bisect_right(stream_pos)):
             changed_entities.extend(self._cache[k])
diff --git a/synapse/util/caches/treecache.py b/synapse/util/caches/treecache.py
index eb4d98f683..4138931e7b 100644
--- a/synapse/util/caches/treecache.py
+++ b/synapse/util/caches/treecache.py
@@ -1,18 +1,43 @@
-from typing import Dict
+# Copyright 2016-2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 SENTINEL = object()
 
 
+class TreeCacheNode(dict):
+    """The type of nodes in our tree.
+
+    Has its own type so we can distinguish it from real dicts that are stored at the
+    leaves.
+    """
+
+    pass
+
+
 class TreeCache:
     """
     Tree-based backing store for LruCache. Allows subtrees of data to be deleted
     efficiently.
     Keys must be tuples.
+
+    The data structure is a chain of TreeCacheNodes:
+        root = {key_1: {key_2: _value}}
     """
 
     def __init__(self):
         self.size = 0
-        self.root = {}  # type: Dict
+        self.root = TreeCacheNode()
 
     def __setitem__(self, key, value):
         return self.set(key, value)
@@ -21,10 +46,23 @@ def __contains__(self, key):
         return self.get(key, SENTINEL) is not SENTINEL
 
     def set(self, key, value):
+        if isinstance(value, TreeCacheNode):
+            # this would mean we couldn't tell where our tree ended and the value
+            # started.
+            raise ValueError("Cannot store TreeCacheNodes in a TreeCache")
+
         node = self.root
         for k in key[:-1]:
-            node = node.setdefault(k, {})
-        node[key[-1]] = _Entry(value)
+            next_node = node.get(k, SENTINEL)
+            if next_node is SENTINEL:
+                next_node = node[k] = TreeCacheNode()
+            elif not isinstance(next_node, TreeCacheNode):
+                # this suggests that the caller is not being consistent with its key
+                # length.
+                raise ValueError("value conflicts with an existing subtree")
+            node = next_node
+
+        node[key[-1]] = value
         self.size += 1
 
     def get(self, key, default=None):
@@ -33,25 +71,44 @@ def get(self, key, default=None):
             node = node.get(k, None)
             if node is None:
                 return default
-        return node.get(key[-1], _Entry(default)).value
+        return node.get(key[-1], default)
 
     def clear(self):
         self.size = 0
-        self.root = {}
+        self.root = TreeCacheNode()
 
     def pop(self, key, default=None):
+        """Remove the given key, or subkey, from the cache
+
+        Args:
+            key: key or subkey to remove.
+            default: value to return if key is not found
+
+        Returns:
+            If the key is not found, 'default'. If the key is complete, the removed
+            value. If the key is partial, the TreeCacheNode corresponding to the part
+            of the tree that was removed.
+        """
+        if not isinstance(key, tuple):
+            raise TypeError("The cache key must be a tuple not %r" % (type(key),))
+
+        # a list of the nodes we have touched on the way down the tree
         nodes = []
 
         node = self.root
         for k in key[:-1]:
             node = node.get(k, None)
-            nodes.append(node)  # don't add the root node
             if node is None:
                 return default
+            if not isinstance(node, TreeCacheNode):
+                # we've gone off the end of the tree
+                raise ValueError("pop() key too long")
+            nodes.append(node)  # don't add the root node
         popped = node.pop(key[-1], SENTINEL)
         if popped is SENTINEL:
             return default
 
+        # working back up the tree, clear out any nodes that are now empty
         node_and_keys = list(zip(nodes, key))
         node_and_keys.reverse()
         node_and_keys.append((self.root, None))
@@ -61,14 +118,15 @@ def pop(self, key, default=None):
 
             if n:
                 break
+            # found an empty node: remove it from its parent, and loop.
             node_and_keys[i + 1][0].pop(k)
 
-        popped, cnt = _strip_and_count_entires(popped)
+        cnt = sum(1 for _ in iterate_tree_cache_entry(popped))
         self.size -= cnt
         return popped
 
     def values(self):
-        return list(iterate_tree_cache_entry(self.root))
+        return iterate_tree_cache_entry(self.root)
 
     def __len__(self):
         return self.size
@@ -78,36 +136,8 @@ def iterate_tree_cache_entry(d):
     """Helper function to iterate over the leaves of a tree, i.e. a dict of that
     can contain dicts.
     """
-    if isinstance(d, dict):
+    if isinstance(d, TreeCacheNode):
         for value_d in d.values():
-            for value in iterate_tree_cache_entry(value_d):
-                yield value
-    else:
-        if isinstance(d, _Entry):
-            yield d.value
-        else:
-            yield d
-
-
-class _Entry:
-    __slots__ = ["value"]
-
-    def __init__(self, value):
-        self.value = value
-
-
-def _strip_and_count_entires(d):
-    """Takes an _Entry or dict with leaves of _Entry's, and either returns the
-    value or a dictionary with _Entry's replaced by their values.
-
-    Also returns the count of _Entry's
-    """
-    if isinstance(d, dict):
-        cnt = 0
-        for key, value in d.items():
-            v, n = _strip_and_count_entires(value)
-            d[key] = v
-            cnt += n
-        return d, cnt
+            yield from iterate_tree_cache_entry(value_d)
     else:
-        return d.value, 1
+        yield d
diff --git a/synapse/util/caches/ttlcache.py b/synapse/util/caches/ttlcache.py
index 96a8274940..46afe3f934 100644
--- a/synapse/util/caches/ttlcache.py
+++ b/synapse/util/caches/ttlcache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,7 +23,7 @@
 
 logger = logging.getLogger(__name__)
 
-SENTINEL = object()  # type: Any
+SENTINEL: Any = object()
 
 T = TypeVar("T")
 KT = TypeVar("KT")
@@ -36,10 +35,10 @@ class TTLCache(Generic[KT, VT]):
 
     def __init__(self, cache_name: str, timer: Callable[[], float] = time.time):
         # map from key to _CacheEntry
-        self._data = {}  # type: Dict[KT, _CacheEntry]
+        self._data: Dict[KT, _CacheEntry] = {}
 
         # the _CacheEntries, sorted by expiry time
-        self._expiry_list = SortedList()  # type: SortedList[_CacheEntry]
+        self._expiry_list: SortedList[_CacheEntry] = SortedList()
 
         self._timer = timer
 
diff --git a/synapse/util/daemonize.py b/synapse/util/daemonize.py
index 23393cf49b..d8532411c2 100644
--- a/synapse/util/daemonize.py
+++ b/synapse/util/daemonize.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2012, 2013, 2014 Ilya Otyutskiy <ilya.otyutskiy@icloud.com>
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -32,13 +31,13 @@ def daemonize_process(pid_file: str, logger: logging.Logger, chdir: str = "/") -
     # If pidfile already exists, we should read pid from there; to overwrite it, if
     # locking will fail, because locking attempt somehow purges the file contents.
     if os.path.isfile(pid_file):
-        with open(pid_file, "r") as pid_fh:
+        with open(pid_file) as pid_fh:
             old_pid = pid_fh.read()
 
     # Create a lockfile so that only one instance of this daemon is running at any time.
     try:
         lock_fh = open(pid_file, "w")
-    except IOError:
+    except OSError:
         print("Unable to create the pidfile.")
         sys.exit(1)
 
@@ -46,7 +45,7 @@ def daemonize_process(pid_file: str, logger: logging.Logger, chdir: str = "/") -
         # Try to get an exclusive lock on the file. This will fail if another process
         # has the file locked.
         fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
-    except IOError:
+    except OSError:
         print("Unable to lock on the pidfile.")
         # We need to overwrite the pidfile if we got here.
         #
@@ -114,7 +113,7 @@ def excepthook(type_, value, traceback):
     try:
         lock_fh.write("%s" % (os.getpid()))
         lock_fh.flush()
-    except IOError:
+    except OSError:
         logger.error("Unable to write pid to the pidfile.")
         print("Unable to write pid to the pidfile.")
         sys.exit(1)
diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py
index 3c47285d05..1f803aef6d 100644
--- a/synapse/util/distributor.py
+++ b/synapse/util/distributor.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/util/file_consumer.py b/synapse/util/file_consumer.py
index 68dc632491..e946189f9a 100644
--- a/synapse/util/file_consumer.py
+++ b/synapse/util/file_consumer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py
index 5ca2e71e60..2ac7c2913c 100644
--- a/synapse/util/frozenutils.py
+++ b/synapse/util/frozenutils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/util/hash.py b/synapse/util/hash.py
index 359168704e..7625ca8c2c 100644
--- a/synapse/util/hash.py
+++ b/synapse/util/hash.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,15 +17,15 @@
 import unpaddedbase64
 
 
-def sha256_and_url_safe_base64(input_text):
+def sha256_and_url_safe_base64(input_text: str) -> str:
     """SHA256 hash an input string, encode the digest as url-safe base64, and
     return
 
-    :param input_text: string to hash
-    :type input_text: str
+    Args:
+        input_text: string to hash
 
-    :returns a sha256 hashed and url-safe base64 encoded digest
-    :rtype: str
+    returns:
+        A sha256 hashed and url-safe base64 encoded digest
     """
     digest = hashlib.sha256(input_text.encode()).digest()
     return unpaddedbase64.encode_base64(digest, urlsafe=True)
diff --git a/synapse/util/iterutils.py b/synapse/util/iterutils.py
index 98707c119d..8ac3eab2f5 100644
--- a/synapse/util/iterutils.py
+++ b/synapse/util/iterutils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -16,6 +15,7 @@
 import heapq
 from itertools import islice
 from typing import (
+    Collection,
     Dict,
     Generator,
     Iterable,
@@ -27,17 +27,15 @@
     TypeVar,
 )
 
-from synapse.types import Collection
-
 T = TypeVar("T")
 
 
-def batch_iter(iterable: Iterable[T], size: int) -> Iterator[Tuple[T]]:
+def batch_iter(iterable: Iterable[T], size: int) -> Iterator[Tuple[T, ...]]:
     """batch an iterable up into tuples with a maximum size
 
     Args:
-        iterable (iterable): the iterable to slice
-        size (int): the maximum batch size
+        iterable: the iterable to slice
+        size: the maximum batch size
 
     Returns:
         an iterator over the chunks
@@ -48,10 +46,7 @@ def batch_iter(iterable: Iterable[T], size: int) -> Iterator[Tuple[T]]:
     return iter(lambda: tuple(islice(sourceiter, size)), ())
 
 
-ISeq = TypeVar("ISeq", bound=Sequence, covariant=True)
-
-
-def chunk_seq(iseq: ISeq, maxlen: int) -> Iterable[ISeq]:
+def chunk_seq(iseq: Sequence[T], maxlen: int) -> Iterable[Sequence[T]]:
     """Split the given sequence into chunks of the given size
 
     The last chunk may be shorter than the given size.
@@ -73,7 +68,7 @@ def sorted_topologically(
     # This is implemented by Kahn's algorithm.
 
     degree_map = {node: 0 for node in nodes}
-    reverse_graph = {}  # type: Dict[T, Set[T]]
+    reverse_graph: Dict[T, Set[T]] = {}
 
     for node, edges in graph.items():
         if node not in degree_map:
diff --git a/synapse/util/jsonobject.py b/synapse/util/jsonobject.py
deleted file mode 100644
index e3a8ed5b2f..0000000000
--- a/synapse/util/jsonobject.py
+++ /dev/null
@@ -1,103 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-class JsonEncodedObject:
-    """A common base class for defining protocol units that are represented
-    as JSON.
-
-    Attributes:
-        unrecognized_keys (dict): A dict containing all the key/value pairs we
-            don't recognize.
-    """
-
-    valid_keys = []  # keys we will store
-    """A list of strings that represent keys we know about
-    and can handle. If we have values for these keys they will be
-    included in the `dictionary` instance variable.
-    """
-
-    internal_keys = []  # keys to ignore while building dict
-    """A list of strings that should *not* be encoded into JSON.
-    """
-
-    required_keys = []
-    """A list of strings that we require to exist. If they are not given upon
-    construction it raises an exception.
-    """
-
-    def __init__(self, **kwargs):
-        """Takes the dict of `kwargs` and loads all keys that are *valid*
-        (i.e., are included in the `valid_keys` list) into the dictionary`
-        instance variable.
-
-        Any keys that aren't recognized are added to the `unrecognized_keys`
-        attribute.
-
-        Args:
-            **kwargs: Attributes associated with this protocol unit.
-        """
-        for required_key in self.required_keys:
-            if required_key not in kwargs:
-                raise RuntimeError("Key %s is required" % required_key)
-
-        self.unrecognized_keys = {}  # Keys we were given not listed as valid
-        for k, v in kwargs.items():
-            if k in self.valid_keys or k in self.internal_keys:
-                self.__dict__[k] = v
-            else:
-                self.unrecognized_keys[k] = v
-
-    def get_dict(self):
-        """Converts this protocol unit into a :py:class:`dict`, ready to be
-        encoded as JSON.
-
-        The keys it encodes are: `valid_keys` - `internal_keys`
-
-        Returns
-            dict
-        """
-        d = {
-            k: _encode(v)
-            for (k, v) in self.__dict__.items()
-            if k in self.valid_keys and k not in self.internal_keys
-        }
-        d.update(self.unrecognized_keys)
-        return d
-
-    def get_internal_dict(self):
-        d = {
-            k: _encode(v, internal=True)
-            for (k, v) in self.__dict__.items()
-            if k in self.valid_keys
-        }
-        d.update(self.unrecognized_keys)
-        return d
-
-    def __str__(self):
-        return "(%s, %s)" % (self.__class__.__name__, repr(self.__dict__))
-
-
-def _encode(obj, internal=False):
-    if type(obj) is list:
-        return [_encode(o, internal=internal) for o in obj]
-
-    if isinstance(obj, JsonEncodedObject):
-        if internal:
-            return obj.get_internal_dict()
-        else:
-            return obj.get_dict()
-
-    return obj
diff --git a/synapse/util/linked_list.py b/synapse/util/linked_list.py
new file mode 100644
index 0000000000..a456b136f0
--- /dev/null
+++ b/synapse/util/linked_list.py
@@ -0,0 +1,150 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A circular doubly linked list implementation.
+"""
+
+import threading
+from typing import Generic, Optional, Type, TypeVar
+
+P = TypeVar("P")
+LN = TypeVar("LN", bound="ListNode")
+
+
+class ListNode(Generic[P]):
+    """A node in a circular doubly linked list, with an (optional) reference to
+    a cache entry.
+
+    The reference should only be `None` for the root node or if the node has
+    been removed from the list.
+    """
+
+    # A lock to protect mutating the list prev/next pointers.
+    _LOCK = threading.Lock()
+
+    # We don't use attrs here as in py3.6 you can't have `attr.s(slots=True)`
+    # and inherit from `Generic` for some reason
+    __slots__ = [
+        "cache_entry",
+        "prev_node",
+        "next_node",
+    ]
+
+    def __init__(self, cache_entry: Optional[P] = None) -> None:
+        self.cache_entry = cache_entry
+        self.prev_node: Optional[ListNode[P]] = None
+        self.next_node: Optional[ListNode[P]] = None
+
+    @classmethod
+    def create_root_node(cls: Type["ListNode[P]"]) -> "ListNode[P]":
+        """Create a new linked list by creating a "root" node, which is a node
+        that has prev_node/next_node pointing to itself and no associated cache
+        entry.
+        """
+        root = cls()
+        root.prev_node = root
+        root.next_node = root
+        return root
+
+    @classmethod
+    def insert_after(
+        cls: Type[LN],
+        cache_entry: P,
+        node: "ListNode[P]",
+    ) -> LN:
+        """Create a new list node that is placed after the given node.
+
+        Args:
+            cache_entry: The associated cache entry.
+            node: The existing node in the list to insert the new entry after.
+        """
+        new_node = cls(cache_entry)
+        with cls._LOCK:
+            new_node._refs_insert_after(node)
+        return new_node
+
+    def remove_from_list(self):
+        """Remove this node from the list."""
+        with self._LOCK:
+            self._refs_remove_node_from_list()
+
+        # We drop the reference to the cache entry to break the reference cycle
+        # between the list node and cache entry, allowing the two to be dropped
+        # immediately rather than at the next GC.
+        self.cache_entry = None
+
+    def move_after(self, node: "ListNode"):
+        """Move this node from its current location in the list to after the
+        given node.
+        """
+        with self._LOCK:
+            # We assert that both this node and the target node is still "alive".
+            assert self.prev_node
+            assert self.next_node
+            assert node.prev_node
+            assert node.next_node
+
+            assert self is not node
+
+            # Remove self from the list
+            self._refs_remove_node_from_list()
+
+            # Insert self back into the list, after target node
+            self._refs_insert_after(node)
+
+    def _refs_remove_node_from_list(self):
+        """Internal method to *just* remove the node from the list, without
+        e.g. clearing out the cache entry.
+        """
+        if self.prev_node is None or self.next_node is None:
+            # We've already been removed from the list.
+            return
+
+        prev_node = self.prev_node
+        next_node = self.next_node
+
+        prev_node.next_node = next_node
+        next_node.prev_node = prev_node
+
+        # We set these to None so that we don't get circular references,
+        # allowing us to be dropped without having to go via the GC.
+        self.prev_node = None
+        self.next_node = None
+
+    def _refs_insert_after(self, node: "ListNode"):
+        """Internal method to insert the node after the given node."""
+
+        # This method should only be called when we're not already in the list.
+        assert self.prev_node is None
+        assert self.next_node is None
+
+        # We expect the given node to be in the list and thus have valid
+        # prev/next refs.
+        assert node.next_node
+        assert node.prev_node
+
+        prev_node = node
+        next_node = node.next_node
+
+        self.prev_node = prev_node
+        self.next_node = next_node
+
+        prev_node.next_node = self
+        next_node.prev_node = self
+
+    def get_cache_entry(self) -> Optional[P]:
+        """Get the cache entry, returns None if this is the root node (i.e.
+        cache_entry is None) or if the entry has been dropped.
+        """
+        return self.cache_entry
diff --git a/synapse/util/macaroons.py b/synapse/util/macaroons.py
index 12cdd53327..d1f76e3dc5 100644
--- a/synapse/util/macaroons.py
+++ b/synapse/util/macaroons.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Quentin Gliech
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
@@ -40,7 +39,7 @@ def get_value_from_macaroon(macaroon: pymacaroons.Macaroon, key: str) -> str:
              caveat in the macaroon, or if the caveat was not found in the macaroon.
     """
     prefix = key + " = "
-    result = None  # type: Optional[str]
+    result: Optional[str] = None
     for caveat in macaroon.caveats:
         if not caveat.caveat_id.startswith(prefix):
             continue
diff --git a/synapse/util/manhole.py b/synapse/util/manhole.py
index da24ba0470..522daa323d 100644
--- a/synapse/util/manhole.py
+++ b/synapse/util/manhole.py
@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import inspect
 import sys
 import traceback
 
@@ -20,6 +21,7 @@
 from twisted.conch.manhole import ColoredManhole, ManholeInterpreter
 from twisted.conch.ssh.keys import Key
 from twisted.cred import checkers, portal
+from twisted.internet import defer
 
 PUBLIC_KEY = (
     "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHhGATaW4KhE23+7nrH4jFx3yLq9OjaEs5"
@@ -141,3 +143,15 @@ def showtraceback(self):
             self.write("".join(lines))
         finally:
             last_tb = ei = None
+
+    def displayhook(self, obj):
+        """
+        We override the displayhook so that we automatically convert coroutines
+        into Deferreds. (Our superclass' displayhook will take care of the rest,
+        by displaying the Deferred if it's ready, or registering a callback
+        if it's not).
+        """
+        if inspect.iscoroutine(obj):
+            super().displayhook(defer.ensureDeferred(obj))
+        else:
+            super().displayhook(obj)
diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py
index 1023c856d1..1b82dca81b 100644
--- a/synapse/util/metrics.py
+++ b/synapse/util/metrics.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -105,7 +104,13 @@ class Measure:
         "start",
     ]
 
-    def __init__(self, clock, name):
+    def __init__(self, clock, name: str):
+        """
+        Args:
+            clock: A n object with a "time()" method, which returns the current
+                time in seconds.
+            name: The name of the metric to report.
+        """
         self.clock = clock
         self.name = name
         curr_context = current_context()
@@ -118,10 +123,8 @@ def __init__(self, clock, name):
         else:
             assert isinstance(curr_context, LoggingContext)
             parent_context = curr_context
-        self._logging_context = LoggingContext(
-            "Measure[%s]" % (self.name,), parent_context
-        )
-        self.start = None
+        self._logging_context = LoggingContext(str(curr_context), parent_context)
+        self.start: Optional[int] = None
 
     def __enter__(self) -> "Measure":
         if self.start is not None:
@@ -130,12 +133,17 @@ def __enter__(self) -> "Measure":
         self.start = self.clock.time()
         self._logging_context.__enter__()
         in_flight.register((self.name,), self._update_in_flight)
+
+        logger.debug("Entering block %s", self.name)
+
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if self.start is None:
             raise RuntimeError("Measure() block exited without being entered")
 
+        logger.debug("Exiting block %s", self.name)
+
         duration = self.clock.time() - self.start
         usage = self.get_resource_usage()
 
diff --git a/synapse/util/module_loader.py b/synapse/util/module_loader.py
index d184e2a90c..5a638c6e9a 100644
--- a/synapse/util/module_loader.py
+++ b/synapse/util/module_loader.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,6 +15,7 @@
 import importlib
 import importlib.util
 import itertools
+from types import ModuleType
 from typing import Any, Iterable, Tuple, Type
 
 import jsonschema
@@ -45,36 +45,41 @@ def load_module(provider: dict, config_path: Iterable[str]) -> Tuple[Type, Any]:
 
     # We need to import the module, and then pick the class out of
     # that, so we split based on the last dot.
-    module, clz = modulename.rsplit(".", 1)
-    module = importlib.import_module(module)
+    module_name, clz = modulename.rsplit(".", 1)
+    module = importlib.import_module(module_name)
     provider_class = getattr(module, clz)
 
     # Load the module config. If None, pass an empty dictionary instead
     module_config = provider.get("config") or {}
-    try:
-        provider_config = provider_class.parse_config(module_config)
-    except jsonschema.ValidationError as e:
-        raise json_error_to_config_error(e, itertools.chain(config_path, ("config",)))
-    except ConfigError as e:
-        raise _wrap_config_error(
-            "Failed to parse config for module %r" % (modulename,),
-            prefix=itertools.chain(config_path, ("config",)),
-            e=e,
-        )
-    except Exception as e:
-        raise ConfigError(
-            "Failed to parse config for module %r" % (modulename,),
-            path=itertools.chain(config_path, ("config",)),
-        ) from e
+    if hasattr(provider_class, "parse_config"):
+        try:
+            provider_config = provider_class.parse_config(module_config)
+        except jsonschema.ValidationError as e:
+            raise json_error_to_config_error(
+                e, itertools.chain(config_path, ("config",))
+            )
+        except ConfigError as e:
+            raise _wrap_config_error(
+                "Failed to parse config for module %r" % (modulename,),
+                prefix=itertools.chain(config_path, ("config",)),
+                e=e,
+            )
+        except Exception as e:
+            raise ConfigError(
+                "Failed to parse config for module %r" % (modulename,),
+                path=itertools.chain(config_path, ("config",)),
+            ) from e
+    else:
+        provider_config = module_config
 
     return provider_class, provider_config
 
 
-def load_python_module(location: str):
+def load_python_module(location: str) -> ModuleType:
     """Load a python module, and return a reference to its global namespace
 
     Args:
-        location (str): path to the module
+        location: path to the module
 
     Returns:
         python module object
diff --git a/synapse/util/msisdn.py b/synapse/util/msisdn.py
index c8bcbe297a..1046224f15 100644
--- a/synapse/util/msisdn.py
+++ b/synapse/util/msisdn.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,19 +17,19 @@
 from synapse.api.errors import SynapseError
 
 
-def phone_number_to_msisdn(country, number):
+def phone_number_to_msisdn(country: str, number: str) -> str:
     """
     Takes an ISO-3166-1 2 letter country code and phone number and
     returns an msisdn representing the canonical version of that
     phone number.
     Args:
-        country (str): ISO-3166-1 2 letter country code
-        number (str): Phone number in a national or international format
+        country: ISO-3166-1 2 letter country code
+        number: Phone number in a national or international format
 
     Returns:
-        (str) The canonical form of the phone number, as an msisdn
+        The canonical form of the phone number, as an msisdn
     Raises:
-            SynapseError if the number could not be parsed.
+        SynapseError if the number could not be parsed.
     """
     try:
         phoneNumber = phonenumbers.parse(number, country)
diff --git a/synapse/util/patch_inline_callbacks.py b/synapse/util/patch_inline_callbacks.py
index d9f9ae99d6..99f01e325c 100644
--- a/synapse/util/patch_inline_callbacks.py
+++ b/synapse/util/patch_inline_callbacks.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -42,7 +41,7 @@ def new_inline_callbacks(f):
         @functools.wraps(f)
         def wrapped(*args, **kwargs):
             start_context = current_context()
-            changes = []  # type: List[str]
+            changes: List[str] = []
             orig = orig_inline_callbacks(_check_yield_points(f, changes))
 
             try:
@@ -132,7 +131,7 @@ def check_yield_points_inner(*args, **kwargs):
         gen = f(*args, **kwargs)
 
         last_yield_line_no = gen.gi_frame.f_lineno
-        result = None  # type: Any
+        result: Any = None
         while True:
             expected_context = current_context()
 
diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py
index 70d11e1ec3..a654c69684 100644
--- a/synapse/util/ratelimitutils.py
+++ b/synapse/util/ratelimitutils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index 4ab379e429..129b47cd49 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -83,11 +82,9 @@ async def get_retry_limiter(destination, clock, store, ignore_backoff=False, **k
     retry_timings = await store.get_destination_retry_timings(destination)
 
     if retry_timings:
-        failure_ts = retry_timings["failure_ts"]
-        retry_last_ts, retry_interval = (
-            retry_timings["retry_last_ts"],
-            retry_timings["retry_interval"],
-        )
+        failure_ts = retry_timings.failure_ts
+        retry_last_ts = retry_timings.retry_last_ts
+        retry_interval = retry_timings.retry_interval
 
         now = int(clock.time_msec())
 
diff --git a/synapse/util/rlimit.py b/synapse/util/rlimit.py
index 207cd17c2a..bf812ab516 100644
--- a/synapse/util/rlimit.py
+++ b/synapse/util/rlimit.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index 9ce7873ab5..da216242c4 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -14,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import itertools
-import random
 import re
+import secrets
 import string
 from collections.abc import Iterable
 from typing import Optional, Tuple
@@ -36,26 +35,27 @@
 #
 MXC_REGEX = re.compile("^mxc://([^/]+)/([^/#?]+)$")
 
-# random_string and random_string_with_symbols are used for a range of things,
-# some cryptographically important, some less so. We use SystemRandom to make sure
-# we get cryptographically-secure randoms.
-rand = random.SystemRandom()
-
 
 def random_string(length: int) -> str:
-    return "".join(rand.choice(string.ascii_letters) for _ in range(length))
+    """Generate a cryptographically secure string of random letters.
+
+    Drawn from the characters: `a-z` and `A-Z`
+    """
+    return "".join(secrets.choice(string.ascii_letters) for _ in range(length))
 
 
 def random_string_with_symbols(length: int) -> str:
-    return "".join(rand.choice(_string_with_symbols) for _ in range(length))
+    """Generate a cryptographically secure string of random letters/numbers/symbols.
+
+    Drawn from the characters: `a-z`, `A-Z`, `0-9`, and `.,;:^&*-_+=#~@`
+    """
+    return "".join(secrets.choice(_string_with_symbols) for _ in range(length))
 
 
 def is_ascii(s: bytes) -> bool:
     try:
         s.decode("ascii").encode("ascii")
-    except UnicodeDecodeError:
-        return False
-    except UnicodeEncodeError:
+    except UnicodeError:
         return False
     return True
 
@@ -133,6 +133,37 @@ def parse_and_validate_server_name(server_name: str) -> Tuple[str, Optional[int]
     return host, port
 
 
+def valid_id_server_location(id_server: str) -> bool:
+    """Check whether an identity server location, such as the one passed as the
+    `id_server` parameter to `/_matrix/client/r0/account/3pid/bind`, is valid.
+
+    A valid identity server location consists of a valid hostname and optional
+    port number, optionally followed by any number of `/` delimited path
+    components, without any fragment or query string parts.
+
+    Args:
+        id_server: identity server location string to validate
+
+    Returns:
+        True if valid, False otherwise.
+    """
+    components = id_server.split("/", 1)
+
+    host = components[0]
+
+    try:
+        parse_and_validate_server_name(host)
+    except ValueError:
+        return False
+
+    if len(components) < 2:
+        # no path
+        return True
+
+    path = components[1]
+    return "#" not in path and "?" not in path
+
+
 def parse_and_validate_mxc_uri(mxc: str) -> Tuple[str, Optional[int], str]:
     """Parse the given string as an MXC URI
 
@@ -189,3 +220,23 @@ def strtobool(val: str) -> bool:
         return False
     else:
         raise ValueError("invalid truth value %r" % (val,))
+
+
+_BASE62 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+
+
+def base62_encode(num: int, minwidth: int = 1) -> str:
+    """Encode a number using base62
+
+    Args:
+        num: number to be encoded
+        minwidth: width to pad to, if the number is small
+    """
+    res = ""
+    while num:
+        num, rem = divmod(num, 62)
+        res = _BASE62[rem] + res
+
+    # pad to minimum width
+    pad = "0" * (minwidth - len(res))
+    return pad + res
diff --git a/synapse/util/templates.py b/synapse/util/templates.py
index 392dae4a40..38543dd1ea 100644
--- a/synapse/util/templates.py
+++ b/synapse/util/templates.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/util/threepids.py b/synapse/util/threepids.py
index 63f955acff..eb005007d5 100644
--- a/synapse/util/threepids.py
+++ b/synapse/util/threepids.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,8 +18,18 @@
 logger = logging.getLogger(__name__)
 
 
+# it's unclear what the maximum length of an email address is. RFC3696 (as corrected
+# by errata) says:
+#    the upper limit on address lengths should normally be considered to be 254.
+#
+# In practice, mail servers appear to be more tolerant and allow 400 characters
+# or so. Let's allow 500, which should be plenty for everyone.
+#
+MAX_EMAIL_ADDRESS_LENGTH = 500
+
+
 async def check_3pid_allowed(hs, medium, address):
-    """Checks whether a given 3PID is allowed to be used on this HS
+    """Checks whether a given format of 3PID is allowed to be used on this HS
 
     Args:
         hs (synapse.server.HomeServer): server
@@ -98,3 +107,23 @@ def canonicalise_email(address: str) -> str:
         raise ValueError("Unable to parse email address")
 
     return parts[0].casefold() + "@" + parts[1].lower()
+
+
+def validate_email(address: str) -> str:
+    """Does some basic validation on an email address.
+
+    Returns the canonicalised email, as returned by `canonicalise_email`.
+
+    Raises a ValueError if the email is invalid.
+    """
+    # First we try canonicalising in case that fails
+    address = canonicalise_email(address)
+
+    # Email addresses have to be at least 3 characters.
+    if len(address) < 3:
+        raise ValueError("Unable to parse email address")
+
+    if len(address) > MAX_EMAIL_ADDRESS_LENGTH:
+        raise ValueError("Unable to parse email address")
+
+    return address
diff --git a/synapse/util/versionstring.py b/synapse/util/versionstring.py
index ab7d03af3a..cb08af7385 100644
--- a/synapse/util/versionstring.py
+++ b/synapse/util/versionstring.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/util/wheel_timer.py b/synapse/util/wheel_timer.py
index be3b22469d..61814aff24 100644
--- a/synapse/util/wheel_timer.py
+++ b/synapse/util/wheel_timer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synapse/visibility.py b/synapse/visibility.py
index ff53a49b3a..17532059e9 100644
--- a/synapse/visibility.py
+++ b/synapse/visibility.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -91,13 +90,13 @@ async def filter_events_for_client(
         AccountDataTypes.IGNORED_USER_LIST, user_id
     )
 
-    ignore_list = frozenset()  # type: FrozenSet[str]
+    ignore_list: FrozenSet[str] = frozenset()
     if ignore_dict_content:
         ignored_users_dict = ignore_dict_content.get("ignored_users", {})
         if isinstance(ignored_users_dict, dict):
             ignore_list = frozenset(ignored_users_dict.keys())
 
-    erased_senders = await storage.main.are_users_erased((e.sender for e in events))
+    erased_senders = await storage.main.are_users_erased(e.sender for e in events)
 
     if filter_send_to_client:
         room_ids = {e.room_id for e in events}
@@ -354,7 +353,7 @@ def check_event_is_visible(event: EventBase, state: StateMap[EventBase]) -> bool
         )
 
     if not check_history_visibility_only:
-        erased_senders = await storage.main.are_users_erased((e.sender for e in events))
+        erased_senders = await storage.main.are_users_erased(e.sender for e in events)
     else:
         # We don't want to check whether users are erased, which is equivalent
         # to no users having been erased.
diff --git a/synctl b/synctl
index 56c0e3940f..90559ded62 100755
--- a/synctl
+++ b/synctl
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -25,12 +24,13 @@ import signal
 import subprocess
 import sys
 import time
+from typing import Iterable
 
 import yaml
 
 from synapse.config import find_config_files
 
-SYNAPSE = [sys.executable, "-m", "synapse.app.homeserver"]
+MAIN_PROCESS = "synapse.app.homeserver"
 
 GREEN = "\x1b[1;32m"
 YELLOW = "\x1b[1;33m"
@@ -69,73 +69,43 @@ def abort(message, colour=RED, stream=sys.stderr):
     sys.exit(1)
 
 
-def start(configfile: str, daemonize: bool = True) -> bool:
-    """Attempts to start synapse.
+def start(pidfile: str, app: str, config_files: Iterable[str], daemonize: bool) -> bool:
+    """Attempts to start a synapse main or worker process.
     Args:
-        configfile: path to a yaml synapse config file
-        daemonize: whether to daemonize synapse or keep it attached to the current
-            session
+        pidfile: the pidfile we expect the process to create
+        app: the python module to run
+        config_files: config files to pass to synapse
+        daemonize: if True, will include a --daemonize argument to synapse
 
     Returns:
-        True if the process started successfully
+        True if the process started successfully or was already running
         False if there was an error starting the process
-
-        If deamonize is False it will only return once synapse exits.
     """
 
-    write("Starting ...")
-    args = SYNAPSE
-
-    if daemonize:
-        args.extend(["--daemonize", "-c", configfile])
-    else:
-        args.extend(["-c", configfile])
-
-    try:
-        subprocess.check_call(args)
-        write("started synapse.app.homeserver(%r)" % (configfile,), colour=GREEN)
+    if os.path.exists(pidfile) and pid_running(int(open(pidfile).read())):
+        print(app + " already running")
         return True
-    except subprocess.CalledProcessError as e:
-        write(
-            "error starting (exit code: %d); see above for logs" % e.returncode,
-            colour=RED,
-        )
-        return False
-
 
-def start_worker(app: str, configfile: str, worker_configfile: str) -> bool:
-    """Attempts to start a synapse worker.
-    Args:
-        app: name of the worker's appservice
-        configfile: path to a yaml synapse config file
-        worker_configfile: path to worker specific yaml synapse file
-
-    Returns:
-        True if the process started successfully
-        False if there was an error starting the process
-    """
-
-    args = [
-        sys.executable,
-        "-m",
-        app,
-        "-c",
-        configfile,
-        "-c",
-        worker_configfile,
-        "--daemonize",
-    ]
+    args = [sys.executable, "-m", app]
+    for c in config_files:
+        args += ["-c", c]
+    if daemonize:
+        args.append("--daemonize")
 
     try:
         subprocess.check_call(args)
-        write("started %s(%r)" % (app, worker_configfile), colour=GREEN)
+        write("started %s(%s)" % (app, ",".join(config_files)), colour=GREEN)
         return True
     except subprocess.CalledProcessError as e:
-        write(
-            "error starting %s(%r) (exit code: %d); see above for logs"
-            % (app, worker_configfile, e.returncode),
-            colour=RED,
+        err = "%s(%s) failed to start (exit code: %d). Check the Synapse logfile" % (
+            app,
+            ",".join(config_files),
+            e.returncode,
         )
+        if daemonize:
+            err += ", or run synctl with --no-daemonize"
+        err += "."
+        write(err, colour=RED, stream=sys.stderr)
         return False
 
 
@@ -225,10 +195,11 @@ def main():
 
     if not os.path.exists(configfile):
         write(
-            "No config file found\n"
-            "To generate a config file, run '%s -c %s --generate-config"
-            " --server-name=<server name> --report-stats=<yes/no>'\n"
-            % (" ".join(SYNAPSE), options.configfile),
+            f"Config file {configfile} does not exist.\n"
+            f"To generate a config file, run:\n"
+            f"    {sys.executable} -m {MAIN_PROCESS}"
+            f" -c {configfile} --generate-config"
+            f" --server-name=<server name> --report-stats=<yes/no>\n",
             stream=sys.stderr,
         )
         sys.exit(1)
@@ -324,7 +295,7 @@ def main():
                 has_stopped = False
 
         if start_stop_synapse:
-            if not stop(pidfile, "synapse.app.homeserver"):
+            if not stop(pidfile, MAIN_PROCESS):
                 has_stopped = False
         if not has_stopped and action == "stop":
             sys.exit(1)
@@ -347,30 +318,24 @@ def main():
     if action == "start" or action == "restart":
         error = False
         if start_stop_synapse:
-            # Check if synapse is already running
-            if os.path.exists(pidfile) and pid_running(int(open(pidfile).read())):
-                abort("synapse.app.homeserver already running")
-
-            if not start(configfile, bool(options.daemonize)):
+            if not start(pidfile, MAIN_PROCESS, (configfile,), options.daemonize):
                 error = True
 
         for worker in workers:
             env = os.environ.copy()
 
-            # Skip starting a worker if its already running
-            if os.path.exists(worker.pidfile) and pid_running(
-                int(open(worker.pidfile).read())
-            ):
-                print(worker.app + " already running")
-                continue
-
             if worker.cache_factor:
                 os.environ["SYNAPSE_CACHE_FACTOR"] = str(worker.cache_factor)
 
             for cache_name, factor in worker.cache_factors.items():
                 os.environ["SYNAPSE_CACHE_FACTOR_" + cache_name.upper()] = str(factor)
 
-            if not start_worker(worker.app, configfile, worker.configfile):
+            if not start(
+                worker.pidfile,
+                worker.app,
+                (configfile, worker.configfile),
+                options.daemonize,
+            ):
                 error = True
 
             # Reset env back to the original
diff --git a/synmark/__init__.py b/synmark/__init__.py
index 3d4ec3e184..2cc00b0f03 100644
--- a/synmark/__init__.py
+++ b/synmark/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synmark/__main__.py b/synmark/__main__.py
index f55968a5a4..35a59e347a 100644
--- a/synmark/__main__.py
+++ b/synmark/__main__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synmark/suites/logging.py b/synmark/suites/logging.py
index c306891b27..9419892e95 100644
--- a/synmark/suites/logging.py
+++ b/synmark/suites/logging.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,8 +15,7 @@
 import logging
 import warnings
 from io import StringIO
-
-from mock import Mock
+from unittest.mock import Mock
 
 from pyperf import perf_counter
 
diff --git a/synmark/suites/lrucache.py b/synmark/suites/lrucache.py
index 69ab042ccc..9b4a424149 100644
--- a/synmark/suites/lrucache.py
+++ b/synmark/suites/lrucache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/synmark/suites/lrucache_evict.py b/synmark/suites/lrucache_evict.py
index 532b1cc702..0ee202ed36 100644
--- a/synmark/suites/lrucache_evict.py
+++ b/synmark/suites/lrucache_evict.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/__init__.py b/tests/__init__.py
index ed805db1c2..5fced5cc4c 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index 34f72ae795..f76fea4f66 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 import pymacaroons
 
@@ -22,13 +21,11 @@
 from synapse.api.errors import (
     AuthError,
     Codes,
-    InvalidClientCredentialsError,
     InvalidClientTokenError,
     MissingClientTokenError,
     ResourceLimitError,
 )
 from synapse.storage.databases.main.registration import TokenLookupResult
-from synapse.types import UserID
 
 from tests import unittest
 from tests.test_utils import simple_async_mock
@@ -61,6 +58,7 @@ def test_get_user_by_req_user_valid_token(self):
             user_id=self.test_user, token_id=5, device_id="device"
         )
         self.store.get_user_by_access_token = simple_async_mock(user_info)
+        self.store.mark_access_token_as_used = simple_async_mock(None)
 
         request = Mock(args={})
         request.args[b"access_token"] = [self.test_token]
@@ -254,67 +252,6 @@ def test_get_guest_user_from_macaroon(self):
         self.assertTrue(user_info.is_guest)
         self.store.get_user_by_id.assert_called_with(user_id)
 
-    def test_cannot_use_regular_token_as_guest(self):
-        USER_ID = "@percy:matrix.org"
-        self.store.add_access_token_to_user = simple_async_mock(None)
-        self.store.get_device = simple_async_mock(None)
-
-        token = self.get_success(
-            self.hs.get_auth_handler().get_access_token_for_user_id(
-                USER_ID, "DEVICE", valid_until_ms=None
-            )
-        )
-        self.store.add_access_token_to_user.assert_called_with(
-            user_id=USER_ID,
-            token=token,
-            device_id="DEVICE",
-            valid_until_ms=None,
-            puppets_user_id=None,
-        )
-
-        async def get_user(tok):
-            if token != tok:
-                return None
-            return TokenLookupResult(
-                user_id=USER_ID,
-                is_guest=False,
-                token_id=1234,
-                device_id="DEVICE",
-            )
-
-        self.store.get_user_by_access_token = get_user
-        self.store.get_user_by_id = simple_async_mock({"is_guest": False})
-
-        # check the token works
-        request = Mock(args={})
-        request.args[b"access_token"] = [token.encode("ascii")]
-        request.requestHeaders.getRawHeaders = mock_getRawHeaders()
-        requester = self.get_success(
-            self.auth.get_user_by_req(request, allow_guest=True)
-        )
-        self.assertEqual(UserID.from_string(USER_ID), requester.user)
-        self.assertFalse(requester.is_guest)
-
-        # add an is_guest caveat
-        mac = pymacaroons.Macaroon.deserialize(token)
-        mac.add_first_party_caveat("guest = true")
-        guest_tok = mac.serialize()
-
-        # the token should *not* work now
-        request = Mock(args={})
-        request.args[b"access_token"] = [guest_tok.encode("ascii")]
-        request.requestHeaders.getRawHeaders = mock_getRawHeaders()
-
-        cm = self.get_failure(
-            self.auth.get_user_by_req(request, allow_guest=True),
-            InvalidClientCredentialsError,
-        )
-
-        self.assertEqual(401, cm.value.code)
-        self.assertEqual("Guest access token used for regular user", cm.value.msg)
-
-        self.store.get_user_by_id.assert_called_with(USER_ID)
-
     def test_blocking_mau(self):
         self.auth_blocking._limit_usage_by_mau = False
         self.auth_blocking._max_mau_value = 50
diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py
index ab7d290724..f44c91a373 100644
--- a/tests/api/test_filtering.py
+++ b/tests/api/test_filtering.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018-2019 New Vector Ltd
diff --git a/tests/api/test_ratelimiting.py b/tests/api/test_ratelimiting.py
index 483418192c..dcf0110c16 100644
--- a/tests/api/test_ratelimiting.py
+++ b/tests/api/test_ratelimiting.py
@@ -5,38 +5,25 @@
 from tests import unittest
 
 
-class TestRatelimiter(unittest.TestCase):
+class TestRatelimiter(unittest.HomeserverTestCase):
     def test_allowed_via_can_do_action(self):
-        limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
-        allowed, time_allowed = limiter.can_do_action(key="test_id", _time_now_s=0)
-        self.assertTrue(allowed)
-        self.assertEquals(10.0, time_allowed)
-
-        allowed, time_allowed = limiter.can_do_action(key="test_id", _time_now_s=5)
-        self.assertFalse(allowed)
-        self.assertEquals(10.0, time_allowed)
-
-        allowed, time_allowed = limiter.can_do_action(key="test_id", _time_now_s=10)
-        self.assertTrue(allowed)
-        self.assertEquals(20.0, time_allowed)
-
-    def test_allowed_user_via_can_requester_do_action(self):
-        user_requester = create_requester("@user:example.com")
-        limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
-        allowed, time_allowed = limiter.can_requester_do_action(
-            user_requester, _time_now_s=0
+        limiter = Ratelimiter(
+            store=self.hs.get_datastore(), clock=None, rate_hz=0.1, burst_count=1
+        )
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(None, key="test_id", _time_now_s=0)
         )
         self.assertTrue(allowed)
         self.assertEquals(10.0, time_allowed)
 
-        allowed, time_allowed = limiter.can_requester_do_action(
-            user_requester, _time_now_s=5
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(None, key="test_id", _time_now_s=5)
         )
         self.assertFalse(allowed)
         self.assertEquals(10.0, time_allowed)
 
-        allowed, time_allowed = limiter.can_requester_do_action(
-            user_requester, _time_now_s=10
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(None, key="test_id", _time_now_s=10)
         )
         self.assertTrue(allowed)
         self.assertEquals(20.0, time_allowed)
@@ -51,21 +38,23 @@ def test_allowed_appservice_ratelimited_via_can_requester_do_action(self):
         )
         as_requester = create_requester("@user:example.com", app_service=appservice)
 
-        limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
-        allowed, time_allowed = limiter.can_requester_do_action(
-            as_requester, _time_now_s=0
+        limiter = Ratelimiter(
+            store=self.hs.get_datastore(), clock=None, rate_hz=0.1, burst_count=1
+        )
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(as_requester, _time_now_s=0)
         )
         self.assertTrue(allowed)
         self.assertEquals(10.0, time_allowed)
 
-        allowed, time_allowed = limiter.can_requester_do_action(
-            as_requester, _time_now_s=5
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(as_requester, _time_now_s=5)
         )
         self.assertFalse(allowed)
         self.assertEquals(10.0, time_allowed)
 
-        allowed, time_allowed = limiter.can_requester_do_action(
-            as_requester, _time_now_s=10
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(as_requester, _time_now_s=10)
         )
         self.assertTrue(allowed)
         self.assertEquals(20.0, time_allowed)
@@ -80,73 +69,89 @@ def test_allowed_appservice_via_can_requester_do_action(self):
         )
         as_requester = create_requester("@user:example.com", app_service=appservice)
 
-        limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
-        allowed, time_allowed = limiter.can_requester_do_action(
-            as_requester, _time_now_s=0
+        limiter = Ratelimiter(
+            store=self.hs.get_datastore(), clock=None, rate_hz=0.1, burst_count=1
+        )
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(as_requester, _time_now_s=0)
         )
         self.assertTrue(allowed)
         self.assertEquals(-1, time_allowed)
 
-        allowed, time_allowed = limiter.can_requester_do_action(
-            as_requester, _time_now_s=5
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(as_requester, _time_now_s=5)
         )
         self.assertTrue(allowed)
         self.assertEquals(-1, time_allowed)
 
-        allowed, time_allowed = limiter.can_requester_do_action(
-            as_requester, _time_now_s=10
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(as_requester, _time_now_s=10)
         )
         self.assertTrue(allowed)
         self.assertEquals(-1, time_allowed)
 
     def test_allowed_via_ratelimit(self):
-        limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
+        limiter = Ratelimiter(
+            store=self.hs.get_datastore(), clock=None, rate_hz=0.1, burst_count=1
+        )
 
         # Shouldn't raise
-        limiter.ratelimit(key="test_id", _time_now_s=0)
+        self.get_success_or_raise(limiter.ratelimit(None, key="test_id", _time_now_s=0))
 
         # Should raise
         with self.assertRaises(LimitExceededError) as context:
-            limiter.ratelimit(key="test_id", _time_now_s=5)
+            self.get_success_or_raise(
+                limiter.ratelimit(None, key="test_id", _time_now_s=5)
+            )
         self.assertEqual(context.exception.retry_after_ms, 5000)
 
         # Shouldn't raise
-        limiter.ratelimit(key="test_id", _time_now_s=10)
+        self.get_success_or_raise(
+            limiter.ratelimit(None, key="test_id", _time_now_s=10)
+        )
 
     def test_allowed_via_can_do_action_and_overriding_parameters(self):
         """Test that we can override options of can_do_action that would otherwise fail
         an action
         """
         # Create a Ratelimiter with a very low allowed rate_hz and burst_count
-        limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
+        limiter = Ratelimiter(
+            store=self.hs.get_datastore(), clock=None, rate_hz=0.1, burst_count=1
+        )
 
         # First attempt should be allowed
-        allowed, time_allowed = limiter.can_do_action(
-            ("test_id",),
-            _time_now_s=0,
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(
+                None,
+                ("test_id",),
+                _time_now_s=0,
+            )
         )
         self.assertTrue(allowed)
         self.assertEqual(10.0, time_allowed)
 
         # Second attempt, 1s later, will fail
-        allowed, time_allowed = limiter.can_do_action(
-            ("test_id",),
-            _time_now_s=1,
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(
+                None,
+                ("test_id",),
+                _time_now_s=1,
+            )
         )
         self.assertFalse(allowed)
         self.assertEqual(10.0, time_allowed)
 
         # But, if we allow 10 actions/sec for this request, we should be allowed
         # to continue.
-        allowed, time_allowed = limiter.can_do_action(
-            ("test_id",), _time_now_s=1, rate_hz=10.0
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(None, ("test_id",), _time_now_s=1, rate_hz=10.0)
         )
         self.assertTrue(allowed)
         self.assertEqual(1.1, time_allowed)
 
         # Similarly if we allow a burst of 10 actions
-        allowed, time_allowed = limiter.can_do_action(
-            ("test_id",), _time_now_s=1, burst_count=10
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(None, ("test_id",), _time_now_s=1, burst_count=10)
         )
         self.assertTrue(allowed)
         self.assertEqual(1.0, time_allowed)
@@ -156,29 +161,129 @@ def test_allowed_via_ratelimit_and_overriding_parameters(self):
         fail an action
         """
         # Create a Ratelimiter with a very low allowed rate_hz and burst_count
-        limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
+        limiter = Ratelimiter(
+            store=self.hs.get_datastore(), clock=None, rate_hz=0.1, burst_count=1
+        )
 
         # First attempt should be allowed
-        limiter.ratelimit(key=("test_id",), _time_now_s=0)
+        self.get_success_or_raise(
+            limiter.ratelimit(None, key=("test_id",), _time_now_s=0)
+        )
 
         # Second attempt, 1s later, will fail
         with self.assertRaises(LimitExceededError) as context:
-            limiter.ratelimit(key=("test_id",), _time_now_s=1)
+            self.get_success_or_raise(
+                limiter.ratelimit(None, key=("test_id",), _time_now_s=1)
+            )
         self.assertEqual(context.exception.retry_after_ms, 9000)
 
         # But, if we allow 10 actions/sec for this request, we should be allowed
         # to continue.
-        limiter.ratelimit(key=("test_id",), _time_now_s=1, rate_hz=10.0)
+        self.get_success_or_raise(
+            limiter.ratelimit(None, key=("test_id",), _time_now_s=1, rate_hz=10.0)
+        )
 
         # Similarly if we allow a burst of 10 actions
-        limiter.ratelimit(key=("test_id",), _time_now_s=1, burst_count=10)
+        self.get_success_or_raise(
+            limiter.ratelimit(None, key=("test_id",), _time_now_s=1, burst_count=10)
+        )
 
     def test_pruning(self):
-        limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
-        limiter.can_do_action(key="test_id_1", _time_now_s=0)
+        limiter = Ratelimiter(
+            store=self.hs.get_datastore(), clock=None, rate_hz=0.1, burst_count=1
+        )
+        self.get_success_or_raise(
+            limiter.can_do_action(None, key="test_id_1", _time_now_s=0)
+        )
 
         self.assertIn("test_id_1", limiter.actions)
 
-        limiter.can_do_action(key="test_id_2", _time_now_s=10)
+        self.get_success_or_raise(
+            limiter.can_do_action(None, key="test_id_2", _time_now_s=10)
+        )
 
         self.assertNotIn("test_id_1", limiter.actions)
+
+    def test_db_user_override(self):
+        """Test that users that have ratelimiting disabled in the DB aren't
+        ratelimited.
+        """
+        store = self.hs.get_datastore()
+
+        user_id = "@user:test"
+        requester = create_requester(user_id)
+
+        self.get_success(
+            store.db_pool.simple_insert(
+                table="ratelimit_override",
+                values={
+                    "user_id": user_id,
+                    "messages_per_second": None,
+                    "burst_count": None,
+                },
+                desc="test_db_user_override",
+            )
+        )
+
+        limiter = Ratelimiter(store=store, clock=None, rate_hz=0.1, burst_count=1)
+
+        # Shouldn't raise
+        for _ in range(20):
+            self.get_success_or_raise(limiter.ratelimit(requester, _time_now_s=0))
+
+    def test_multiple_actions(self):
+        limiter = Ratelimiter(
+            store=self.hs.get_datastore(), clock=None, rate_hz=0.1, burst_count=3
+        )
+        # Test that 4 actions aren't allowed with a maximum burst of 3.
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(None, key="test_id", n_actions=4, _time_now_s=0)
+        )
+        self.assertFalse(allowed)
+
+        # Test that 3 actions are allowed with a maximum burst of 3.
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(None, key="test_id", n_actions=3, _time_now_s=0)
+        )
+        self.assertTrue(allowed)
+        self.assertEquals(10.0, time_allowed)
+
+        # Test that, after doing these 3 actions, we can't do any more action without
+        # waiting.
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(None, key="test_id", n_actions=1, _time_now_s=0)
+        )
+        self.assertFalse(allowed)
+        self.assertEquals(10.0, time_allowed)
+
+        # Test that after waiting we can do only 1 action.
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(
+                None,
+                key="test_id",
+                update=False,
+                n_actions=1,
+                _time_now_s=10,
+            )
+        )
+        self.assertTrue(allowed)
+        # The time allowed is the current time because we could still repeat the action
+        # once.
+        self.assertEquals(10.0, time_allowed)
+
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(None, key="test_id", n_actions=2, _time_now_s=10)
+        )
+        self.assertFalse(allowed)
+        # The time allowed doesn't change despite allowed being False because, while we
+        # don't allow 2 actions, we could still do 1.
+        self.assertEquals(10.0, time_allowed)
+
+        # Test that after waiting a bit more we can do 2 actions.
+        allowed, time_allowed = self.get_success_or_raise(
+            limiter.can_do_action(None, key="test_id", n_actions=2, _time_now_s=20)
+        )
+        self.assertTrue(allowed)
+        # The time allowed is the current time because we could still repeat the action
+        # once.
+        self.assertEquals(20.0, time_allowed)
diff --git a/tests/app/test_frontend_proxy.py b/tests/app/test_frontend_proxy.py
deleted file mode 100644
index e0ca288829..0000000000
--- a/tests/app/test_frontend_proxy.py
+++ /dev/null
@@ -1,84 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.app.generic_worker import GenericWorkerServer
-
-from tests.server import make_request
-from tests.unittest import HomeserverTestCase
-
-
-class FrontendProxyTests(HomeserverTestCase):
-    def make_homeserver(self, reactor, clock):
-
-        hs = self.setup_test_homeserver(
-            federation_http_client=None, homeserver_to_use=GenericWorkerServer
-        )
-
-        return hs
-
-    def default_config(self):
-        c = super().default_config()
-        c["worker_app"] = "synapse.app.frontend_proxy"
-
-        c["worker_listeners"] = [
-            {
-                "type": "http",
-                "port": 8080,
-                "bind_addresses": ["0.0.0.0"],
-                "resources": [{"names": ["client"]}],
-            }
-        ]
-
-        return c
-
-    def test_listen_http_with_presence_enabled(self):
-        """
-        When presence is on, the stub servlet will not register.
-        """
-        # Presence is on
-        self.hs.config.use_presence = True
-
-        # Listen with the config
-        self.hs._listen_http(self.hs.config.worker.worker_listeners[0])
-
-        # Grab the resource from the site that was told to listen
-        self.assertEqual(len(self.reactor.tcpServers), 1)
-        site = self.reactor.tcpServers[0][1]
-
-        channel = make_request(self.reactor, site, "PUT", "presence/a/status")
-
-        # 400 + unrecognised, because nothing is registered
-        self.assertEqual(channel.code, 400)
-        self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED")
-
-    def test_listen_http_with_presence_disabled(self):
-        """
-        When presence is off, the stub servlet will register.
-        """
-        # Presence is off
-        self.hs.config.use_presence = False
-
-        # Listen with the config
-        self.hs._listen_http(self.hs.config.worker.worker_listeners[0])
-
-        # Grab the resource from the site that was told to listen
-        self.assertEqual(len(self.reactor.tcpServers), 1)
-        site = self.reactor.tcpServers[0][1]
-
-        channel = make_request(self.reactor, site, "PUT", "presence/a/status")
-
-        # 401, because the stub servlet still checks authentication
-        self.assertEqual(channel.code, 401)
-        self.assertEqual(channel.json_body["errcode"], "M_MISSING_TOKEN")
diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py
index 467033e201..264e101082 100644
--- a/tests/app/test_openid_listener.py
+++ b/tests/app/test_openid_listener.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mock import Mock, patch
+from unittest.mock import Mock, patch
 
 from parameterized import parameterized
 
@@ -110,7 +109,7 @@ def test_openid_listener(self, names, expectation):
         }
 
         # Listen with the config
-        self.hs._listener_http(self.hs.get_config(), parse_listener_def(config))
+        self.hs._listener_http(self.hs.config, parse_listener_def(config))
 
         # Grab the resource from the site that was told to listen
         site = self.reactor.tcpServers[0][1]
diff --git a/tests/app/test_phone_stats_home.py b/tests/app/test_phone_stats_home.py
new file mode 100644
index 0000000000..d66aeb00eb
--- /dev/null
+++ b/tests/app/test_phone_stats_home.py
@@ -0,0 +1,395 @@
+import synapse
+from synapse.app.phone_stats_home import start_phone_stats_home
+from synapse.rest.client import login, room
+
+from tests import unittest
+from tests.unittest import HomeserverTestCase
+
+FIVE_MINUTES_IN_SECONDS = 300
+ONE_DAY_IN_SECONDS = 86400
+
+
+class PhoneHomeTestCase(HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    # Override the retention time for the user_ips table because otherwise it
+    # gets pruned too aggressively for our R30 test.
+    @unittest.override_config({"user_ips_max_age": "365d"})
+    def test_r30_minimum_usage(self):
+        """
+        Tests the minimum amount of interaction necessary for the R30 metric
+        to consider a user 'retained'.
+        """
+
+        # Register a user, log it in, create a room and send a message
+        user_id = self.register_user("u1", "secret!")
+        access_token = self.login("u1", "secret!")
+        room_id = self.helper.create_room_as(room_creator=user_id, tok=access_token)
+        self.helper.send(room_id, "message", tok=access_token)
+
+        # Check the R30 results do not count that user.
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 0})
+
+        # Advance 30 days (+ 1 second, because strict inequality causes issues if we are
+        # bang on 30 days later).
+        self.reactor.advance(30 * ONE_DAY_IN_SECONDS + 1)
+
+        # (Make sure the user isn't somehow counted by this point.)
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 0})
+
+        # Send a message (this counts as activity)
+        self.helper.send(room_id, "message2", tok=access_token)
+
+        # We have to wait some time for _update_client_ips_batch to get
+        # called and update the user_ips table.
+        self.reactor.advance(2 * 60 * 60)
+
+        # *Now* the user is counted.
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 1, "unknown": 1})
+
+        # Advance 29 days. The user has now not posted for 29 days.
+        self.reactor.advance(29 * ONE_DAY_IN_SECONDS)
+
+        # The user is still counted.
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 1, "unknown": 1})
+
+        # Advance another day. The user has now not posted for 30 days.
+        self.reactor.advance(ONE_DAY_IN_SECONDS)
+
+        # The user is now no longer counted in R30.
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 0})
+
+    def test_r30_minimum_usage_using_default_config(self):
+        """
+        Tests the minimum amount of interaction necessary for the R30 metric
+        to consider a user 'retained'.
+
+        N.B. This test does not override the `user_ips_max_age` config setting,
+        which defaults to 28 days.
+        """
+
+        # Register a user, log it in, create a room and send a message
+        user_id = self.register_user("u1", "secret!")
+        access_token = self.login("u1", "secret!")
+        room_id = self.helper.create_room_as(room_creator=user_id, tok=access_token)
+        self.helper.send(room_id, "message", tok=access_token)
+
+        # Check the R30 results do not count that user.
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 0})
+
+        # Advance 30 days (+ 1 second, because strict inequality causes issues if we are
+        # bang on 30 days later).
+        self.reactor.advance(30 * ONE_DAY_IN_SECONDS + 1)
+
+        # (Make sure the user isn't somehow counted by this point.)
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 0})
+
+        # Send a message (this counts as activity)
+        self.helper.send(room_id, "message2", tok=access_token)
+
+        # We have to wait some time for _update_client_ips_batch to get
+        # called and update the user_ips table.
+        self.reactor.advance(2 * 60 * 60)
+
+        # *Now* the user is counted.
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 1, "unknown": 1})
+
+        # Advance 27 days. The user has now not posted for 27 days.
+        self.reactor.advance(27 * ONE_DAY_IN_SECONDS)
+
+        # The user is still counted.
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 1, "unknown": 1})
+
+        # Advance another day. The user has now not posted for 28 days.
+        self.reactor.advance(ONE_DAY_IN_SECONDS)
+
+        # The user is now no longer counted in R30.
+        # (This is because the user_ips table has been pruned, which by default
+        # only preserves the last 28 days of entries.)
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 0})
+
+    def test_r30_user_must_be_retained_for_at_least_a_month(self):
+        """
+        Tests that a newly-registered user must be retained for a whole month
+        before appearing in the R30 statistic, even if they post every day
+        during that time!
+        """
+        # Register a user and send a message
+        user_id = self.register_user("u1", "secret!")
+        access_token = self.login("u1", "secret!")
+        room_id = self.helper.create_room_as(room_creator=user_id, tok=access_token)
+        self.helper.send(room_id, "message", tok=access_token)
+
+        # Check the user does not contribute to R30 yet.
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 0})
+
+        for _ in range(30):
+            # This loop posts a message every day for 30 days
+            self.reactor.advance(ONE_DAY_IN_SECONDS)
+            self.helper.send(room_id, "I'm still here", tok=access_token)
+
+            # Notice that the user *still* does not contribute to R30!
+            r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+            self.assertEqual(r30_results, {"all": 0})
+
+        self.reactor.advance(ONE_DAY_IN_SECONDS)
+        self.helper.send(room_id, "Still here!", tok=access_token)
+
+        # *Now* the user appears in R30.
+        r30_results = self.get_success(self.hs.get_datastore().count_r30_users())
+        self.assertEqual(r30_results, {"all": 1, "unknown": 1})
+
+
+class PhoneHomeR30V2TestCase(HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def _advance_to(self, desired_time_secs: float):
+        now = self.hs.get_clock().time()
+        assert now < desired_time_secs
+        self.reactor.advance(desired_time_secs - now)
+
+    def make_homeserver(self, reactor, clock):
+        hs = super(PhoneHomeR30V2TestCase, self).make_homeserver(reactor, clock)
+
+        # We don't want our tests to actually report statistics, so check
+        # that it's not enabled
+        assert not hs.config.report_stats
+
+        # This starts the needed data collection that we rely on to calculate
+        # R30v2 metrics.
+        start_phone_stats_home(hs)
+        return hs
+
+    def test_r30v2_minimum_usage(self):
+        """
+        Tests the minimum amount of interaction necessary for the R30v2 metric
+        to consider a user 'retained'.
+        """
+
+        # Register a user, log it in, create a room and send a message
+        user_id = self.register_user("u1", "secret!")
+        access_token = self.login("u1", "secret!")
+        room_id = self.helper.create_room_as(room_creator=user_id, tok=access_token)
+        self.helper.send(room_id, "message", tok=access_token)
+        first_post_at = self.hs.get_clock().time()
+
+        # Give time for user_daily_visits table to be updated.
+        # (user_daily_visits is updated every 5 minutes using a looping call.)
+        self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
+
+        store = self.hs.get_datastore()
+
+        # Check the R30 results do not count that user.
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
+        )
+
+        # Advance 31 days.
+        # (R30v2 includes users with **more** than 30 days between the two visits,
+        #  and user_daily_visits records the timestamp as the start of the day.)
+        self.reactor.advance(31 * ONE_DAY_IN_SECONDS)
+        # Also advance 5 minutes to let another user_daily_visits update occur
+        self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
+
+        # (Make sure the user isn't somehow counted by this point.)
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
+        )
+
+        # Send a message (this counts as activity)
+        self.helper.send(room_id, "message2", tok=access_token)
+
+        # We have to wait a few minutes for the user_daily_visits table to
+        # be updated by a background process.
+        self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
+
+        # *Now* the user is counted.
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 1, "android": 0, "electron": 0, "ios": 0, "web": 0}
+        )
+
+        # Advance to JUST under 60 days after the user's first post
+        self._advance_to(first_post_at + 60 * ONE_DAY_IN_SECONDS - 5)
+
+        # Check the user is still counted.
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 1, "android": 0, "electron": 0, "ios": 0, "web": 0}
+        )
+
+        # Advance into the next day. The user's first activity is now more than 60 days old.
+        self._advance_to(first_post_at + 60 * ONE_DAY_IN_SECONDS + 5)
+
+        # Check the user is now no longer counted in R30.
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
+        )
+
+    def test_r30v2_user_must_be_retained_for_at_least_a_month(self):
+        """
+        Tests that a newly-registered user must be retained for a whole month
+        before appearing in the R30v2 statistic, even if they post every day
+        during that time!
+        """
+
+        # set a custom user-agent to impersonate Element/Android.
+        headers = (
+            (
+                "User-Agent",
+                "Element/1.1 (Linux; U; Android 9; MatrixAndroidSDK_X 0.0.1)",
+            ),
+        )
+
+        # Register a user and send a message
+        user_id = self.register_user("u1", "secret!")
+        access_token = self.login("u1", "secret!", custom_headers=headers)
+        room_id = self.helper.create_room_as(
+            room_creator=user_id, tok=access_token, custom_headers=headers
+        )
+        self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
+
+        # Give time for user_daily_visits table to be updated.
+        # (user_daily_visits is updated every 5 minutes using a looping call.)
+        self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
+
+        store = self.hs.get_datastore()
+
+        # Check the user does not contribute to R30 yet.
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
+        )
+
+        for _ in range(30):
+            # This loop posts a message every day for 30 days
+            self.reactor.advance(ONE_DAY_IN_SECONDS - FIVE_MINUTES_IN_SECONDS)
+            self.helper.send(
+                room_id, "I'm still here", tok=access_token, custom_headers=headers
+            )
+
+            # give time for user_daily_visits to update
+            self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
+
+            # Notice that the user *still* does not contribute to R30!
+            r30_results = self.get_success(store.count_r30v2_users())
+            self.assertEqual(
+                r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
+            )
+
+        # advance yet another day with more activity
+        self.reactor.advance(ONE_DAY_IN_SECONDS)
+        self.helper.send(
+            room_id, "Still here!", tok=access_token, custom_headers=headers
+        )
+
+        # give time for user_daily_visits to update
+        self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
+
+        # *Now* the user appears in R30.
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 1, "android": 1, "electron": 0, "ios": 0, "web": 0}
+        )
+
+    def test_r30v2_returning_dormant_users_not_counted(self):
+        """
+        Tests that dormant users (users inactive for a long time) do not
+        contribute to R30v2 when they return for just a single day.
+        This is a key difference between R30 and R30v2.
+        """
+
+        # set a custom user-agent to impersonate Element/iOS.
+        headers = (
+            (
+                "User-Agent",
+                "Riot/1.4 (iPhone; iOS 13; Scale/4.00)",
+            ),
+        )
+
+        # Register a user and send a message
+        user_id = self.register_user("u1", "secret!")
+        access_token = self.login("u1", "secret!", custom_headers=headers)
+        room_id = self.helper.create_room_as(
+            room_creator=user_id, tok=access_token, custom_headers=headers
+        )
+        self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
+
+        # the user goes inactive for 2 months
+        self.reactor.advance(60 * ONE_DAY_IN_SECONDS)
+
+        # the user returns for one day, perhaps just to check out a new feature
+        self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
+
+        # Give time for user_daily_visits table to be updated.
+        # (user_daily_visits is updated every 5 minutes using a looping call.)
+        self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
+
+        store = self.hs.get_datastore()
+
+        # Check that the user does not contribute to R30v2, even though it's been
+        # more than 30 days since registration.
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
+        )
+
+        # Check that this is a situation where old R30 differs:
+        # old R30 DOES count this as 'retained'.
+        r30_results = self.get_success(store.count_r30_users())
+        self.assertEqual(r30_results, {"all": 1, "ios": 1})
+
+        # Now we want to check that the user will still be able to appear in
+        # R30v2 as long as the user performs some other activity between
+        # 30 and 60 days later.
+        self.reactor.advance(32 * ONE_DAY_IN_SECONDS)
+        self.helper.send(room_id, "message", tok=access_token, custom_headers=headers)
+
+        # (give time for tables to update)
+        self.reactor.advance(FIVE_MINUTES_IN_SECONDS)
+
+        # Check the user now satisfies the requirements to appear in R30v2.
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 1, "ios": 1, "android": 0, "electron": 0, "web": 0}
+        )
+
+        # Advance to 59.5 days after the user's first R30v2-eligible activity.
+        self.reactor.advance(27.5 * ONE_DAY_IN_SECONDS)
+
+        # Check the user still appears in R30v2.
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 1, "ios": 1, "android": 0, "electron": 0, "web": 0}
+        )
+
+        # Advance to 60.5 days after the user's first R30v2-eligible activity.
+        self.reactor.advance(ONE_DAY_IN_SECONDS)
+
+        # Check the user no longer appears in R30v2.
+        r30_results = self.get_success(store.count_r30v2_users())
+        self.assertEqual(
+            r30_results, {"all": 0, "android": 0, "electron": 0, "ios": 0, "web": 0}
+        )
diff --git a/tests/appservice/__init__.py b/tests/appservice/__init__.py
index fe0ac3f8e9..629e2df74a 100644
--- a/tests/appservice/__init__.py
+++ b/tests/appservice/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py
index 0bffeb1150..f386b5e128 100644
--- a/tests/appservice/test_appservice.py
+++ b/tests/appservice/test_appservice.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import re
-
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py
index 97f8cad0dd..a2b5ed2030 100644
--- a/tests/appservice/test_scheduler.py
+++ b/tests/appservice/test_scheduler.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
diff --git a/tests/config/__init__.py b/tests/config/__init__.py
index b7df13c9ee..f43a360a80 100644
--- a/tests/config/__init__.py
+++ b/tests/config/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/config/test_base.py b/tests/config/test_base.py
index 42ee5f56d9..baa5313fb3 100644
--- a/tests/config/test_base.py
+++ b/tests/config/test_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -31,7 +30,7 @@ def test_loading_missing_templates(self):
         # contain template files
         with tempfile.TemporaryDirectory() as tmp_dir:
             # Attempt to load an HTML template from our custom template directory
-            template = self.hs.config.read_templates(["sso_error.html"], tmp_dir)[0]
+            template = self.hs.config.read_templates(["sso_error.html"], (tmp_dir,))[0]
 
         # If no errors, we should've gotten the default template instead
 
@@ -61,7 +60,7 @@ def test_loading_custom_templates(self):
 
                 # Attempt to load the template from our custom template directory
                 template = (
-                    self.hs.config.read_templates([template_filename], tmp_dir)
+                    self.hs.config.read_templates([template_filename], (tmp_dir,))
                 )[0]
 
         # Render the template
@@ -75,8 +74,66 @@ def test_loading_custom_templates(self):
             "Template file did not contain our test string",
         )
 
+    def test_multiple_custom_template_directories(self):
+        """Tests that directories are searched in the right order if multiple custom
+        template directories are provided.
+        """
+        # Create two temporary directories on the filesystem.
+        tempdirs = [
+            tempfile.TemporaryDirectory(),
+            tempfile.TemporaryDirectory(),
+        ]
+
+        # Create one template in each directory, whose content is the index of the
+        # directory in the list.
+        template_filename = "my_template.html.j2"
+        for i in range(len(tempdirs)):
+            tempdir = tempdirs[i]
+            template_path = os.path.join(tempdir.name, template_filename)
+
+            with open(template_path, "w") as fp:
+                fp.write(str(i))
+                fp.flush()
+
+        # Retrieve the template.
+        template = (
+            self.hs.config.read_templates(
+                [template_filename],
+                (td.name for td in tempdirs),
+            )
+        )[0]
+
+        # Test that we got the template we dropped in the first directory in the list.
+        self.assertEqual(template.render(), "0")
+
+        # Add another template, this one only in the second directory in the list, so we
+        # can test that the second directory is still searched into when no matching file
+        # could be found in the first one.
+        other_template_name = "my_other_template.html.j2"
+        other_template_path = os.path.join(tempdirs[1].name, other_template_name)
+
+        with open(other_template_path, "w") as fp:
+            fp.write("hello world")
+            fp.flush()
+
+        # Retrieve the template.
+        template = (
+            self.hs.config.read_templates(
+                [other_template_name],
+                (td.name for td in tempdirs),
+            )
+        )[0]
+
+        # Test that the file has the expected content.
+        self.assertEqual(template.render(), "hello world")
+
+        # Cleanup the temporary directories manually since we're not using a context
+        # manager.
+        for td in tempdirs:
+            td.cleanup()
+
     def test_loading_template_from_nonexistent_custom_directory(self):
         with self.assertRaises(ConfigError):
             self.hs.config.read_templates(
-                ["some_filename.html"], "a_nonexistent_directory"
+                ["some_filename.html"], ("a_nonexistent_directory",)
             )
diff --git a/tests/config/test_cache.py b/tests/config/test_cache.py
index 2b7f09c14b..857d9cd096 100644
--- a/tests/config/test_cache.py
+++ b/tests/config/test_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/config/test_database.py b/tests/config/test_database.py
index f675bde68e..9eca10bbe9 100644
--- a/tests/config/test_database.py
+++ b/tests/config/test_database.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/config/test_generate.py b/tests/config/test_generate.py
index 463855ecc8..fdfbb0e38e 100644
--- a/tests/config/test_generate.py
+++ b/tests/config/test_generate.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/config/test_load.py b/tests/config/test_load.py
index 734a9983e8..903c69127d 100644
--- a/tests/config/test_load.py
+++ b/tests/config/test_load.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,6 +19,7 @@
 
 import yaml
 
+from synapse.config import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 
 from tests import unittest
@@ -35,15 +35,15 @@ def tearDown(self):
 
     def test_load_fails_if_server_name_missing(self):
         self.generate_config_and_remove_lines_containing("server_name")
-        with self.assertRaises(Exception):
+        with self.assertRaises(ConfigError):
             HomeServerConfig.load_config("", ["-c", self.file])
-        with self.assertRaises(Exception):
+        with self.assertRaises(ConfigError):
             HomeServerConfig.load_or_generate_config("", ["-c", self.file])
 
     def test_generates_and_loads_macaroon_secret_key(self):
         self.generate_config()
 
-        with open(self.file, "r") as f:
+        with open(self.file) as f:
             raw = yaml.safe_load(f)
         self.assertIn("macaroon_secret_key", raw)
 
@@ -120,7 +120,7 @@ def generate_config(self):
     def generate_config_and_remove_lines_containing(self, needle):
         self.generate_config()
 
-        with open(self.file, "r") as f:
+        with open(self.file) as f:
             contents = f.readlines()
         contents = [line for line in contents if needle not in line]
         with open(self.file, "w") as f:
diff --git a/tests/config/test_ratelimiting.py b/tests/config/test_ratelimiting.py
index 13ab282384..3c7bb32e07 100644
--- a/tests/config/test_ratelimiting.py
+++ b/tests/config/test_ratelimiting.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/config/test_room_directory.py b/tests/config/test_room_directory.py
index 0ec10019b3..db745815ef 100644
--- a/tests/config/test_room_directory.py
+++ b/tests/config/test_room_directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/config/test_server.py b/tests/config/test_server.py
index 98af7aa675..6f2b9e997d 100644
--- a/tests/config/test_server.py
+++ b/tests/config/test_server.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py
index ec32d4b1ca..b6bc1876b5 100644
--- a/tests/config/test_tls.py
+++ b/tests/config/test_tls.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
@@ -14,10 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-
 import idna
-import yaml
 
 from OpenSSL import SSL
 
@@ -40,59 +36,6 @@ class TestConfig(RootConfig):
 
 
 class TLSConfigTests(TestCase):
-    def test_warn_self_signed(self):
-        """
-        Synapse will give a warning when it loads a self-signed certificate.
-        """
-        config_dir = self.mktemp()
-        os.mkdir(config_dir)
-        with open(os.path.join(config_dir, "cert.pem"), "w") as f:
-            f.write(
-                """-----BEGIN CERTIFICATE-----
-MIID6DCCAtACAws9CjANBgkqhkiG9w0BAQUFADCBtzELMAkGA1UEBhMCVFIxDzAN
-BgNVBAgMBsOHb3J1bTEUMBIGA1UEBwwLQmHFn21ha8OnxLExEjAQBgNVBAMMCWxv
-Y2FsaG9zdDEcMBoGA1UECgwTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECwwb
-QXV0b21hdGVkIFRlc3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1
-cml0eUB0d2lzdGVkbWF0cml4LmNvbTAgFw0xNzA3MTIxNDAxNTNaGA8yMTE3MDYx
-ODE0MDE1M1owgbcxCzAJBgNVBAYTAlRSMQ8wDQYDVQQIDAbDh29ydW0xFDASBgNV
-BAcMC0JhxZ9tYWvDp8SxMRIwEAYDVQQDDAlsb2NhbGhvc3QxHDAaBgNVBAoME1R3
-aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNVBAsMG0F1dG9tYXRlZCBUZXN0aW5nIEF1
-dGhvcml0eTEpMCcGCSqGSIb3DQEJARYac2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5j
-b20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDwT6kbqtMUI0sMkx4h
-I+L780dA59KfksZCqJGmOsMD6hte9EguasfkZzvCF3dk3NhwCjFSOvKx6rCwiteo
-WtYkVfo+rSuVNmt7bEsOUDtuTcaxTzIFB+yHOYwAaoz3zQkyVW0c4pzioiLCGCmf
-FLdiDBQGGp74tb+7a0V6kC3vMLFoM3L6QWq5uYRB5+xLzlPJ734ltyvfZHL3Us6p
-cUbK+3WTWvb4ER0W2RqArAj6Bc/ERQKIAPFEiZi9bIYTwvBH27OKHRz+KoY/G8zY
-+l+WZoJqDhupRAQAuh7O7V/y6bSP+KNxJRie9QkZvw1PSaGSXtGJI3WWdO12/Ulg
-epJpAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAJXEq5P9xwvP9aDkXIqzcD0L8sf8
-ewlhlxTQdeqt2Nace0Yk18lIo2oj1t86Y8jNbpAnZJeI813Rr5M7FbHCXoRc/SZG
-I8OtG1xGwcok53lyDuuUUDexnK4O5BkjKiVlNPg4HPim5Kuj2hRNFfNt/F2BVIlj
-iZupikC5MT1LQaRwidkSNxCku1TfAyueiBwhLnFwTmIGNnhuDCutEVAD9kFmcJN2
-SznugAcPk4doX2+rL+ila+ThqgPzIkwTUHtnmjI0TI6xsDUlXz5S3UyudrE2Qsfz
-s4niecZKPBizL6aucT59CsunNmmb5Glq8rlAcU+1ZTZZzGYqVYhF6axB9Qg=
------END CERTIFICATE-----"""
-            )
-
-        config = {
-            "tls_certificate_path": os.path.join(config_dir, "cert.pem"),
-            "tls_fingerprints": [],
-        }
-
-        t = TestConfig()
-        t.read_config(config, config_dir_path="", data_dir_path="")
-        t.read_certificate_from_disk(require_cert_and_key=False)
-
-        warnings = self.flushWarnings()
-        self.assertEqual(len(warnings), 1)
-        self.assertEqual(
-            warnings[0]["message"],
-            (
-                "Self-signed TLS certificates will not be accepted by "
-                "Synapse 1.0. Please either provide a valid certificate, "
-                "or use Synapse's ACME support to provision one."
-            ),
-        )
-
     def test_tls_client_minimum_default(self):
         """
         The default client TLS version is 1.0.
@@ -204,48 +147,6 @@ def test_tls_client_minimum_set_passed_through_1_0(self):
         self.assertEqual(options & SSL.OP_NO_TLSv1_1, 0)
         self.assertEqual(options & SSL.OP_NO_TLSv1_2, 0)
 
-    def test_acme_disabled_in_generated_config_no_acme_domain_provied(self):
-        """
-        Checks acme is disabled by default.
-        """
-        conf = TestConfig()
-        conf.read_config(
-            yaml.safe_load(
-                TestConfig().generate_config(
-                    "/config_dir_path",
-                    "my_super_secure_server",
-                    "/data_dir_path",
-                    tls_certificate_path="/tls_cert_path",
-                    tls_private_key_path="tls_private_key",
-                    acme_domain=None,  # This is the acme_domain
-                )
-            ),
-            "/config_dir_path",
-        )
-
-        self.assertFalse(conf.acme_enabled)
-
-    def test_acme_enabled_in_generated_config_domain_provided(self):
-        """
-        Checks acme is enabled if the acme_domain arg is set to some string.
-        """
-        conf = TestConfig()
-        conf.read_config(
-            yaml.safe_load(
-                TestConfig().generate_config(
-                    "/config_dir_path",
-                    "my_super_secure_server",
-                    "/data_dir_path",
-                    tls_certificate_path="/tls_cert_path",
-                    tls_private_key_path="tls_private_key",
-                    acme_domain="my_supe_secure_server",  # This is the acme_domain
-                )
-            ),
-            "/config_dir_path",
-        )
-
-        self.assertTrue(conf.acme_enabled)
-
     def test_whitelist_idna_failure(self):
         """
         The federation certificate whitelist will not allow IDNA domain names.
diff --git a/tests/config/test_util.py b/tests/config/test_util.py
index 10363e3765..3d4929daac 100644
--- a/tests/config/test_util.py
+++ b/tests/config/test_util.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/crypto/__init__.py b/tests/crypto/__init__.py
index bfebb0f644..5e83dba2ed 100644
--- a/tests/crypto/__init__.py
+++ b/tests/crypto/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py
index 62f639a18d..1c920157f5 100644
--- a/tests/crypto/test_event_signing.py
+++ b/tests/crypto/test_event_signing.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py
index 30fcc4c1bf..745c295d3b 100644
--- a/tests/crypto/test_keyring.py
+++ b/tests/crypto/test_keyring.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import time
+from typing import Dict, List
+from unittest.mock import Mock
 
-from mock import Mock
-
+import attr
 import canonicaljson
 import signedjson.key
 import signedjson.sign
 from nacl.signing import SigningKey
 from signedjson.key import encode_verify_key_base64, get_verify_key
 
-from twisted.internet import defer
 from twisted.internet.defer import Deferred, ensureDeferred
 
 from synapse.api.errors import SynapseError
@@ -68,6 +67,11 @@ def sign_response(self, res):
         signedjson.sign.sign_json(res, self.server_name, self.key)
 
 
+@attr.s(slots=True)
+class FakeRequest:
+    id = attr.ib()
+
+
 @logcontext_clean
 class KeyringTestCase(unittest.HomeserverTestCase):
     def check_context(self, val, expected):
@@ -88,25 +92,23 @@ def test_verify_json_objects_for_server_awaits_previous_requests(self):
         # deferred completes.
         first_lookup_deferred = Deferred()
 
-        async def first_lookup_fetch(keys_to_fetch):
-            self.assertEquals(current_context().request, "context_11")
-            self.assertEqual(keys_to_fetch, {"server10": {get_key_id(key1): 0}})
+        async def first_lookup_fetch(
+            server_name: str, key_ids: List[str], minimum_valid_until_ts: int
+        ) -> Dict[str, FetchKeyResult]:
+            # self.assertEquals(current_context().request.id, "context_11")
+            self.assertEqual(server_name, "server10")
+            self.assertEqual(key_ids, [get_key_id(key1)])
+            self.assertEqual(minimum_valid_until_ts, 0)
 
             await make_deferred_yieldable(first_lookup_deferred)
-            return {
-                "server10": {
-                    get_key_id(key1): FetchKeyResult(get_verify_key(key1), 100)
-                }
-            }
+            return {get_key_id(key1): FetchKeyResult(get_verify_key(key1), 100)}
 
         mock_fetcher.get_keys.side_effect = first_lookup_fetch
 
         async def first_lookup():
-            with LoggingContext("context_11") as context_11:
-                context_11.request = "context_11"
-
+            with LoggingContext("context_11", request=FakeRequest("context_11")):
                 res_deferreds = kr.verify_json_objects_for_server(
-                    [("server10", json1, 0, "test10"), ("server11", {}, 0, "test11")]
+                    [("server10", json1, 0), ("server11", {}, 0)]
                 )
 
                 # the unsigned json should be rejected pretty quickly
@@ -124,29 +126,33 @@ async def first_lookup():
 
         d0 = ensureDeferred(first_lookup())
 
+        self.pump()
+
         mock_fetcher.get_keys.assert_called_once()
 
         # a second request for a server with outstanding requests
         # should block rather than start a second call
 
-        async def second_lookup_fetch(keys_to_fetch):
-            self.assertEquals(current_context().request, "context_12")
-            return {
-                "server10": {
-                    get_key_id(key1): FetchKeyResult(get_verify_key(key1), 100)
-                }
-            }
+        async def second_lookup_fetch(
+            server_name: str, key_ids: List[str], minimum_valid_until_ts: int
+        ) -> Dict[str, FetchKeyResult]:
+            # self.assertEquals(current_context().request.id, "context_12")
+            return {get_key_id(key1): FetchKeyResult(get_verify_key(key1), 100)}
 
         mock_fetcher.get_keys.reset_mock()
         mock_fetcher.get_keys.side_effect = second_lookup_fetch
         second_lookup_state = [0]
 
         async def second_lookup():
-            with LoggingContext("context_12") as context_12:
-                context_12.request = "context_12"
-
+            with LoggingContext("context_12", request=FakeRequest("context_12")):
                 res_deferreds_2 = kr.verify_json_objects_for_server(
-                    [("server10", json1, 0, "test")]
+                    [
+                        (
+                            "server10",
+                            json1,
+                            0,
+                        )
+                    ]
                 )
                 res_deferreds_2[0].addBoth(self.check_context, None)
                 second_lookup_state[0] = 1
@@ -183,11 +189,11 @@ def test_verify_json_for_server(self):
         signedjson.sign.sign_json(json1, "server9", key1)
 
         # should fail immediately on an unsigned object
-        d = _verify_json_for_server(kr, "server9", {}, 0, "test unsigned")
+        d = kr.verify_json_for_server("server9", {}, 0)
         self.get_failure(d, SynapseError)
 
         # should succeed on a signed object
-        d = _verify_json_for_server(kr, "server9", json1, 500, "test signed")
+        d = kr.verify_json_for_server("server9", json1, 500)
         # self.assertFalse(d.called)
         self.get_success(d)
 
@@ -214,24 +220,24 @@ def test_verify_json_for_server_with_null_valid_until_ms(self):
         signedjson.sign.sign_json(json1, "server9", key1)
 
         # should fail immediately on an unsigned object
-        d = _verify_json_for_server(kr, "server9", {}, 0, "test unsigned")
+        d = kr.verify_json_for_server("server9", {}, 0)
         self.get_failure(d, SynapseError)
 
         # should fail on a signed object with a non-zero minimum_valid_until_ms,
         # as it tries to refetch the keys and fails.
-        d = _verify_json_for_server(
-            kr, "server9", json1, 500, "test signed non-zero min"
-        )
+        d = kr.verify_json_for_server("server9", json1, 500)
         self.get_failure(d, SynapseError)
 
         # We expect the keyring tried to refetch the key once.
         mock_fetcher.get_keys.assert_called_once_with(
-            {"server9": {get_key_id(key1): 500}}
+            "server9", [get_key_id(key1)], 500
         )
 
         # should succeed on a signed object with a 0 minimum_valid_until_ms
-        d = _verify_json_for_server(
-            kr, "server9", json1, 0, "test signed with zero min"
+        d = kr.verify_json_for_server(
+            "server9",
+            json1,
+            0,
         )
         self.get_success(d)
 
@@ -239,15 +245,15 @@ def test_verify_json_dedupes_key_requests(self):
         """Two requests for the same key should be deduped."""
         key1 = signedjson.key.generate_signing_key(1)
 
-        async def get_keys(keys_to_fetch):
+        async def get_keys(
+            server_name: str, key_ids: List[str], minimum_valid_until_ts: int
+        ) -> Dict[str, FetchKeyResult]:
             # there should only be one request object (with the max validity)
-            self.assertEqual(keys_to_fetch, {"server1": {get_key_id(key1): 1500}})
+            self.assertEqual(server_name, "server1")
+            self.assertEqual(key_ids, [get_key_id(key1)])
+            self.assertEqual(minimum_valid_until_ts, 1500)
 
-            return {
-                "server1": {
-                    get_key_id(key1): FetchKeyResult(get_verify_key(key1), 1200)
-                }
-            }
+            return {get_key_id(key1): FetchKeyResult(get_verify_key(key1), 1200)}
 
         mock_fetcher = Mock()
         mock_fetcher.get_keys = Mock(side_effect=get_keys)
@@ -259,7 +265,14 @@ async def get_keys(keys_to_fetch):
         # the first request should succeed; the second should fail because the key
         # has expired
         results = kr.verify_json_objects_for_server(
-            [("server1", json1, 500, "test1"), ("server1", json1, 1500, "test2")]
+            [
+                (
+                    "server1",
+                    json1,
+                    500,
+                ),
+                ("server1", json1, 1500),
+            ]
         )
         self.assertEqual(len(results), 2)
         self.get_success(results[0])
@@ -274,19 +287,21 @@ def test_verify_json_falls_back_to_other_fetchers(self):
         """If the first fetcher cannot provide a recent enough key, we fall back"""
         key1 = signedjson.key.generate_signing_key(1)
 
-        async def get_keys1(keys_to_fetch):
-            self.assertEqual(keys_to_fetch, {"server1": {get_key_id(key1): 1500}})
-            return {
-                "server1": {get_key_id(key1): FetchKeyResult(get_verify_key(key1), 800)}
-            }
-
-        async def get_keys2(keys_to_fetch):
-            self.assertEqual(keys_to_fetch, {"server1": {get_key_id(key1): 1500}})
-            return {
-                "server1": {
-                    get_key_id(key1): FetchKeyResult(get_verify_key(key1), 1200)
-                }
-            }
+        async def get_keys1(
+            server_name: str, key_ids: List[str], minimum_valid_until_ts: int
+        ) -> Dict[str, FetchKeyResult]:
+            self.assertEqual(server_name, "server1")
+            self.assertEqual(key_ids, [get_key_id(key1)])
+            self.assertEqual(minimum_valid_until_ts, 1500)
+            return {get_key_id(key1): FetchKeyResult(get_verify_key(key1), 800)}
+
+        async def get_keys2(
+            server_name: str, key_ids: List[str], minimum_valid_until_ts: int
+        ) -> Dict[str, FetchKeyResult]:
+            self.assertEqual(server_name, "server1")
+            self.assertEqual(key_ids, [get_key_id(key1)])
+            self.assertEqual(minimum_valid_until_ts, 1500)
+            return {get_key_id(key1): FetchKeyResult(get_verify_key(key1), 1200)}
 
         mock_fetcher1 = Mock()
         mock_fetcher1.get_keys = Mock(side_effect=get_keys1)
@@ -298,7 +313,18 @@ async def get_keys2(keys_to_fetch):
         signedjson.sign.sign_json(json1, "server1", key1)
 
         results = kr.verify_json_objects_for_server(
-            [("server1", json1, 1200, "test1"), ("server1", json1, 1500, "test2")]
+            [
+                (
+                    "server1",
+                    json1,
+                    1200,
+                ),
+                (
+                    "server1",
+                    json1,
+                    1500,
+                ),
+            ]
         )
         self.assertEqual(len(results), 2)
         self.get_success(results[0])
@@ -349,9 +375,8 @@ async def get_json(destination, path, **kwargs):
 
         self.http_client.get_json.side_effect = get_json
 
-        keys_to_fetch = {SERVER_NAME: {"key1": 0}}
-        keys = self.get_success(fetcher.get_keys(keys_to_fetch))
-        k = keys[SERVER_NAME][testverifykey_id]
+        keys = self.get_success(fetcher.get_keys(SERVER_NAME, ["key1"], 0))
+        k = keys[testverifykey_id]
         self.assertEqual(k.valid_until_ts, VALID_UNTIL_TS)
         self.assertEqual(k.verify_key, testverifykey)
         self.assertEqual(k.verify_key.alg, "ed25519")
@@ -378,7 +403,7 @@ async def get_json(destination, path, **kwargs):
         # change the server name: the result should be ignored
         response["server_name"] = "OTHER_SERVER"
 
-        keys = self.get_success(fetcher.get_keys(keys_to_fetch))
+        keys = self.get_success(fetcher.get_keys(SERVER_NAME, ["key1"], 0))
         self.assertEqual(keys, {})
 
 
@@ -465,10 +490,9 @@ def test_get_keys_from_perspectives(self):
 
         self.expect_outgoing_key_query(SERVER_NAME, "key1", response)
 
-        keys_to_fetch = {SERVER_NAME: {"key1": 0}}
-        keys = self.get_success(fetcher.get_keys(keys_to_fetch))
-        self.assertIn(SERVER_NAME, keys)
-        k = keys[SERVER_NAME][testverifykey_id]
+        keys = self.get_success(fetcher.get_keys(SERVER_NAME, ["key1"], 0))
+        self.assertIn(testverifykey_id, keys)
+        k = keys[testverifykey_id]
         self.assertEqual(k.valid_until_ts, VALID_UNTIL_TS)
         self.assertEqual(k.verify_key, testverifykey)
         self.assertEqual(k.verify_key.alg, "ed25519")
@@ -515,10 +539,9 @@ def test_get_perspectives_own_key(self):
 
         self.expect_outgoing_key_query(SERVER_NAME, "key1", response)
 
-        keys_to_fetch = {SERVER_NAME: {"key1": 0}}
-        keys = self.get_success(fetcher.get_keys(keys_to_fetch))
-        self.assertIn(SERVER_NAME, keys)
-        k = keys[SERVER_NAME][testverifykey_id]
+        keys = self.get_success(fetcher.get_keys(SERVER_NAME, ["key1"], 0))
+        self.assertIn(testverifykey_id, keys)
+        k = keys[testverifykey_id]
         self.assertEqual(k.valid_until_ts, VALID_UNTIL_TS)
         self.assertEqual(k.verify_key, testverifykey)
         self.assertEqual(k.verify_key.alg, "ed25519")
@@ -559,14 +582,13 @@ def build_response():
 
         def get_key_from_perspectives(response):
             fetcher = PerspectivesKeyFetcher(self.hs)
-            keys_to_fetch = {SERVER_NAME: {"key1": 0}}
             self.expect_outgoing_key_query(SERVER_NAME, "key1", response)
-            return self.get_success(fetcher.get_keys(keys_to_fetch))
+            return self.get_success(fetcher.get_keys(SERVER_NAME, ["key1"], 0))
 
         # start with a valid response so we can check we are testing the right thing
         response = build_response()
         keys = get_key_from_perspectives(response)
-        k = keys[SERVER_NAME][testverifykey_id]
+        k = keys[testverifykey_id]
         self.assertEqual(k.verify_key, testverifykey)
 
         # remove the perspectives server's signature
@@ -585,26 +607,3 @@ def get_key_from_perspectives(response):
 def get_key_id(key):
     """Get the matrix ID tag for a given SigningKey or VerifyKey"""
     return "%s:%s" % (key.alg, key.version)
-
-
-@defer.inlineCallbacks
-def run_in_context(f, *args, **kwargs):
-    with LoggingContext("testctx") as ctx:
-        # we set the "request" prop to make it easier to follow what's going on in the
-        # logs.
-        ctx.request = "testctx"
-        rv = yield f(*args, **kwargs)
-    return rv
-
-
-def _verify_json_for_server(kr, *args):
-    """thin wrapper around verify_json_for_server which makes sure it is wrapped
-    with the patched defer.inlineCallbacks.
-    """
-
-    @defer.inlineCallbacks
-    def v():
-        rv1 = yield kr.verify_json_for_server(*args)
-        return rv1
-
-    return run_in_context(v)
diff --git a/tests/events/test_presence_router.py b/tests/events/test_presence_router.py
new file mode 100644
index 0000000000..6b87f571b8
--- /dev/null
+++ b/tests/events/test_presence_router.py
@@ -0,0 +1,400 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
+from unittest.mock import Mock
+
+import attr
+
+from synapse.api.constants import EduTypes
+from synapse.events.presence_router import PresenceRouter
+from synapse.federation.units import Transaction
+from synapse.handlers.presence import UserPresenceState
+from synapse.module_api import ModuleApi
+from synapse.rest import admin
+from synapse.rest.client import login, presence, room
+from synapse.types import JsonDict, StreamToken, create_requester
+
+from tests.handlers.test_sync import generate_sync_config
+from tests.unittest import FederatingHomeserverTestCase, TestCase, override_config
+
+
+@attr.s
+class PresenceRouterTestConfig:
+    users_who_should_receive_all_presence = attr.ib(type=List[str], default=[])
+
+
+class PresenceRouterTestModule:
+    def __init__(self, config: PresenceRouterTestConfig, module_api: ModuleApi):
+        self._config = config
+        self._module_api = module_api
+
+    async def get_users_for_states(
+        self, state_updates: Iterable[UserPresenceState]
+    ) -> Dict[str, Set[UserPresenceState]]:
+        users_to_state = {
+            user_id: set(state_updates)
+            for user_id in self._config.users_who_should_receive_all_presence
+        }
+        return users_to_state
+
+    async def get_interested_users(
+        self, user_id: str
+    ) -> Union[Set[str], PresenceRouter.ALL_USERS]:
+        if user_id in self._config.users_who_should_receive_all_presence:
+            return PresenceRouter.ALL_USERS
+
+        return set()
+
+    @staticmethod
+    def parse_config(config_dict: dict) -> PresenceRouterTestConfig:
+        """Parse a configuration dictionary from the homeserver config, do
+        some validation and return a typed PresenceRouterConfig.
+
+        Args:
+            config_dict: The configuration dictionary.
+
+        Returns:
+            A validated config object.
+        """
+        # Initialise a typed config object
+        config = PresenceRouterTestConfig()
+
+        config.users_who_should_receive_all_presence = config_dict.get(
+            "users_who_should_receive_all_presence"
+        )
+
+        return config
+
+
+class PresenceRouterTestCase(FederatingHomeserverTestCase):
+    servlets = [
+        admin.register_servlets,
+        login.register_servlets,
+        room.register_servlets,
+        presence.register_servlets,
+    ]
+
+    def make_homeserver(self, reactor, clock):
+        return self.setup_test_homeserver(
+            federation_transport_client=Mock(spec=["send_transaction"]),
+        )
+
+    def prepare(self, reactor, clock, homeserver):
+        self.sync_handler = self.hs.get_sync_handler()
+        self.module_api = homeserver.get_module_api()
+
+    @override_config(
+        {
+            "presence": {
+                "presence_router": {
+                    "module": __name__ + ".PresenceRouterTestModule",
+                    "config": {
+                        "users_who_should_receive_all_presence": [
+                            "@presence_gobbler:test",
+                        ]
+                    },
+                }
+            },
+            "send_federation": True,
+        }
+    )
+    def test_receiving_all_presence(self):
+        """Test that a user that does not share a room with another other can receive
+        presence for them, due to presence routing.
+        """
+        # Create a user who should receive all presence of others
+        self.presence_receiving_user_id = self.register_user(
+            "presence_gobbler", "monkey"
+        )
+        self.presence_receiving_user_tok = self.login("presence_gobbler", "monkey")
+
+        # And two users who should not have any special routing
+        self.other_user_one_id = self.register_user("other_user_one", "monkey")
+        self.other_user_one_tok = self.login("other_user_one", "monkey")
+        self.other_user_two_id = self.register_user("other_user_two", "monkey")
+        self.other_user_two_tok = self.login("other_user_two", "monkey")
+
+        # Put the other two users in a room with each other
+        room_id = self.helper.create_room_as(
+            self.other_user_one_id, tok=self.other_user_one_tok
+        )
+
+        self.helper.invite(
+            room_id,
+            self.other_user_one_id,
+            self.other_user_two_id,
+            tok=self.other_user_one_tok,
+        )
+        self.helper.join(room_id, self.other_user_two_id, tok=self.other_user_two_tok)
+        # User one sends some presence
+        send_presence_update(
+            self,
+            self.other_user_one_id,
+            self.other_user_one_tok,
+            "online",
+            "boop",
+        )
+
+        # Check that the presence receiving user gets user one's presence when syncing
+        presence_updates, sync_token = sync_presence(
+            self, self.presence_receiving_user_id
+        )
+        self.assertEqual(len(presence_updates), 1)
+
+        presence_update: UserPresenceState = presence_updates[0]
+        self.assertEqual(presence_update.user_id, self.other_user_one_id)
+        self.assertEqual(presence_update.state, "online")
+        self.assertEqual(presence_update.status_msg, "boop")
+
+        # Have all three users send presence
+        send_presence_update(
+            self,
+            self.other_user_one_id,
+            self.other_user_one_tok,
+            "online",
+            "user_one",
+        )
+        send_presence_update(
+            self,
+            self.other_user_two_id,
+            self.other_user_two_tok,
+            "online",
+            "user_two",
+        )
+        send_presence_update(
+            self,
+            self.presence_receiving_user_id,
+            self.presence_receiving_user_tok,
+            "online",
+            "presence_gobbler",
+        )
+
+        # Check that the presence receiving user gets everyone's presence
+        presence_updates, _ = sync_presence(
+            self, self.presence_receiving_user_id, sync_token
+        )
+        self.assertEqual(len(presence_updates), 3)
+
+        # But that User One only get itself and User Two's presence
+        presence_updates, _ = sync_presence(self, self.other_user_one_id)
+        self.assertEqual(len(presence_updates), 2)
+
+        found = False
+        for update in presence_updates:
+            if update.user_id == self.other_user_two_id:
+                self.assertEqual(update.state, "online")
+                self.assertEqual(update.status_msg, "user_two")
+                found = True
+
+        self.assertTrue(found)
+
+    @override_config(
+        {
+            "presence": {
+                "presence_router": {
+                    "module": __name__ + ".PresenceRouterTestModule",
+                    "config": {
+                        "users_who_should_receive_all_presence": [
+                            "@presence_gobbler1:test",
+                            "@presence_gobbler2:test",
+                            "@far_away_person:island",
+                        ]
+                    },
+                }
+            },
+            "send_federation": True,
+        }
+    )
+    def test_send_local_online_presence_to_with_module(self):
+        """Tests that send_local_presence_to_users sends local online presence to a set
+        of specified local and remote users, with a custom PresenceRouter module enabled.
+        """
+        # Create a user who will send presence updates
+        self.other_user_id = self.register_user("other_user", "monkey")
+        self.other_user_tok = self.login("other_user", "monkey")
+
+        # And another two users that will also send out presence updates, as well as receive
+        # theirs and everyone else's
+        self.presence_receiving_user_one_id = self.register_user(
+            "presence_gobbler1", "monkey"
+        )
+        self.presence_receiving_user_one_tok = self.login("presence_gobbler1", "monkey")
+        self.presence_receiving_user_two_id = self.register_user(
+            "presence_gobbler2", "monkey"
+        )
+        self.presence_receiving_user_two_tok = self.login("presence_gobbler2", "monkey")
+
+        # Have all three users send some presence updates
+        send_presence_update(
+            self,
+            self.other_user_id,
+            self.other_user_tok,
+            "online",
+            "I'm online!",
+        )
+        send_presence_update(
+            self,
+            self.presence_receiving_user_one_id,
+            self.presence_receiving_user_one_tok,
+            "online",
+            "I'm also online!",
+        )
+        send_presence_update(
+            self,
+            self.presence_receiving_user_two_id,
+            self.presence_receiving_user_two_tok,
+            "unavailable",
+            "I'm in a meeting!",
+        )
+
+        # Mark each presence-receiving user for receiving all user presence
+        self.get_success(
+            self.module_api.send_local_online_presence_to(
+                [
+                    self.presence_receiving_user_one_id,
+                    self.presence_receiving_user_two_id,
+                ]
+            )
+        )
+
+        # Perform a sync for each user
+
+        # The other user should only receive their own presence
+        presence_updates, _ = sync_presence(self, self.other_user_id)
+        self.assertEqual(len(presence_updates), 1)
+
+        presence_update: UserPresenceState = presence_updates[0]
+        self.assertEqual(presence_update.user_id, self.other_user_id)
+        self.assertEqual(presence_update.state, "online")
+        self.assertEqual(presence_update.status_msg, "I'm online!")
+
+        # Whereas both presence receiving users should receive everyone's presence updates
+        presence_updates, _ = sync_presence(self, self.presence_receiving_user_one_id)
+        self.assertEqual(len(presence_updates), 3)
+        presence_updates, _ = sync_presence(self, self.presence_receiving_user_two_id)
+        self.assertEqual(len(presence_updates), 3)
+
+        # We stagger sending of presence, so we need to wait a bit for them to
+        # get sent out.
+        self.reactor.advance(60)
+
+        # Test that sending to a remote user works
+        remote_user_id = "@far_away_person:island"
+
+        # Note that due to the remote user being in our module's
+        # users_who_should_receive_all_presence config, they would have
+        # received user presence updates already.
+        #
+        # Thus we reset the mock, and try sending all online local user
+        # presence again
+        self.hs.get_federation_transport_client().send_transaction.reset_mock()
+
+        # Broadcast local user online presence
+        self.get_success(
+            self.module_api.send_local_online_presence_to([remote_user_id])
+        )
+
+        # We stagger sending of presence, so we need to wait a bit for them to
+        # get sent out.
+        self.reactor.advance(60)
+
+        # Check that the expected presence updates were sent
+        # We explicitly compare using sets as we expect that calling
+        # module_api.send_local_online_presence_to will create a presence
+        # update that is a duplicate of the specified user's current presence.
+        # These are sent to clients and will be picked up below, thus we use a
+        # set to deduplicate. We're just interested that non-offline updates were
+        # sent out for each user ID.
+        expected_users = {
+            self.other_user_id,
+            self.presence_receiving_user_one_id,
+            self.presence_receiving_user_two_id,
+        }
+        found_users = set()
+
+        calls = (
+            self.hs.get_federation_transport_client().send_transaction.call_args_list
+        )
+        for call in calls:
+            call_args = call[0]
+            federation_transaction: Transaction = call_args[0]
+
+            # Get the sent EDUs in this transaction
+            edus = federation_transaction.get_dict()["edus"]
+
+            for edu in edus:
+                # Make sure we're only checking presence-type EDUs
+                if edu["edu_type"] != EduTypes.Presence:
+                    continue
+
+                # EDUs can contain multiple presence updates
+                for presence_update in edu["content"]["push"]:
+                    # Check for presence updates that contain the user IDs we're after
+                    found_users.add(presence_update["user_id"])
+
+                    # Ensure that no offline states are being sent out
+                    self.assertNotEqual(presence_update["presence"], "offline")
+
+        self.assertEqual(found_users, expected_users)
+
+
+def send_presence_update(
+    testcase: TestCase,
+    user_id: str,
+    access_token: str,
+    presence_state: str,
+    status_message: Optional[str] = None,
+) -> JsonDict:
+    # Build the presence body
+    body = {"presence": presence_state}
+    if status_message:
+        body["status_msg"] = status_message
+
+    # Update the user's presence state
+    channel = testcase.make_request(
+        "PUT", "/presence/%s/status" % (user_id,), body, access_token=access_token
+    )
+    testcase.assertEqual(channel.code, 200)
+
+    return channel.json_body
+
+
+def sync_presence(
+    testcase: TestCase,
+    user_id: str,
+    since_token: Optional[StreamToken] = None,
+) -> Tuple[List[UserPresenceState], StreamToken]:
+    """Perform a sync request for the given user and return the user presence updates
+    they've received, as well as the next_batch token.
+
+    This method assumes testcase.sync_handler points to the homeserver's sync handler.
+
+    Args:
+        testcase: The testcase that is currently being run.
+        user_id: The ID of the user to generate a sync response for.
+        since_token: An optional token to indicate from at what point to sync from.
+
+    Returns:
+        A tuple containing a list of presence updates, and the sync response's
+        next_batch token.
+    """
+    requester = create_requester(user_id)
+    sync_config = generate_sync_config(requester.user.to_string())
+    sync_result = testcase.get_success(
+        testcase.sync_handler.wait_for_sync_for_user(
+            requester, sync_config, since_token
+        )
+    )
+
+    return sync_result.presence, sync_result.next_batch
diff --git a/tests/events/test_snapshot.py b/tests/events/test_snapshot.py
index ec85324c0c..ca27388ae8 100644
--- a/tests/events/test_snapshot.py
+++ b/tests/events/test_snapshot.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +14,7 @@
 
 from synapse.events.snapshot import EventContext
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 
 from tests import unittest
 from tests.test_utils.event_injection import create_event
diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py
index 8ba36c6074..7a826c086e 100644
--- a/tests/events/test_utils.py
+++ b/tests/events/test_utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
@@ -302,6 +301,49 @@ def test_redacts(self):
             room_version=RoomVersions.MSC2176,
         )
 
+    def test_join_rules(self):
+        """Join rules events have changed behavior starting with MSC3083."""
+        self.run_test(
+            {
+                "type": "m.room.join_rules",
+                "event_id": "$test:domain",
+                "content": {
+                    "join_rule": "invite",
+                    "allow": [],
+                    "other_key": "stripped",
+                },
+            },
+            {
+                "type": "m.room.join_rules",
+                "event_id": "$test:domain",
+                "content": {"join_rule": "invite"},
+                "signatures": {},
+                "unsigned": {},
+            },
+        )
+
+        # After MSC3083, alias events have no special behavior.
+        self.run_test(
+            {
+                "type": "m.room.join_rules",
+                "content": {
+                    "join_rule": "invite",
+                    "allow": [],
+                    "other_key": "stripped",
+                },
+            },
+            {
+                "type": "m.room.join_rules",
+                "content": {
+                    "join_rule": "invite",
+                    "allow": [],
+                },
+                "signatures": {},
+                "unsigned": {},
+            },
+            room_version=RoomVersions.V8,
+        )
+
 
 class SerializeEventTestCase(unittest.TestCase):
     def serialize(self, ev, fields):
diff --git a/tests/federation/test_complexity.py b/tests/federation/test_complexity.py
index 8186b8ca01..7b486aba4a 100644
--- a/tests/federation/test_complexity.py
+++ b/tests/federation/test_complexity.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 Matrix.org Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.api.errors import Codes, SynapseError
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.types import UserID
 
 from tests import unittest
diff --git a/tests/federation/test_federation_catch_up.py b/tests/federation/test_federation_catch_up.py
index 95eac6a5a3..f0aa8ed9db 100644
--- a/tests/federation/test_federation_catch_up.py
+++ b/tests/federation/test_federation_catch_up.py
@@ -1,13 +1,12 @@
 from typing import List, Tuple
-
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.api.constants import EventTypes
 from synapse.events import EventBase
 from synapse.federation.sender import PerDestinationQueue, TransactionManager
 from synapse.federation.units import Edu
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.util.retryutils import NotRetryingDestination
 
 from tests.test_utils import event_injection, make_awaitable
diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py
index ecc3faa572..65b18fbd7a 100644
--- a/tests/federation/test_federation_sender.py
+++ b/tests/federation/test_federation_sender.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from typing import Optional
-
-from mock import Mock
+from unittest.mock import Mock
 
 from signedjson import key, sign
 from signedjson.types import BaseKey, SigningKey
@@ -23,7 +21,7 @@
 
 from synapse.api.constants import RoomEncryptionAlgorithms
 from synapse.rest import admin
-from synapse.rest.client.v1 import login
+from synapse.rest.client import login
 from synapse.types import JsonDict, ReadReceipt
 
 from tests.test_utils import make_awaitable
diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py
index cfeccc0577..0b60cc4261 100644
--- a/tests/federation/test_federation_server.py
+++ b/tests/federation/test_federation_server.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 # Copyright 2019 Matrix.org Federation C.I.C
 #
@@ -20,7 +19,7 @@
 from synapse.events import make_event_from_dict
 from synapse.federation.federation_server import server_matches_acl_event
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 
 from tests import unittest
 
@@ -75,6 +74,25 @@ def test_block_ip_literals(self):
         self.assertFalse(server_matches_acl_event("[1:2::]", e))
         self.assertTrue(server_matches_acl_event("1:2:3:4", e))
 
+    def test_wildcard_matching(self):
+        e = _create_acl_event({"allow": ["good*.com"]})
+        self.assertTrue(
+            server_matches_acl_event("good.com", e),
+            "* matches 0 characters",
+        )
+        self.assertTrue(
+            server_matches_acl_event("GOOD.COM", e),
+            "pattern is case-insensitive",
+        )
+        self.assertTrue(
+            server_matches_acl_event("good.aa.com", e),
+            "* matches several characters, including '.'",
+        )
+        self.assertFalse(
+            server_matches_acl_event("ishgood.com", e),
+            "pattern does not allow prefixes",
+        )
+
 
 class StateQueryTests(unittest.FederatingHomeserverTestCase):
 
diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py
index fbce19f38b..383214ab50 100644
--- a/tests/federation/transport/test_knocking.py
+++ b/tests/federation/transport/test_knocking.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Matrix.org Federation C.I.C
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,22 +14,17 @@
 from collections import OrderedDict
 from typing import Dict, List
 
-from twisted.internet.defer import succeed
-
 from synapse.api.constants import EventTypes, JoinRules, Membership
 from synapse.api.room_versions import RoomVersions
 from synapse.events import builder
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.server import HomeServer
 from synapse.types import RoomAlias
 
 from tests.test_utils import event_injection
 from tests.unittest import FederatingHomeserverTestCase, TestCase, override_config
 
-# An identifier to use while MSC2304 is not in a stable release of the spec
-KNOCK_UNSTABLE_IDENTIFIER = "xyz.amorgan.knock"
-
 
 class KnockingStrippedStateEventHelperMixin(TestCase):
     def send_example_state_events_to_room(
@@ -133,6 +127,16 @@ def send_example_state_events_to_room(
                 )
             )
 
+        # Finally, we expect to see the m.room.create event of the room as part of the
+        # stripped state. We don't need to inject this event though.
+        room_state[EventTypes.Create] = {
+            "content": {
+                "creator": sender,
+                "room_version": RoomVersions.V7.identifier,
+            },
+            "state_key": "",
+        }
+
         return room_state
 
     def check_knock_room_state_against_room_state(
@@ -192,19 +196,19 @@ def prepare(self, reactor, clock, homeserver):
         # Note that these checks are not relevant to this test case.
 
         # Have this homeserver auto-approve all event signature checking.
-        def approve_all_signature_checking(_, ev):
-            return [succeed(ev[0])]
+        async def approve_all_signature_checking(_, pdu):
+            return pdu
 
-        homeserver.get_federation_server()._check_sigs_and_hashes = (
+        homeserver.get_federation_server()._check_sigs_and_hash = (
             approve_all_signature_checking
         )
 
         # Have this homeserver skip event auth checks. This is necessary due to
-        # event auth checks ensuring that events were signed the sender's homeserver.
-        async def do_auth(origin, event, context, auth_events):
+        # event auth checks ensuring that events were signed by the sender's homeserver.
+        async def _check_event_auth(origin, event, context, *args, **kwargs):
             return context
 
-        homeserver.get_federation_handler().do_auth = do_auth
+        homeserver.get_federation_handler()._check_event_auth = _check_event_auth
 
         return super().prepare(reactor, clock, homeserver)
 
@@ -234,9 +238,8 @@ def test_room_state_returned_when_knocking(self):
 
         channel = self.make_request(
             "GET",
-            "/_matrix/federation/unstable/%s/make_knock/%s/%s?ver=%s"
+            "/_matrix/federation/v1/make_knock/%s/%s?ver=%s"
             % (
-                KNOCK_UNSTABLE_IDENTIFIER,
                 room_id,
                 fake_knocking_user_id,
                 # Inform the remote that we support the room version of the room we're
@@ -278,8 +281,8 @@ def test_room_state_returned_when_knocking(self):
         # Send the signed knock event into the room
         channel = self.make_request(
             "PUT",
-            "/_matrix/federation/unstable/%s/send_knock/%s/%s"
-            % (KNOCK_UNSTABLE_IDENTIFIER, room_id, signed_knock_event.event_id),
+            "/_matrix/federation/v1/send_knock/%s/%s"
+            % (room_id, signed_knock_event.event_id),
             signed_knock_event_json,
         )
         self.assertEquals(200, channel.code, channel.result)
diff --git a/tests/federation/transport/test_server.py b/tests/federation/transport/test_server.py
index 85500e169c..84fa72b9ff 100644
--- a/tests/federation/transport/test_server.py
+++ b/tests/federation/transport/test_server.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/handlers/test_admin.py b/tests/handlers/test_admin.py
index a01fdd0839..59de1142b1 100644
--- a/tests/handlers/test_admin.py
+++ b/tests/handlers/test_admin.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,15 +13,12 @@
 # limitations under the License.
 
 from collections import Counter
+from unittest.mock import Mock
 
-from mock import Mock
-
-import synapse.api.errors
-import synapse.handlers.admin
 import synapse.rest.admin
 import synapse.storage
 from synapse.api.constants import EventTypes
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 
 from tests import unittest
 
diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py
index d5d3fdd99a..43998020b2 100644
--- a/tests/handlers/test_appservice.py
+++ b/tests/handlers/test_appservice.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
@@ -27,7 +26,7 @@
 
 
 class AppServiceHandlerTestCase(unittest.TestCase):
-    """ Tests the ApplicationServicesHandler. """
+    """Tests the ApplicationServicesHandler."""
 
     def setUp(self):
         self.mock_store = Mock()
@@ -58,10 +57,10 @@ def test_notify_interested_services(self):
             sender="@someone:anywhere", type="m.room.message", room_id="!foo:bar"
         )
         self.mock_store.get_new_events_for_appservice.side_effect = [
-            make_awaitable((0, [event])),
             make_awaitable((0, [])),
+            make_awaitable((1, [event])),
         ]
-        self.handler.notify_interested_services(RoomStreamToken(None, 0))
+        self.handler.notify_interested_services(RoomStreamToken(None, 1))
 
         self.mock_scheduler.submit_event_for_as.assert_called_once_with(
             interested_service, event
@@ -78,7 +77,6 @@ def test_query_user_exists_unknown_user(self):
         self.mock_as_api.query_user.return_value = make_awaitable(True)
         self.mock_store.get_new_events_for_appservice.side_effect = [
             make_awaitable((0, [event])),
-            make_awaitable((0, [])),
         ]
 
         self.handler.notify_interested_services(RoomStreamToken(None, 0))
@@ -96,7 +94,6 @@ def test_query_user_exists_known_user(self):
         self.mock_as_api.query_user.return_value = make_awaitable(True)
         self.mock_store.get_new_events_for_appservice.side_effect = [
             make_awaitable((0, [event])),
-            make_awaitable((0, [])),
         ]
 
         self.handler.notify_interested_services(RoomStreamToken(None, 0))
@@ -136,11 +133,131 @@ def test_query_room_alias_exists(self):
         self.assertEquals(result.room_id, room_id)
         self.assertEquals(result.servers, servers)
 
-    def _mkservice(self, is_interested):
+    def test_get_3pe_protocols_no_appservices(self):
+        self.mock_store.get_app_services.return_value = []
+        response = self.successResultOf(
+            defer.ensureDeferred(self.handler.get_3pe_protocols("my-protocol"))
+        )
+        self.mock_as_api.get_3pe_protocol.assert_not_called()
+        self.assertEquals(response, {})
+
+    def test_get_3pe_protocols_no_protocols(self):
+        service = self._mkservice(False, [])
+        self.mock_store.get_app_services.return_value = [service]
+        response = self.successResultOf(
+            defer.ensureDeferred(self.handler.get_3pe_protocols())
+        )
+        self.mock_as_api.get_3pe_protocol.assert_not_called()
+        self.assertEquals(response, {})
+
+    def test_get_3pe_protocols_protocol_no_response(self):
+        service = self._mkservice(False, ["my-protocol"])
+        self.mock_store.get_app_services.return_value = [service]
+        self.mock_as_api.get_3pe_protocol.return_value = make_awaitable(None)
+        response = self.successResultOf(
+            defer.ensureDeferred(self.handler.get_3pe_protocols())
+        )
+        self.mock_as_api.get_3pe_protocol.assert_called_once_with(
+            service, "my-protocol"
+        )
+        self.assertEquals(response, {})
+
+    def test_get_3pe_protocols_select_one_protocol(self):
+        service = self._mkservice(False, ["my-protocol"])
+        self.mock_store.get_app_services.return_value = [service]
+        self.mock_as_api.get_3pe_protocol.return_value = make_awaitable(
+            {"x-protocol-data": 42, "instances": []}
+        )
+        response = self.successResultOf(
+            defer.ensureDeferred(self.handler.get_3pe_protocols("my-protocol"))
+        )
+        self.mock_as_api.get_3pe_protocol.assert_called_once_with(
+            service, "my-protocol"
+        )
+        self.assertEquals(
+            response, {"my-protocol": {"x-protocol-data": 42, "instances": []}}
+        )
+
+    def test_get_3pe_protocols_one_protocol(self):
+        service = self._mkservice(False, ["my-protocol"])
+        self.mock_store.get_app_services.return_value = [service]
+        self.mock_as_api.get_3pe_protocol.return_value = make_awaitable(
+            {"x-protocol-data": 42, "instances": []}
+        )
+        response = self.successResultOf(
+            defer.ensureDeferred(self.handler.get_3pe_protocols())
+        )
+        self.mock_as_api.get_3pe_protocol.assert_called_once_with(
+            service, "my-protocol"
+        )
+        self.assertEquals(
+            response, {"my-protocol": {"x-protocol-data": 42, "instances": []}}
+        )
+
+    def test_get_3pe_protocols_multiple_protocol(self):
+        service_one = self._mkservice(False, ["my-protocol"])
+        service_two = self._mkservice(False, ["other-protocol"])
+        self.mock_store.get_app_services.return_value = [service_one, service_two]
+        self.mock_as_api.get_3pe_protocol.return_value = make_awaitable(
+            {"x-protocol-data": 42, "instances": []}
+        )
+        response = self.successResultOf(
+            defer.ensureDeferred(self.handler.get_3pe_protocols())
+        )
+        self.mock_as_api.get_3pe_protocol.assert_called()
+        self.assertEquals(
+            response,
+            {
+                "my-protocol": {"x-protocol-data": 42, "instances": []},
+                "other-protocol": {"x-protocol-data": 42, "instances": []},
+            },
+        )
+
+    def test_get_3pe_protocols_multiple_info(self):
+        service_one = self._mkservice(False, ["my-protocol"])
+        service_two = self._mkservice(False, ["my-protocol"])
+
+        async def get_3pe_protocol(service, unusedProtocol):
+            if service == service_one:
+                return {
+                    "x-protocol-data": 42,
+                    "instances": [{"desc": "Alice's service"}],
+                }
+            if service == service_two:
+                return {
+                    "x-protocol-data": 36,
+                    "x-not-used": 45,
+                    "instances": [{"desc": "Bob's service"}],
+                }
+            raise Exception("Unexpected service")
+
+        self.mock_store.get_app_services.return_value = [service_one, service_two]
+        self.mock_as_api.get_3pe_protocol = get_3pe_protocol
+        response = self.successResultOf(
+            defer.ensureDeferred(self.handler.get_3pe_protocols())
+        )
+        # It's expected that the second service's data doesn't appear in the response
+        self.assertEquals(
+            response,
+            {
+                "my-protocol": {
+                    "x-protocol-data": 42,
+                    "instances": [
+                        {
+                            "desc": "Alice's service",
+                        },
+                        {"desc": "Bob's service"},
+                    ],
+                },
+            },
+        )
+
+    def _mkservice(self, is_interested, protocols=None):
         service = Mock()
         service.is_interested.return_value = make_awaitable(is_interested)
         service.token = "mock_service_token"
         service.url = "mock_service_url"
+        service.protocols = protocols
         return service
 
     def _mkservice_alias(self, is_interested_in_alias):
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index c9f889b511..5f3350e490 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,17 +11,22 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
 
 import pymacaroons
 
 from synapse.api.errors import AuthError, ResourceLimitError
+from synapse.rest import admin
 
 from tests import unittest
 from tests.test_utils import make_awaitable
 
 
 class AuthTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        admin.register_servlets,
+    ]
+
     def prepare(self, reactor, clock, hs):
         self.auth_handler = hs.get_auth_handler()
         self.macaroon_generator = hs.get_macaroon_generator()
@@ -36,16 +40,10 @@ def prepare(self, reactor, clock, hs):
         self.small_number_of_users = 1
         self.large_number_of_users = 100
 
-    def test_token_is_a_macaroon(self):
-        token = self.macaroon_generator.generate_access_token("some_user")
-        # Check that we can parse the thing with pymacaroons
-        macaroon = pymacaroons.Macaroon.deserialize(token)
-        # The most basic of sanity checks
-        if "some_user" not in macaroon.inspect():
-            self.fail("some_user was not in %s" % macaroon.inspect())
+        self.user1 = self.register_user("a_user", "pass")
 
     def test_macaroon_caveats(self):
-        token = self.macaroon_generator.generate_access_token("a_user")
+        token = self.macaroon_generator.generate_guest_access_token("a_user")
         macaroon = pymacaroons.Macaroon.deserialize(token)
 
         def verify_gen(caveat):
@@ -60,19 +58,23 @@ def verify_type(caveat):
         def verify_nonce(caveat):
             return caveat.startswith("nonce =")
 
+        def verify_guest(caveat):
+            return caveat == "guest = true"
+
         v = pymacaroons.Verifier()
         v.satisfy_general(verify_gen)
         v.satisfy_general(verify_user)
         v.satisfy_general(verify_type)
         v.satisfy_general(verify_nonce)
+        v.satisfy_general(verify_guest)
         v.verify(macaroon, self.hs.config.macaroon_secret_key)
 
     def test_short_term_login_token_gives_user_id(self):
         token = self.macaroon_generator.generate_short_term_login_token(
-            "a_user", "", 5000
+            self.user1, "", 5000
         )
         res = self.get_success(self.auth_handler.validate_short_term_login_token(token))
-        self.assertEqual("a_user", res.user_id)
+        self.assertEqual(self.user1, res.user_id)
         self.assertEqual("", res.auth_provider_id)
 
         # when we advance the clock, the token should be rejected
@@ -84,22 +86,22 @@ def test_short_term_login_token_gives_user_id(self):
 
     def test_short_term_login_token_gives_auth_provider(self):
         token = self.macaroon_generator.generate_short_term_login_token(
-            "a_user", auth_provider_id="my_idp"
+            self.user1, auth_provider_id="my_idp"
         )
         res = self.get_success(self.auth_handler.validate_short_term_login_token(token))
-        self.assertEqual("a_user", res.user_id)
+        self.assertEqual(self.user1, res.user_id)
         self.assertEqual("my_idp", res.auth_provider_id)
 
     def test_short_term_login_token_cannot_replace_user_id(self):
         token = self.macaroon_generator.generate_short_term_login_token(
-            "a_user", "", 5000
+            self.user1, "", 5000
         )
         macaroon = pymacaroons.Macaroon.deserialize(token)
 
         res = self.get_success(
             self.auth_handler.validate_short_term_login_token(macaroon.serialize())
         )
-        self.assertEqual("a_user", res.user_id)
+        self.assertEqual(self.user1, res.user_id)
 
         # add another "user_id" caveat, which might allow us to override the
         # user_id.
@@ -115,7 +117,7 @@ def test_mau_limits_disabled(self):
         # Ensure does not throw exception
         self.get_success(
             self.auth_handler.get_access_token_for_user_id(
-                "user_a", device_id=None, valid_until_ms=None
+                self.user1, device_id=None, valid_until_ms=None
             )
         )
 
@@ -133,7 +135,7 @@ def test_mau_limits_exceeded_large(self):
 
         self.get_failure(
             self.auth_handler.get_access_token_for_user_id(
-                "user_a", device_id=None, valid_until_ms=None
+                self.user1, device_id=None, valid_until_ms=None
             ),
             ResourceLimitError,
         )
@@ -161,7 +163,7 @@ def test_mau_limits_parity(self):
         # If not in monthly active cohort
         self.get_failure(
             self.auth_handler.get_access_token_for_user_id(
-                "user_a", device_id=None, valid_until_ms=None
+                self.user1, device_id=None, valid_until_ms=None
             ),
             ResourceLimitError,
         )
@@ -178,7 +180,7 @@ def test_mau_limits_parity(self):
         )
         self.get_success(
             self.auth_handler.get_access_token_for_user_id(
-                "user_a", device_id=None, valid_until_ms=None
+                self.user1, device_id=None, valid_until_ms=None
             )
         )
         self.get_success(
@@ -196,7 +198,7 @@ def test_mau_limits_not_exceeded(self):
         # Ensure does not raise exception
         self.get_success(
             self.auth_handler.get_access_token_for_user_id(
-                "user_a", device_id=None, valid_until_ms=None
+                self.user1, device_id=None, valid_until_ms=None
             )
         )
 
@@ -211,6 +213,6 @@ def test_mau_limits_not_exceeded(self):
 
     def _get_macaroon(self):
         token = self.macaroon_generator.generate_short_term_login_token(
-            "user_a", "", 5000
+            self.user1, "", 5000
         )
         return pymacaroons.Macaroon.deserialize(token)
diff --git a/tests/handlers/test_cas.py b/tests/handlers/test_cas.py
index 7975af243c..b625995d12 100644
--- a/tests/handlers/test_cas.py
+++ b/tests/handlers/test_cas.py
@@ -11,9 +11,9 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
 
-from synapse.handlers.cas_handler import CasResponse
+from synapse.handlers.cas import CasResponse
 
 from tests.test_utils import simple_async_mock
 from tests.unittest import HomeserverTestCase, override_config
diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py
index 821629bc38..3ac48e5e95 100644
--- a/tests/handlers/test_device.py
+++ b/tests/handlers/test_device.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
@@ -258,7 +257,7 @@ def test_dehydrate_and_rehydrate_device(self):
         self.assertEqual(device_data, {"device_data": {"foo": "bar"}})
 
         # Create a new login for the user and dehydrated the device
-        device_id, access_token = self.get_success(
+        device_id, access_token, _expiration_time, _refresh_token = self.get_success(
             self.registration.register_device(
                 user_id=user_id,
                 device_id=None,
diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py
index fadec16e13..98d748b6cd 100644
--- a/tests/handlers/test_directory.py
+++ b/tests/handlers/test_directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,13 +13,13 @@
 # limitations under the License.
 
 
-from mock import Mock
+from unittest.mock import Mock
 
 import synapse
 import synapse.api.errors
 from synapse.api.constants import EventTypes
 from synapse.config.room_directory import RoomDirectoryConfig
-from synapse.rest.client.v1 import directory, login, room
+from synapse.rest.client import directory, login, room
 from synapse.types import RoomAlias, create_requester
 
 from tests import unittest
@@ -28,7 +27,7 @@
 
 
 class DirectoryTestCase(unittest.HomeserverTestCase):
-    """ Tests the directory service. """
+    """Tests the directory service."""
 
     def make_homeserver(self, reactor, clock):
         self.mock_federation = Mock()
diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py
index 5e86c5e56b..39e7b1ab25 100644
--- a/tests/handlers/test_e2e_keys.py
+++ b/tests/handlers/test_e2e_keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -14,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import mock
+from unittest import mock
 
 from signedjson import key as key, sign as sign
 
@@ -48,12 +47,16 @@ def test_reupload_one_time_keys(self):
             "alg2:k3": {"key": "key3"},
         }
 
+        # Note that "signed_curve25519" is always returned in key count responses. This is necessary until
+        # https://github.com/matrix-org/matrix-doc/issues/3298 is fixed.
         res = self.get_success(
             self.handler.upload_keys_for_user(
                 local_user, device_id, {"one_time_keys": keys}
             )
         )
-        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1, "alg2": 2}})
+        self.assertDictEqual(
+            res, {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}}
+        )
 
         # we should be able to change the signature without a problem
         keys["alg2:k2"]["signatures"]["k1"] = "sig2"
@@ -62,7 +65,9 @@ def test_reupload_one_time_keys(self):
                 local_user, device_id, {"one_time_keys": keys}
             )
         )
-        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1, "alg2": 2}})
+        self.assertDictEqual(
+            res, {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}}
+        )
 
     def test_change_one_time_keys(self):
         """attempts to change one-time-keys should be rejected"""
@@ -80,7 +85,9 @@ def test_change_one_time_keys(self):
                 local_user, device_id, {"one_time_keys": keys}
             )
         )
-        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1, "alg2": 2}})
+        self.assertDictEqual(
+            res, {"one_time_key_counts": {"alg1": 1, "alg2": 2, "signed_curve25519": 0}}
+        )
 
         # Error when changing string key
         self.get_failure(
@@ -90,7 +97,7 @@ def test_change_one_time_keys(self):
             SynapseError,
         )
 
-        # Error when replacing dict key with strin
+        # Error when replacing dict key with string
         self.get_failure(
             self.handler.upload_keys_for_user(
                 local_user, device_id, {"one_time_keys": {"alg2:k3": "key2"}}
@@ -132,7 +139,9 @@ def test_claim_one_time_key(self):
                 local_user, device_id, {"one_time_keys": keys}
             )
         )
-        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1}})
+        self.assertDictEqual(
+            res, {"one_time_key_counts": {"alg1": 1, "signed_curve25519": 0}}
+        )
 
         res2 = self.get_success(
             self.handler.claim_one_time_keys(
@@ -258,7 +267,9 @@ def test_replace_master_key(self):
         self.get_success(self.handler.upload_signing_keys_for_user(local_user, keys2))
 
         devices = self.get_success(
-            self.handler.query_devices({"device_keys": {local_user: []}}, 0, local_user)
+            self.handler.query_devices(
+                {"device_keys": {local_user: []}}, 0, local_user, "device123"
+            )
         )
         self.assertDictEqual(devices["master_keys"], {local_user: keys2["master_key"]})
 
@@ -358,7 +369,9 @@ def test_reupload_signatures(self):
         device_key_1["signatures"][local_user]["ed25519:abc"] = "base64+signature"
         device_key_2["signatures"][local_user]["ed25519:def"] = "base64+signature"
         devices = self.get_success(
-            self.handler.query_devices({"device_keys": {local_user: []}}, 0, local_user)
+            self.handler.query_devices(
+                {"device_keys": {local_user: []}}, 0, local_user, "device123"
+            )
         )
         del devices["device_keys"][local_user]["abc"]["unsigned"]
         del devices["device_keys"][local_user]["def"]["unsigned"]
@@ -592,7 +605,10 @@ def test_upload_signatures(self):
         # fetch the signed keys/devices and make sure that the signatures are there
         ret = self.get_success(
             self.handler.query_devices(
-                {"device_keys": {local_user: [], other_user: []}}, 0, local_user
+                {"device_keys": {local_user: [], other_user: []}},
+                0,
+                local_user,
+                "device123",
             )
         )
 
diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
index d7498aa51a..9b7e7a8e9a 100644
--- a/tests/handlers/test_e2e_room_keys.py
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2017 New Vector Ltd
 # Copyright 2019 Matrix.org Foundation C.I.C.
@@ -16,8 +15,7 @@
 # limitations under the License.
 
 import copy
-
-import mock
+from unittest import mock
 
 from synapse.api.errors import SynapseError
 
diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py
index 3af361195b..c72a8972a3 100644
--- a/tests/handlers/test_federation.py
+++ b/tests/handlers/test_federation.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+from typing import List
 from unittest import TestCase
 
 from synapse.api.constants import EventTypes
@@ -22,7 +22,8 @@
 from synapse.federation.federation_base import event_from_pdu_json
 from synapse.logging.context import LoggingContext, run_in_background
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
+from synapse.util.stringutils import random_string
 
 from tests import unittest
 
@@ -40,6 +41,8 @@ def make_homeserver(self, reactor, clock):
         hs = self.setup_test_homeserver(federation_http_client=None)
         self.handler = hs.get_federation_handler()
         self.store = hs.get_datastore()
+        self.state_store = hs.get_storage().state
+        self._event_auth_handler = hs.get_event_auth_handler()
         return hs
 
     def test_exchange_revoked_invite(self):
@@ -191,6 +194,133 @@ def test_rejected_state_event_state(self):
 
         self.assertEqual(sg, sg2)
 
+    def test_backfill_floating_outlier_membership_auth(self):
+        """
+        As the local homeserver, check that we can properly process a federated
+        event from the OTHER_SERVER with auth_events that include a floating
+        membership event from the OTHER_SERVER.
+
+        Regression test, see #10439.
+        """
+        OTHER_SERVER = "otherserver"
+        OTHER_USER = "@otheruser:" + OTHER_SERVER
+
+        # create the room
+        user_id = self.register_user("kermit", "test")
+        tok = self.login("kermit", "test")
+        room_id = self.helper.create_room_as(
+            room_creator=user_id,
+            is_public=True,
+            tok=tok,
+            extra_content={
+                "preset": "public_chat",
+            },
+        )
+        room_version = self.get_success(self.store.get_room_version(room_id))
+
+        prev_event_ids = self.get_success(self.store.get_prev_events_for_room(room_id))
+        (
+            most_recent_prev_event_id,
+            most_recent_prev_event_depth,
+        ) = self.get_success(self.store.get_max_depth_of(prev_event_ids))
+        # mapping from (type, state_key) -> state_event_id
+        prev_state_map = self.get_success(
+            self.state_store.get_state_ids_for_event(most_recent_prev_event_id)
+        )
+        # List of state event ID's
+        prev_state_ids = list(prev_state_map.values())
+        auth_event_ids = prev_state_ids
+        auth_events = list(
+            self.get_success(self.store.get_events(auth_event_ids)).values()
+        )
+
+        # build a floating outlier member state event
+        fake_prev_event_id = "$" + random_string(43)
+        member_event_dict = {
+            "type": EventTypes.Member,
+            "content": {
+                "membership": "join",
+            },
+            "state_key": OTHER_USER,
+            "room_id": room_id,
+            "sender": OTHER_USER,
+            "depth": most_recent_prev_event_depth,
+            "prev_events": [fake_prev_event_id],
+            "origin_server_ts": self.clock.time_msec(),
+            "signatures": {OTHER_SERVER: {"ed25519:key_version": "SomeSignatureHere"}},
+        }
+        builder = self.hs.get_event_builder_factory().for_room_version(
+            room_version, member_event_dict
+        )
+        member_event = self.get_success(
+            builder.build(
+                prev_event_ids=member_event_dict["prev_events"],
+                auth_event_ids=self._event_auth_handler.compute_auth_events(
+                    builder,
+                    prev_state_map,
+                    for_verification=False,
+                ),
+                depth=member_event_dict["depth"],
+            )
+        )
+        # Override the signature added from "test" homeserver that we created the event with
+        member_event.signatures = member_event_dict["signatures"]
+
+        # Add the new member_event to the StateMap
+        prev_state_map[
+            (member_event.type, member_event.state_key)
+        ] = member_event.event_id
+        auth_events.append(member_event)
+
+        # build and send an event authed based on the member event
+        message_event_dict = {
+            "type": EventTypes.Message,
+            "content": {},
+            "room_id": room_id,
+            "sender": OTHER_USER,
+            "depth": most_recent_prev_event_depth,
+            "prev_events": prev_event_ids.copy(),
+            "origin_server_ts": self.clock.time_msec(),
+            "signatures": {OTHER_SERVER: {"ed25519:key_version": "SomeSignatureHere"}},
+        }
+        builder = self.hs.get_event_builder_factory().for_room_version(
+            room_version, message_event_dict
+        )
+        message_event = self.get_success(
+            builder.build(
+                prev_event_ids=message_event_dict["prev_events"],
+                auth_event_ids=self._event_auth_handler.compute_auth_events(
+                    builder,
+                    prev_state_map,
+                    for_verification=False,
+                ),
+                depth=message_event_dict["depth"],
+            )
+        )
+        # Override the signature added from "test" homeserver that we created the event with
+        message_event.signatures = message_event_dict["signatures"]
+
+        # Stub the /event_auth response from the OTHER_SERVER
+        async def get_event_auth(
+            destination: str, room_id: str, event_id: str
+        ) -> List[EventBase]:
+            return auth_events
+
+        self.handler.federation_client.get_event_auth = get_event_auth
+
+        with LoggingContext("receive_pdu"):
+            # Fake the OTHER_SERVER federating the message event over to our local homeserver
+            d = run_in_background(
+                self.handler.on_receive_pdu, OTHER_SERVER, message_event
+            )
+        self.get_success(d)
+
+        # Now try and get the events on our local homeserver
+        stored_event = self.get_success(
+            self.store.get_event(message_event.event_id, allow_none=True)
+        )
+        self.assertTrue(stored_event is not None)
+
     @unittest.override_config(
         {"rc_invites": {"per_user": {"per_second": 0.5, "burst_count": 3}}}
     )
@@ -223,7 +353,7 @@ def create_invite():
                 room_version,
             )
 
-        for i in range(3):
+        for _ in range(3):
             event = create_invite()
             self.get_success(
                 self.handler.on_invite_request(
@@ -252,7 +382,7 @@ def _build_and_send_join_event(self, other_server, other_user, room_id):
         join_event.signatures[other_server] = {"x": "y"}
         with LoggingContext("send_join"):
             d = run_in_background(
-                self.handler.on_send_join_request, other_server, join_event
+                self.handler.on_send_membership_event, other_server, join_event
             )
         self.get_success(d)
 
diff --git a/tests/handlers/test_identity.py b/tests/handlers/test_identity.py
index b7d340bcb8..6a0e784dd7 100644
--- a/tests/handlers/test_identity.py
+++ b/tests/handlers/test_identity.py
@@ -13,13 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
 import synapse.rest.admin
-from synapse.rest.client.v1 import login
-from synapse.rest.client.v2_alpha import account
+from synapse.rest.client import account, login
 
 from tests import unittest
 
diff --git a/tests/handlers/test_message.py b/tests/handlers/test_message.py
index a0d1ebdbe3..8a8d369fac 100644
--- a/tests/handlers/test_message.py
+++ b/tests/handlers/test_message.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,7 +18,7 @@
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.types import create_requester
 from synapse.util.stringutils import random_string
 
diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py
index c7796fb837..a25c89bd5b 100644
--- a/tests/handlers/test_oidc.py
+++ b/tests/handlers/test_oidc.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Quentin Gliech
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,10 +13,9 @@
 # limitations under the License.
 import json
 import os
+from unittest.mock import ANY, Mock, patch
 from urllib.parse import parse_qs, urlparse
 
-from mock import ANY, Mock, patch
-
 import pymacaroons
 
 from synapse.handlers.sso import MappingException
@@ -501,7 +499,7 @@ def test_callback(self):
         self.assertRenderedError("fetch_error")
 
         # Handle code exchange failure
-        from synapse.handlers.oidc_handler import OidcError
+        from synapse.handlers.oidc import OidcError
 
         self.provider._exchange_code = simple_async_mock(
             raises=OidcError("invalid_request")
@@ -585,7 +583,7 @@ def test_exchange_code(self):
                 body=b'{"error": "foo", "error_description": "bar"}',
             )
         )
-        from synapse.handlers.oidc_handler import OidcError
+        from synapse.handlers.oidc import OidcError
 
         exc = self.get_failure(self.provider._exchange_code(code), OidcError)
         self.assertEqual(exc.value.error, "foo")
@@ -1128,7 +1126,7 @@ def _generate_oidc_session_token(
         client_redirect_url: str,
         ui_auth_session_id: str = "",
     ) -> str:
-        from synapse.handlers.oidc_handler import OidcSessionData
+        from synapse.handlers.oidc import OidcSessionData
 
         return self.handler._token_generator.generate_oidc_session_token(
             state=state,
@@ -1154,7 +1152,7 @@ async def _make_callback_with_userinfo(
         userinfo: the OIDC userinfo dict
         client_redirect_url: the URL to redirect to on success.
     """
-    from synapse.handlers.oidc_handler import OidcSessionData
+    from synapse.handlers.oidc import OidcSessionData
 
     handler = hs.get_oidc_handler()
     provider = handler._providers["oidc"]
diff --git a/tests/handlers/test_password_providers.py b/tests/handlers/test_password_providers.py
index a98a65ae67..38e6d9f536 100644
--- a/tests/handlers/test_password_providers.py
+++ b/tests/handlers/test_password_providers.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,14 +15,12 @@
 """Tests for the password_auth_provider interface"""
 
 from typing import Any, Type, Union
-
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
 import synapse
-from synapse.rest.client.v1 import login
-from synapse.rest.client.v2_alpha import devices
+from synapse.rest.client import devices, login
 from synapse.types import JsonDict
 
 from tests import unittest
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index 77330f59a9..0a52bc8b72 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from mock import Mock, call
+from typing import Optional
+from unittest.mock import Mock, call
 
 from signedjson.key import generate_signing_key
 
@@ -22,6 +21,7 @@
 from synapse.api.presence import UserPresenceState
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
 from synapse.events.builder import EventBuilder
+from synapse.federation.sender import FederationSender
 from synapse.handlers.presence import (
     EXTERNAL_PROCESS_EXPIRY,
     FEDERATION_PING_INTERVAL,
@@ -32,13 +32,19 @@
     handle_timeout,
     handle_update,
 )
-from synapse.rest.client.v1 import room
+from synapse.rest import admin
+from synapse.rest.client import room
 from synapse.types import UserID, get_domain_from_id
 
 from tests import unittest
 
 
-class PresenceUpdateTestCase(unittest.TestCase):
+class PresenceUpdateTestCase(unittest.HomeserverTestCase):
+    servlets = [admin.register_servlets]
+
+    def prepare(self, reactor, clock, homeserver):
+        self.store = homeserver.get_datastore()
+
     def test_offline_to_online(self):
         wheel_timer = Mock()
         user_id = "@foo:bar"
@@ -292,10 +298,52 @@ def test_online_to_idle(self):
             any_order=True,
         )
 
+    def test_persisting_presence_updates(self):
+        """Tests that the latest presence state for each user is persisted correctly"""
+        # Create some test users and presence states for them
+        presence_states = []
+        for i in range(5):
+            user_id = self.register_user(f"user_{i}", "password")
+
+            presence_state = UserPresenceState(
+                user_id=user_id,
+                state="online",
+                last_active_ts=1,
+                last_federation_update_ts=1,
+                last_user_sync_ts=1,
+                status_msg="I'm online!",
+                currently_active=True,
+            )
+            presence_states.append(presence_state)
+
+        # Persist these presence updates to the database
+        self.get_success(self.store.update_presence(presence_states))
+
+        # Check that each update is present in the database
+        db_presence_states = self.get_success(
+            self.store.get_all_presence_updates(
+                instance_name="master",
+                last_id=0,
+                current_id=len(presence_states) + 1,
+                limit=len(presence_states),
+            )
+        )
+
+        # Extract presence update user ID and state information into lists of tuples
+        db_presence_states = [(ps[0], ps[1]) for _, ps in db_presence_states[0]]
+        presence_states = [(ps.user_id, ps.state) for ps in presence_states]
+
+        # Compare what we put into the storage with what we got out.
+        # They should be identical.
+        self.assertEqual(presence_states, db_presence_states)
+
 
 class PresenceTimeoutTestCase(unittest.TestCase):
+    """Tests different timers and that the timer does not change `status_msg` of user."""
+
     def test_idle_timer(self):
         user_id = "@foo:bar"
+        status_msg = "I'm here!"
         now = 5000000
 
         state = UserPresenceState.default(user_id)
@@ -303,12 +351,14 @@ def test_idle_timer(self):
             state=PresenceState.ONLINE,
             last_active_ts=now - IDLE_TIMER - 1,
             last_user_sync_ts=now,
+            status_msg=status_msg,
         )
 
         new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
 
         self.assertIsNotNone(new_state)
         self.assertEquals(new_state.state, PresenceState.UNAVAILABLE)
+        self.assertEquals(new_state.status_msg, status_msg)
 
     def test_busy_no_idle(self):
         """
@@ -316,6 +366,7 @@ def test_busy_no_idle(self):
         presence state into unavailable.
         """
         user_id = "@foo:bar"
+        status_msg = "I'm here!"
         now = 5000000
 
         state = UserPresenceState.default(user_id)
@@ -323,15 +374,18 @@ def test_busy_no_idle(self):
             state=PresenceState.BUSY,
             last_active_ts=now - IDLE_TIMER - 1,
             last_user_sync_ts=now,
+            status_msg=status_msg,
         )
 
         new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
 
         self.assertIsNotNone(new_state)
         self.assertEquals(new_state.state, PresenceState.BUSY)
+        self.assertEquals(new_state.status_msg, status_msg)
 
     def test_sync_timeout(self):
         user_id = "@foo:bar"
+        status_msg = "I'm here!"
         now = 5000000
 
         state = UserPresenceState.default(user_id)
@@ -339,15 +393,18 @@ def test_sync_timeout(self):
             state=PresenceState.ONLINE,
             last_active_ts=0,
             last_user_sync_ts=now - SYNC_ONLINE_TIMEOUT - 1,
+            status_msg=status_msg,
         )
 
         new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
 
         self.assertIsNotNone(new_state)
         self.assertEquals(new_state.state, PresenceState.OFFLINE)
+        self.assertEquals(new_state.status_msg, status_msg)
 
     def test_sync_online(self):
         user_id = "@foo:bar"
+        status_msg = "I'm here!"
         now = 5000000
 
         state = UserPresenceState.default(user_id)
@@ -355,6 +412,7 @@ def test_sync_online(self):
             state=PresenceState.ONLINE,
             last_active_ts=now - SYNC_ONLINE_TIMEOUT - 1,
             last_user_sync_ts=now - SYNC_ONLINE_TIMEOUT - 1,
+            status_msg=status_msg,
         )
 
         new_state = handle_timeout(
@@ -363,9 +421,11 @@ def test_sync_online(self):
 
         self.assertIsNotNone(new_state)
         self.assertEquals(new_state.state, PresenceState.ONLINE)
+        self.assertEquals(new_state.status_msg, status_msg)
 
     def test_federation_ping(self):
         user_id = "@foo:bar"
+        status_msg = "I'm here!"
         now = 5000000
 
         state = UserPresenceState.default(user_id)
@@ -374,12 +434,13 @@ def test_federation_ping(self):
             last_active_ts=now,
             last_user_sync_ts=now,
             last_federation_update_ts=now - FEDERATION_PING_INTERVAL - 1,
+            status_msg=status_msg,
         )
 
         new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
 
         self.assertIsNotNone(new_state)
-        self.assertEquals(new_state, new_state)
+        self.assertEquals(state, new_state)
 
     def test_no_timeout(self):
         user_id = "@foo:bar"
@@ -399,6 +460,7 @@ def test_no_timeout(self):
 
     def test_federation_timeout(self):
         user_id = "@foo:bar"
+        status_msg = "I'm here!"
         now = 5000000
 
         state = UserPresenceState.default(user_id)
@@ -407,6 +469,7 @@ def test_federation_timeout(self):
             last_active_ts=now,
             last_user_sync_ts=now,
             last_federation_update_ts=now - FEDERATION_TIMEOUT - 1,
+            status_msg=status_msg,
         )
 
         new_state = handle_timeout(
@@ -415,9 +478,11 @@ def test_federation_timeout(self):
 
         self.assertIsNotNone(new_state)
         self.assertEquals(new_state.state, PresenceState.OFFLINE)
+        self.assertEquals(new_state.status_msg, status_msg)
 
     def test_last_active(self):
         user_id = "@foo:bar"
+        status_msg = "I'm here!"
         now = 5000000
 
         state = UserPresenceState.default(user_id)
@@ -426,6 +491,7 @@ def test_last_active(self):
             last_active_ts=now - LAST_ACTIVE_GRANULARITY - 1,
             last_user_sync_ts=now,
             last_federation_update_ts=now,
+            status_msg=status_msg,
         )
 
         new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
@@ -471,6 +537,328 @@ def test_external_process_timeout(self):
         )
         self.assertEqual(state.state, PresenceState.OFFLINE)
 
+    def test_user_goes_offline_by_timeout_status_msg_remain(self):
+        """Test that if a user doesn't update the records for a while
+        users presence goes `OFFLINE` because of timeout and `status_msg` remains.
+        """
+        user_id = "@test:server"
+        status_msg = "I'm here!"
+
+        # Mark user as online
+        self._set_presencestate_with_status_msg(
+            user_id, PresenceState.ONLINE, status_msg
+        )
+
+        # Check that if we wait a while without telling the handler the user has
+        # stopped syncing that their presence state doesn't get timed out.
+        self.reactor.advance(SYNC_ONLINE_TIMEOUT / 2)
+
+        state = self.get_success(
+            self.presence_handler.get_state(UserID.from_string(user_id))
+        )
+        self.assertEqual(state.state, PresenceState.ONLINE)
+        self.assertEqual(state.status_msg, status_msg)
+
+        # Check that if the timeout fires, then the syncing user gets timed out
+        self.reactor.advance(SYNC_ONLINE_TIMEOUT)
+
+        state = self.get_success(
+            self.presence_handler.get_state(UserID.from_string(user_id))
+        )
+        # status_msg should remain even after going offline
+        self.assertEqual(state.state, PresenceState.OFFLINE)
+        self.assertEqual(state.status_msg, status_msg)
+
+    def test_user_goes_offline_manually_with_no_status_msg(self):
+        """Test that if a user change presence manually to `OFFLINE`
+        and no status is set, that `status_msg` is `None`.
+        """
+        user_id = "@test:server"
+        status_msg = "I'm here!"
+
+        # Mark user as online
+        self._set_presencestate_with_status_msg(
+            user_id, PresenceState.ONLINE, status_msg
+        )
+
+        # Mark user as offline
+        self.get_success(
+            self.presence_handler.set_state(
+                UserID.from_string(user_id), {"presence": PresenceState.OFFLINE}
+            )
+        )
+
+        state = self.get_success(
+            self.presence_handler.get_state(UserID.from_string(user_id))
+        )
+        self.assertEqual(state.state, PresenceState.OFFLINE)
+        self.assertEqual(state.status_msg, None)
+
+    def test_user_goes_offline_manually_with_status_msg(self):
+        """Test that if a user change presence manually to `OFFLINE`
+        and a status is set, that `status_msg` appears.
+        """
+        user_id = "@test:server"
+        status_msg = "I'm here!"
+
+        # Mark user as online
+        self._set_presencestate_with_status_msg(
+            user_id, PresenceState.ONLINE, status_msg
+        )
+
+        # Mark user as offline
+        self._set_presencestate_with_status_msg(
+            user_id, PresenceState.OFFLINE, "And now here."
+        )
+
+    def test_user_reset_online_with_no_status(self):
+        """Test that if a user set again the presence manually
+        and no status is set, that `status_msg` is `None`.
+        """
+        user_id = "@test:server"
+        status_msg = "I'm here!"
+
+        # Mark user as online
+        self._set_presencestate_with_status_msg(
+            user_id, PresenceState.ONLINE, status_msg
+        )
+
+        # Mark user as online again
+        self.get_success(
+            self.presence_handler.set_state(
+                UserID.from_string(user_id), {"presence": PresenceState.ONLINE}
+            )
+        )
+
+        state = self.get_success(
+            self.presence_handler.get_state(UserID.from_string(user_id))
+        )
+        # status_msg should remain even after going offline
+        self.assertEqual(state.state, PresenceState.ONLINE)
+        self.assertEqual(state.status_msg, None)
+
+    def test_set_presence_with_status_msg_none(self):
+        """Test that if a user set again the presence manually
+        and status is `None`, that `status_msg` is `None`.
+        """
+        user_id = "@test:server"
+        status_msg = "I'm here!"
+
+        # Mark user as online
+        self._set_presencestate_with_status_msg(
+            user_id, PresenceState.ONLINE, status_msg
+        )
+
+        # Mark user as online and `status_msg = None`
+        self._set_presencestate_with_status_msg(user_id, PresenceState.ONLINE, None)
+
+    def _set_presencestate_with_status_msg(
+        self, user_id: str, state: PresenceState, status_msg: Optional[str]
+    ):
+        """Set a PresenceState and status_msg and check the result.
+
+        Args:
+            user_id: User for that the status is to be set.
+            PresenceState: The new PresenceState.
+            status_msg: Status message that is to be set.
+        """
+        self.get_success(
+            self.presence_handler.set_state(
+                UserID.from_string(user_id),
+                {"presence": state, "status_msg": status_msg},
+            )
+        )
+
+        new_state = self.get_success(
+            self.presence_handler.get_state(UserID.from_string(user_id))
+        )
+        self.assertEqual(new_state.state, state)
+        self.assertEqual(new_state.status_msg, status_msg)
+
+
+class PresenceFederationQueueTestCase(unittest.HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
+        self.presence_handler = hs.get_presence_handler()
+        self.clock = hs.get_clock()
+        self.instance_name = hs.get_instance_name()
+
+        self.queue = self.presence_handler.get_federation_queue()
+
+    def test_send_and_get(self):
+        state1 = UserPresenceState.default("@user1:test")
+        state2 = UserPresenceState.default("@user2:test")
+        state3 = UserPresenceState.default("@user3:test")
+
+        prev_token = self.queue.get_current_token(self.instance_name)
+
+        self.queue.send_presence_to_destinations((state1, state2), ("dest1", "dest2"))
+        self.queue.send_presence_to_destinations((state3,), ("dest3",))
+
+        now_token = self.queue.get_current_token(self.instance_name)
+
+        rows, upto_token, limited = self.get_success(
+            self.queue.get_replication_rows("master", prev_token, now_token, 10)
+        )
+
+        self.assertEqual(upto_token, now_token)
+        self.assertFalse(limited)
+
+        expected_rows = [
+            (1, ("dest1", "@user1:test")),
+            (1, ("dest2", "@user1:test")),
+            (1, ("dest1", "@user2:test")),
+            (1, ("dest2", "@user2:test")),
+            (2, ("dest3", "@user3:test")),
+        ]
+
+        self.assertCountEqual(rows, expected_rows)
+
+        now_token = self.queue.get_current_token(self.instance_name)
+        rows, upto_token, limited = self.get_success(
+            self.queue.get_replication_rows("master", upto_token, now_token, 10)
+        )
+        self.assertEqual(upto_token, now_token)
+        self.assertFalse(limited)
+        self.assertCountEqual(rows, [])
+
+    def test_send_and_get_split(self):
+        state1 = UserPresenceState.default("@user1:test")
+        state2 = UserPresenceState.default("@user2:test")
+        state3 = UserPresenceState.default("@user3:test")
+
+        prev_token = self.queue.get_current_token(self.instance_name)
+
+        self.queue.send_presence_to_destinations((state1, state2), ("dest1", "dest2"))
+
+        now_token = self.queue.get_current_token(self.instance_name)
+
+        self.queue.send_presence_to_destinations((state3,), ("dest3",))
+
+        rows, upto_token, limited = self.get_success(
+            self.queue.get_replication_rows("master", prev_token, now_token, 10)
+        )
+
+        self.assertEqual(upto_token, now_token)
+        self.assertFalse(limited)
+
+        expected_rows = [
+            (1, ("dest1", "@user1:test")),
+            (1, ("dest2", "@user1:test")),
+            (1, ("dest1", "@user2:test")),
+            (1, ("dest2", "@user2:test")),
+        ]
+
+        self.assertCountEqual(rows, expected_rows)
+
+        now_token = self.queue.get_current_token(self.instance_name)
+        rows, upto_token, limited = self.get_success(
+            self.queue.get_replication_rows("master", upto_token, now_token, 10)
+        )
+
+        self.assertEqual(upto_token, now_token)
+        self.assertFalse(limited)
+
+        expected_rows = [
+            (2, ("dest3", "@user3:test")),
+        ]
+
+        self.assertCountEqual(rows, expected_rows)
+
+    def test_clear_queue_all(self):
+        state1 = UserPresenceState.default("@user1:test")
+        state2 = UserPresenceState.default("@user2:test")
+        state3 = UserPresenceState.default("@user3:test")
+
+        prev_token = self.queue.get_current_token(self.instance_name)
+
+        self.queue.send_presence_to_destinations((state1, state2), ("dest1", "dest2"))
+        self.queue.send_presence_to_destinations((state3,), ("dest3",))
+
+        self.reactor.advance(10 * 60 * 1000)
+
+        now_token = self.queue.get_current_token(self.instance_name)
+
+        rows, upto_token, limited = self.get_success(
+            self.queue.get_replication_rows("master", prev_token, now_token, 10)
+        )
+        self.assertEqual(upto_token, now_token)
+        self.assertFalse(limited)
+        self.assertCountEqual(rows, [])
+
+        prev_token = self.queue.get_current_token(self.instance_name)
+
+        self.queue.send_presence_to_destinations((state1, state2), ("dest1", "dest2"))
+        self.queue.send_presence_to_destinations((state3,), ("dest3",))
+
+        now_token = self.queue.get_current_token(self.instance_name)
+
+        rows, upto_token, limited = self.get_success(
+            self.queue.get_replication_rows("master", prev_token, now_token, 10)
+        )
+        self.assertEqual(upto_token, now_token)
+        self.assertFalse(limited)
+
+        expected_rows = [
+            (3, ("dest1", "@user1:test")),
+            (3, ("dest2", "@user1:test")),
+            (3, ("dest1", "@user2:test")),
+            (3, ("dest2", "@user2:test")),
+            (4, ("dest3", "@user3:test")),
+        ]
+
+        self.assertCountEqual(rows, expected_rows)
+
+    def test_partially_clear_queue(self):
+        state1 = UserPresenceState.default("@user1:test")
+        state2 = UserPresenceState.default("@user2:test")
+        state3 = UserPresenceState.default("@user3:test")
+
+        prev_token = self.queue.get_current_token(self.instance_name)
+
+        self.queue.send_presence_to_destinations((state1, state2), ("dest1", "dest2"))
+
+        self.reactor.advance(2 * 60 * 1000)
+
+        self.queue.send_presence_to_destinations((state3,), ("dest3",))
+
+        self.reactor.advance(4 * 60 * 1000)
+
+        now_token = self.queue.get_current_token(self.instance_name)
+
+        rows, upto_token, limited = self.get_success(
+            self.queue.get_replication_rows("master", prev_token, now_token, 10)
+        )
+        self.assertEqual(upto_token, now_token)
+        self.assertFalse(limited)
+
+        expected_rows = [
+            (2, ("dest3", "@user3:test")),
+        ]
+        self.assertCountEqual(rows, [])
+
+        prev_token = self.queue.get_current_token(self.instance_name)
+
+        self.queue.send_presence_to_destinations((state1, state2), ("dest1", "dest2"))
+        self.queue.send_presence_to_destinations((state3,), ("dest3",))
+
+        now_token = self.queue.get_current_token(self.instance_name)
+
+        rows, upto_token, limited = self.get_success(
+            self.queue.get_replication_rows("master", prev_token, now_token, 10)
+        )
+        self.assertEqual(upto_token, now_token)
+        self.assertFalse(limited)
+
+        expected_rows = [
+            (3, ("dest1", "@user1:test")),
+            (3, ("dest2", "@user1:test")),
+            (3, ("dest1", "@user2:test")),
+            (3, ("dest2", "@user2:test")),
+            (4, ("dest3", "@user3:test")),
+        ]
+
+        self.assertCountEqual(rows, expected_rows)
+
 
 class PresenceJoinTestCase(unittest.HomeserverTestCase):
     """Tests remote servers get told about presence of users in the room when
@@ -483,10 +871,17 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase):
 
     def make_homeserver(self, reactor, clock):
         hs = self.setup_test_homeserver(
-            "server", federation_http_client=None, federation_sender=Mock()
+            "server",
+            federation_http_client=None,
+            federation_sender=Mock(spec=FederationSender),
         )
         return hs
 
+    def default_config(self):
+        config = super().default_config()
+        config["send_federation"] = True
+        return config
+
     def prepare(self, reactor, clock, hs):
         self.federation_sender = hs.get_federation_sender()
         self.event_builder_factory = hs.get_event_builder_factory()
@@ -498,7 +893,7 @@ def prepare(self, reactor, clock, hs):
 
         self.store = hs.get_datastore()
         self.state = hs.get_state_handler()
-        self.auth = hs.get_auth()
+        self._event_auth_handler = hs.get_event_auth_handler()
 
         # We don't actually check signatures in tests, so lets just create a
         # random key to use.
@@ -530,9 +925,6 @@ def test_remote_joins(self):
         # Add a new remote server to the room
         self._add_new_user(room_id, "@alice:server2")
 
-        # We shouldn't have sent out any local presence *updates*
-        self.federation_sender.send_presence.assert_not_called()
-
         # When new server is joined we send it the local users presence states.
         # We expect to only see user @test2:server, as @test:server is offline
         # and has a zero last_active_ts
@@ -541,7 +933,7 @@ def test_remote_joins(self):
         )
         self.assertEqual(expected_state.state, PresenceState.ONLINE)
         self.federation_sender.send_presence_to_destinations.assert_called_once_with(
-            destinations=["server2"], states={expected_state}
+            destinations={"server2"}, states=[expected_state]
         )
 
         #
@@ -551,9 +943,8 @@ def test_remote_joins(self):
         self.federation_sender.reset_mock()
         self._add_new_user(room_id, "@bob:server3")
 
-        self.federation_sender.send_presence.assert_not_called()
         self.federation_sender.send_presence_to_destinations.assert_called_once_with(
-            destinations=["server3"], states={expected_state}
+            destinations={"server3"}, states=[expected_state]
         )
 
     def test_remote_gets_presence_when_local_user_joins(self):
@@ -596,22 +987,13 @@ def test_remote_gets_presence_when_local_user_joins(self):
 
         self.reactor.pump([0])  # Wait for presence updates to be handled
 
-        # We shouldn't have sent out any local presence *updates*
-        self.federation_sender.send_presence.assert_not_called()
-
         # We expect to only send test2 presence to server2 and server3
         expected_state = self.get_success(
             self.presence_handler.current_state_for_user("@test2:server")
         )
         self.assertEqual(expected_state.state, PresenceState.ONLINE)
-        self.assertEqual(
-            self.federation_sender.send_presence_to_destinations.call_count, 2
-        )
-        self.federation_sender.send_presence_to_destinations.assert_any_call(
-            destinations=["server3"], states={expected_state}
-        )
-        self.federation_sender.send_presence_to_destinations.assert_any_call(
-            destinations=["server2"], states={expected_state}
+        self.federation_sender.send_presence_to_destinations.assert_called_once_with(
+            destinations={"server2", "server3"}, states=[expected_state]
         )
 
     def _add_new_user(self, room_id, user_id):
@@ -623,7 +1005,7 @@ def _add_new_user(self, room_id, user_id):
 
         builder = EventBuilder(
             state=self.state,
-            auth=self.auth,
+            event_auth_handler=self._event_auth_handler,
             store=self.store,
             clock=self.clock,
             hostname=hostname,
@@ -640,7 +1022,9 @@ def _add_new_user(self, room_id, user_id):
             self.store.get_latest_event_ids_in_room(room_id)
         )
 
-        event = self.get_success(builder.build(prev_event_ids, None))
+        event = self.get_success(
+            builder.build(prev_event_ids=prev_event_ids, auth_event_ids=None)
+        )
 
         self.get_success(self.federation_handler.on_receive_pdu(hostname, event))
 
diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py
index cbbe7280c7..6ff1248a9c 100644
--- a/tests/handlers/test_profile.py
+++ b/tests/handlers/test_profile.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 import synapse.types
 from synapse.api.errors import AuthError, SynapseError
@@ -24,7 +23,7 @@
 
 
 class ProfileTestCase(unittest.HomeserverTestCase):
-    """ Tests profile management. """
+    """Tests profile management."""
 
     def make_homeserver(self, reactor, clock):
         self.mock_federation = Mock()
@@ -104,7 +103,7 @@ def test_set_my_name(self):
         )
 
         self.assertIsNone(
-            (self.get_success(self.store.get_profile_displayname(self.frank.localpart)))
+            self.get_success(self.store.get_profile_displayname(self.frank.localpart))
         )
 
     def test_set_my_name_if_disabled(self):
diff --git a/tests/handlers/test_receipts.py b/tests/handlers/test_receipts.py
new file mode 100644
index 0000000000..732a12c9bd
--- /dev/null
+++ b/tests/handlers/test_receipts.py
@@ -0,0 +1,317 @@
+# Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import List
+
+from synapse.api.constants import ReadReceiptEventFields
+from synapse.types import JsonDict
+
+from tests import unittest
+
+
+class ReceiptsTestCase(unittest.HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
+        self.event_source = hs.get_event_sources().sources["receipt"]
+
+    # In the first param of _test_filters_hidden we use "hidden" instead of
+    # ReadReceiptEventFields.MSC2285_HIDDEN. We do this because we're mocking
+    # the data from the database which doesn't use the prefix
+
+    def test_filters_out_hidden_receipt(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@rikj:jki.re": {
+                                    "ts": 1436451550453,
+                                    "hidden": True,
+                                }
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [],
+        )
+
+    def test_does_not_filter_out_our_hidden_receipt(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$1435641916hfgh4394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@me:server.org": {
+                                    "ts": 1436451550453,
+                                    "hidden": True,
+                                },
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [
+                {
+                    "content": {
+                        "$1435641916hfgh4394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@me:server.org": {
+                                    "ts": 1436451550453,
+                                    ReadReceiptEventFields.MSC2285_HIDDEN: True,
+                                },
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_filters_out_hidden_receipt_and_ignores_rest(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$1dgdgrd5641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@rikj:jki.re": {
+                                    "ts": 1436451550453,
+                                    "hidden": True,
+                                },
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                },
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [
+                {
+                    "content": {
+                        "$1dgdgrd5641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_filters_out_event_with_only_hidden_receipts_and_ignores_the_rest(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$14356419edgd14394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@rikj:jki.re": {
+                                    "ts": 1436451550453,
+                                    "hidden": True,
+                                },
+                            }
+                        },
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [
+                {
+                    "content": {
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_handles_missing_content_of_m_read(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$14356419ggffg114394fHBLK:matrix.org": {"m.read": {}},
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [
+                {
+                    "content": {
+                        "$14356419ggffg114394fHBLK:matrix.org": {"m.read": {}},
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_handles_empty_event(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$143564gdfg6114394fHBLK:matrix.org": {},
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+            [
+                {
+                    "content": {
+                        "$143564gdfg6114394fHBLK:matrix.org": {},
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_filters_out_receipt_event_with_only_hidden_receipt_and_ignores_rest(self):
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$14356419edgd14394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@rikj:jki.re": {
+                                    "ts": 1436451550453,
+                                    "hidden": True,
+                                },
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                },
+                {
+                    "content": {
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                },
+            ],
+            [
+                {
+                    "content": {
+                        "$1435641916114394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@user:jki.re": {
+                                    "ts": 1436451550453,
+                                }
+                            }
+                        }
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                }
+            ],
+        )
+
+    def test_handles_string_data(self):
+        """
+        Tests that an invalid shape for read-receipts is handled.
+        Context: https://github.com/matrix-org/synapse/issues/10603
+        """
+
+        self._test_filters_hidden(
+            [
+                {
+                    "content": {
+                        "$14356419edgd14394fHBLK:matrix.org": {
+                            "m.read": {
+                                "@rikj:jki.re": "string",
+                            }
+                        },
+                    },
+                    "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+                    "type": "m.receipt",
+                },
+            ],
+            [],
+        )
+
+    def _test_filters_hidden(
+        self, events: List[JsonDict], expected_output: List[JsonDict]
+    ):
+        """Tests that the _filter_out_hidden returns the expected output"""
+        filtered_events = self.event_source.filter_out_hidden(events, "@me:server.org")
+        self.assertEquals(filtered_events, expected_output)
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 00a0bc5274..6b993defd8 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,17 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.api.auth import Auth
 from synapse.api.constants import UserTypes
 from synapse.api.errors import Codes, ResourceLimitError, SynapseError
-from synapse.rest.client.v2_alpha.register import (
-    _map_email_to_displayname,
-    register_servlets,
-)
+from synapse.events.spamcheck import load_legacy_spam_checkers
+from synapse.rest.client.register import _map_email_to_displayname, register_servlets
 from synapse.spam_checker_api import RegistrationBehaviour
-from synapse.types import RoomAlias, UserID, create_requester
+from synapse.types import RoomAlias, RoomID, UserID, create_requester
 
 from tests.server import FakeChannel
 from tests.test_utils import make_awaitable
@@ -33,8 +30,93 @@
 from .. import unittest
 
 
+class TestSpamChecker:
+    def __init__(self, config, api):
+        api.register_spam_checker_callbacks(
+            check_registration_for_spam=self.check_registration_for_spam,
+        )
+
+    @staticmethod
+    def parse_config(config):
+        return config
+
+    async def check_registration_for_spam(
+        self,
+        email_threepid,
+        username,
+        request_info,
+        auth_provider_id,
+    ):
+        pass
+
+
+class DenyAll(TestSpamChecker):
+    async def check_registration_for_spam(
+        self,
+        email_threepid,
+        username,
+        request_info,
+        auth_provider_id,
+    ):
+        return RegistrationBehaviour.DENY
+
+
+class BanAll(TestSpamChecker):
+    async def check_registration_for_spam(
+        self,
+        email_threepid,
+        username,
+        request_info,
+        auth_provider_id,
+    ):
+        return RegistrationBehaviour.SHADOW_BAN
+
+
+class BanBadIdPUser(TestSpamChecker):
+    async def check_registration_for_spam(
+        self, email_threepid, username, request_info, auth_provider_id=None
+    ):
+        # Reject any user coming from CAS and whose username contains profanity
+        if auth_provider_id == "cas" and "flimflob" in username:
+            return RegistrationBehaviour.DENY
+        return RegistrationBehaviour.ALLOW
+
+
+class TestLegacyRegistrationSpamChecker:
+    def __init__(self, config, api):
+        pass
+
+    async def check_registration_for_spam(
+        self,
+        email_threepid,
+        username,
+        request_info,
+    ):
+        pass
+
+
+class LegacyAllowAll(TestLegacyRegistrationSpamChecker):
+    async def check_registration_for_spam(
+        self,
+        email_threepid,
+        username,
+        request_info,
+    ):
+        return RegistrationBehaviour.ALLOW
+
+
+class LegacyDenyAll(TestLegacyRegistrationSpamChecker):
+    async def check_registration_for_spam(
+        self,
+        email_threepid,
+        username,
+        request_info,
+    ):
+        return RegistrationBehaviour.DENY
+
+
 class RegistrationTestCase(unittest.HomeserverTestCase):
-    """ Tests the RegistrationHandler. """
+    """Tests the RegistrationHandler."""
 
     servlets = [
         register_servlets,
@@ -52,16 +134,19 @@ def make_homeserver(self, reactor, clock):
         hs_config["limit_usage_by_mau"] = True
 
         hs = self.setup_test_homeserver(config=hs_config)
+
+        load_legacy_spam_checkers(hs)
+
+        module_api = hs.get_module_api()
+        for module, config in hs.config.modules.loaded_modules:
+            module(config=config, api=module_api)
+
         return hs
 
     def prepare(self, reactor, clock, hs):
         self.mock_distributor = Mock()
         self.mock_distributor.declare("registered_user")
         self.mock_captcha_client = Mock()
-        self.macaroon_generator = Mock(
-            generate_access_token=Mock(return_value="secret")
-        )
-        self.hs.get_macaroon_generator = Mock(return_value=self.macaroon_generator)
         self.handler = self.hs.get_registration_handler()
         self.store = self.hs.get_datastore()
         self.lots_of_users = 100
@@ -77,8 +162,8 @@ def test_user_is_created_and_logged_in_if_doesnt_exist(self):
             self.get_or_create_user(requester, frank.localpart, "Frankie")
         )
         self.assertEquals(result_user_id, user_id)
-        self.assertTrue(result_token is not None)
-        self.assertEquals(result_token, "secret")
+        self.assertIsInstance(result_token, str)
+        self.assertGreater(len(result_token), 20)
 
     def test_if_user_exists(self):
         store = self.hs.get_datastore()
@@ -479,38 +564,74 @@ def test_invalid_user_id_length(self):
             self.handler.register_user(localpart=invalid_user_id), SynapseError
         )
 
+    @override_config(
+        {
+            "modules": [
+                {
+                    "module": TestSpamChecker.__module__ + ".DenyAll",
+                }
+            ]
+        }
+    )
     def test_spam_checker_deny(self):
         """A spam checker can deny registration, which results in an error."""
+        self.get_failure(self.handler.register_user(localpart="user"), SynapseError)
 
-        class DenyAll:
-            def check_registration_for_spam(
-                self, email_threepid, username, request_info
-            ):
-                return RegistrationBehaviour.DENY
-
-        # Configure a spam checker that denies all users.
-        spam_checker = self.hs.get_spam_checker()
-        spam_checker.spam_checkers = [DenyAll()]
+    @override_config(
+        {
+            "spam_checker": [
+                {
+                    "module": TestSpamChecker.__module__ + ".LegacyAllowAll",
+                }
+            ]
+        }
+    )
+    def test_spam_checker_legacy_allow(self):
+        """Tests that a legacy spam checker implementing the legacy 3-arg version of the
+        check_registration_for_spam callback is correctly called.
+
+        In this test and the following one we test both success and failure to make sure
+        any failure comes from the spam checker (and not something else failing in the
+        call stack) and any success comes from the spam checker (and not because a
+        misconfiguration prevented it from being loaded).
+        """
+        self.get_success(self.handler.register_user(localpart="user"))
 
+    @override_config(
+        {
+            "spam_checker": [
+                {
+                    "module": TestSpamChecker.__module__ + ".LegacyDenyAll",
+                }
+            ]
+        }
+    )
+    def test_spam_checker_legacy_deny(self):
+        """Tests that a legacy spam checker implementing the legacy 3-arg version of the
+        check_registration_for_spam callback is correctly called.
+
+        In this test and the previous one we test both success and failure to make sure
+        any failure comes from the spam checker (and not something else failing in the
+        call stack) and any success comes from the spam checker (and not because a
+        misconfiguration prevented it from being loaded).
+        """
         self.get_failure(self.handler.register_user(localpart="user"), SynapseError)
 
+    @override_config(
+        {
+            "modules": [
+                {
+                    "module": TestSpamChecker.__module__ + ".BanAll",
+                }
+            ]
+        }
+    )
     def test_spam_checker_shadow_ban(self):
         """A spam checker can choose to shadow-ban a user, which allows registration to succeed."""
-
-        class BanAll:
-            def check_registration_for_spam(
-                self, email_threepid, username, request_info
-            ):
-                return RegistrationBehaviour.SHADOW_BAN
-
-        # Configure a spam checker that denies all users.
-        spam_checker = self.hs.get_spam_checker()
-        spam_checker.spam_checkers = [BanAll()]
-
         user_id = self.get_success(self.handler.register_user(localpart="user"))
 
         # Get an access token.
-        token = self.macaroon_generator.generate_access_token(user_id)
+        token = "testtok"
         self.get_success(
             self.store.add_access_token_to_user(
                 user_id=user_id, token=token, device_id=None, valid_until_ms=None
@@ -526,22 +647,17 @@ def check_registration_for_spam(
 
         self.assertTrue(requester.shadow_banned)
 
+    @override_config(
+        {
+            "modules": [
+                {
+                    "module": TestSpamChecker.__module__ + ".BanBadIdPUser",
+                }
+            ]
+        }
+    )
     def test_spam_checker_receives_sso_type(self):
         """Test rejecting registration based on SSO type"""
-
-        class BanBadIdPUser:
-            def check_registration_for_spam(
-                self, email_threepid, username, request_info, auth_provider_id=None
-            ):
-                # Reject any user coming from CAS and whose username contains profanity
-                if auth_provider_id == "cas" and "flimflob" in username:
-                    return RegistrationBehaviour.DENY
-                return RegistrationBehaviour.ALLOW
-
-        # Configure a spam checker that denies a certain user on a specific IdP
-        spam_checker = self.hs.get_spam_checker()
-        spam_checker.spam_checkers = [BanBadIdPUser()]
-
         f = self.get_failure(
             self.handler.register_user(localpart="bobflimflob", auth_provider_id="cas"),
             SynapseError,
@@ -684,7 +800,7 @@ async def get_or_create_user(
 
         user = UserID(localpart, self.hs.hostname)
         user_id = user.to_string()
-        token = self.macaroon_generator.generate_access_token(user_id)
+        token = self.hs.get_auth_handler().generate_access_token(user)
 
         if need_register:
             await self.handler.register_with_store(
@@ -706,3 +822,50 @@ async def get_or_create_user(
             )
 
         return user_id, token
+
+
+class RemoteAutoJoinTestCase(unittest.HomeserverTestCase):
+    """Tests auto-join on remote rooms."""
+
+    def make_homeserver(self, reactor, clock):
+        self.room_id = "!roomid:remotetest"
+
+        async def update_membership(*args, **kwargs):
+            pass
+
+        async def lookup_room_alias(*args, **kwargs):
+            return RoomID.from_string(self.room_id), ["remotetest"]
+
+        self.room_member_handler = Mock(spec=["update_membership", "lookup_room_alias"])
+        self.room_member_handler.update_membership.side_effect = update_membership
+        self.room_member_handler.lookup_room_alias.side_effect = lookup_room_alias
+
+        hs = self.setup_test_homeserver(room_member_handler=self.room_member_handler)
+        return hs
+
+    def prepare(self, reactor, clock, hs):
+        self.handler = self.hs.get_registration_handler()
+        self.store = self.hs.get_datastore()
+
+    @override_config({"auto_join_rooms": ["#room:remotetest"]})
+    def test_auto_create_auto_join_remote_room(self):
+        """Tests that we don't attempt to create remote rooms, and that we don't attempt
+        to invite ourselves to rooms we're not in."""
+
+        # Register a first user; this should call _create_and_join_rooms
+        self.get_success(self.handler.register_user(localpart="jeff"))
+
+        _, kwargs = self.room_member_handler.update_membership.call_args
+
+        self.assertEqual(kwargs["room_id"], self.room_id)
+        self.assertEqual(kwargs["action"], "join")
+        self.assertEqual(kwargs["remote_room_hosts"], ["remotetest"])
+
+        # Register a second user; this should call _join_rooms
+        self.get_success(self.handler.register_user(localpart="jeff2"))
+
+        _, kwargs = self.room_member_handler.update_membership.call_args
+
+        self.assertEqual(kwargs["room_id"], self.room_id)
+        self.assertEqual(kwargs["action"], "join")
+        self.assertEqual(kwargs["remote_room_hosts"], ["remotetest"])
diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py
new file mode 100644
index 0000000000..ac800afa7d
--- /dev/null
+++ b/tests/handlers/test_room_summary.py
@@ -0,0 +1,959 @@
+#  Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import Any, Iterable, List, Optional, Tuple
+from unittest import mock
+
+from synapse.api.constants import (
+    EventContentFields,
+    EventTypes,
+    HistoryVisibility,
+    JoinRules,
+    Membership,
+    RestrictedJoinRuleTypes,
+    RoomTypes,
+)
+from synapse.api.errors import AuthError, NotFoundError, SynapseError
+from synapse.api.room_versions import RoomVersions
+from synapse.events import make_event_from_dict
+from synapse.handlers.room_summary import _child_events_comparison_key, _RoomEntry
+from synapse.rest import admin
+from synapse.rest.client import login, room
+from synapse.server import HomeServer
+from synapse.types import JsonDict, UserID
+
+from tests import unittest
+
+
+def _create_event(room_id: str, order: Optional[Any] = None):
+    result = mock.Mock()
+    result.room_id = room_id
+    result.content = {}
+    if order is not None:
+        result.content["order"] = order
+    return result
+
+
+def _order(*events):
+    return sorted(events, key=_child_events_comparison_key)
+
+
+class TestSpaceSummarySort(unittest.TestCase):
+    def test_no_order_last(self):
+        """An event with no ordering is placed behind those with an ordering."""
+        ev1 = _create_event("!abc:test")
+        ev2 = _create_event("!xyz:test", "xyz")
+
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+    def test_order(self):
+        """The ordering should be used."""
+        ev1 = _create_event("!abc:test", "xyz")
+        ev2 = _create_event("!xyz:test", "abc")
+
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+    def test_order_room_id(self):
+        """Room ID is a tie-breaker for ordering."""
+        ev1 = _create_event("!abc:test", "abc")
+        ev2 = _create_event("!xyz:test", "abc")
+
+        self.assertEqual([ev1, ev2], _order(ev1, ev2))
+
+    def test_invalid_ordering_type(self):
+        """Invalid orderings are considered the same as missing."""
+        ev1 = _create_event("!abc:test", 1)
+        ev2 = _create_event("!xyz:test", "xyz")
+
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+        ev1 = _create_event("!abc:test", {})
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+        ev1 = _create_event("!abc:test", [])
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+        ev1 = _create_event("!abc:test", True)
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+    def test_invalid_ordering_value(self):
+        """Invalid orderings are considered the same as missing."""
+        ev1 = _create_event("!abc:test", "foo\n")
+        ev2 = _create_event("!xyz:test", "xyz")
+
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+        ev1 = _create_event("!abc:test", "a" * 51)
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+
+class SpaceSummaryTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        admin.register_servlets_for_client_rest_resource,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs: HomeServer):
+        self.hs = hs
+        self.handler = self.hs.get_room_summary_handler()
+
+        # Create a user.
+        self.user = self.register_user("user", "pass")
+        self.token = self.login("user", "pass")
+
+        # Create a space and a child room.
+        self.space = self.helper.create_room_as(
+            self.user,
+            tok=self.token,
+            extra_content={
+                "creation_content": {EventContentFields.ROOM_TYPE: RoomTypes.SPACE}
+            },
+        )
+        self.room = self.helper.create_room_as(self.user, tok=self.token)
+        self._add_child(self.space, self.room, self.token)
+
+    def _add_child(
+        self, space_id: str, room_id: str, token: str, order: Optional[str] = None
+    ) -> None:
+        """Add a child room to a space."""
+        content: JsonDict = {"via": [self.hs.hostname]}
+        if order is not None:
+            content["order"] = order
+        self.helper.send_state(
+            space_id,
+            event_type=EventTypes.SpaceChild,
+            body=content,
+            tok=token,
+            state_key=room_id,
+        )
+
+    def _assert_rooms(
+        self, result: JsonDict, rooms_and_children: Iterable[Tuple[str, Iterable[str]]]
+    ) -> None:
+        """
+        Assert that the expected room IDs and events are in the response.
+
+        Args:
+            result: The result from the API call.
+            rooms_and_children: An iterable of tuples where each tuple is:
+                The expected room ID.
+                The expected IDs of any children rooms.
+        """
+        room_ids = []
+        children_ids = []
+        for room_id, children in rooms_and_children:
+            room_ids.append(room_id)
+            if children:
+                children_ids.extend([(room_id, child_id) for child_id in children])
+        self.assertCountEqual(
+            [room.get("room_id") for room in result["rooms"]], room_ids
+        )
+        self.assertCountEqual(
+            [
+                (event.get("room_id"), event.get("state_key"))
+                for event in result["events"]
+            ],
+            children_ids,
+        )
+
+    def _assert_hierarchy(
+        self, result: JsonDict, rooms_and_children: Iterable[Tuple[str, Iterable[str]]]
+    ) -> None:
+        """
+        Assert that the expected room IDs are in the response.
+
+        Args:
+            result: The result from the API call.
+            rooms_and_children: An iterable of tuples where each tuple is:
+                The expected room ID.
+                The expected IDs of any children rooms.
+        """
+        result_room_ids = []
+        result_children_ids = []
+        for result_room in result["rooms"]:
+            result_room_ids.append(result_room["room_id"])
+            result_children_ids.append(
+                [
+                    (cs["room_id"], cs["state_key"])
+                    for cs in result_room.get("children_state")
+                ]
+            )
+
+        room_ids = []
+        children_ids = []
+        for room_id, children in rooms_and_children:
+            room_ids.append(room_id)
+            children_ids.append([(room_id, child_id) for child_id in children])
+
+        # Note that order matters.
+        self.assertEqual(result_room_ids, room_ids)
+        self.assertEqual(result_children_ids, children_ids)
+
+    def _poke_fed_invite(self, room_id: str, from_user: str) -> None:
+        """
+        Creates a invite (as if received over federation) for the room from the
+        given hostname.
+
+        Args:
+            room_id: The room ID to issue an invite for.
+            fed_hostname: The user to invite from.
+        """
+        # Poke an invite over federation into the database.
+        fed_handler = self.hs.get_federation_handler()
+        fed_hostname = UserID.from_string(from_user).domain
+        event = make_event_from_dict(
+            {
+                "room_id": room_id,
+                "event_id": "!abcd:" + fed_hostname,
+                "type": EventTypes.Member,
+                "sender": from_user,
+                "state_key": self.user,
+                "content": {"membership": Membership.INVITE},
+                "prev_events": [],
+                "auth_events": [],
+                "depth": 1,
+                "origin_server_ts": 1234,
+            }
+        )
+        self.get_success(
+            fed_handler.on_invite_request(fed_hostname, event, RoomVersions.V6)
+        )
+
+    def test_simple_space(self):
+        """Test a simple space with a single room."""
+        result = self.get_success(self.handler.get_space_summary(self.user, self.space))
+        # The result should have the space and the room in it, along with a link
+        # from space -> room.
+        expected = [(self.space, [self.room]), (self.room, ())]
+        self._assert_rooms(result, expected)
+
+        result = self.get_success(
+            self.handler.get_room_hierarchy(self.user, self.space)
+        )
+        self._assert_hierarchy(result, expected)
+
+    def test_visibility(self):
+        """A user not in a space cannot inspect it."""
+        user2 = self.register_user("user2", "pass")
+        token2 = self.login("user2", "pass")
+
+        # The user can see the space since it is publicly joinable.
+        result = self.get_success(self.handler.get_space_summary(user2, self.space))
+        expected = [(self.space, [self.room]), (self.room, ())]
+        self._assert_rooms(result, expected)
+
+        result = self.get_success(self.handler.get_room_hierarchy(user2, self.space))
+        self._assert_hierarchy(result, expected)
+
+        # If the space is made invite-only, it should no longer be viewable.
+        self.helper.send_state(
+            self.space,
+            event_type=EventTypes.JoinRules,
+            body={"join_rule": JoinRules.INVITE},
+            tok=self.token,
+        )
+        self.get_failure(self.handler.get_space_summary(user2, self.space), AuthError)
+        self.get_failure(self.handler.get_room_hierarchy(user2, self.space), AuthError)
+
+        # If the space is made world-readable it should return a result.
+        self.helper.send_state(
+            self.space,
+            event_type=EventTypes.RoomHistoryVisibility,
+            body={"history_visibility": HistoryVisibility.WORLD_READABLE},
+            tok=self.token,
+        )
+        result = self.get_success(self.handler.get_space_summary(user2, self.space))
+        self._assert_rooms(result, expected)
+
+        result = self.get_success(self.handler.get_room_hierarchy(user2, self.space))
+        self._assert_hierarchy(result, expected)
+
+        # Make it not world-readable again and confirm it results in an error.
+        self.helper.send_state(
+            self.space,
+            event_type=EventTypes.RoomHistoryVisibility,
+            body={"history_visibility": HistoryVisibility.JOINED},
+            tok=self.token,
+        )
+        self.get_failure(self.handler.get_space_summary(user2, self.space), AuthError)
+        self.get_failure(self.handler.get_room_hierarchy(user2, self.space), AuthError)
+
+        # Join the space and results should be returned.
+        self.helper.invite(self.space, targ=user2, tok=self.token)
+        self.helper.join(self.space, user2, tok=token2)
+        result = self.get_success(self.handler.get_space_summary(user2, self.space))
+        self._assert_rooms(result, expected)
+
+        result = self.get_success(self.handler.get_room_hierarchy(user2, self.space))
+        self._assert_hierarchy(result, expected)
+
+        # Attempting to view an unknown room returns the same error.
+        self.get_failure(
+            self.handler.get_space_summary(user2, "#not-a-space:" + self.hs.hostname),
+            AuthError,
+        )
+        self.get_failure(
+            self.handler.get_room_hierarchy(user2, "#not-a-space:" + self.hs.hostname),
+            AuthError,
+        )
+
+    def _create_room_with_join_rule(
+        self, join_rule: str, room_version: Optional[str] = None, **extra_content
+    ) -> str:
+        """Create a room with the given join rule and add it to the space."""
+        room_id = self.helper.create_room_as(
+            self.user,
+            room_version=room_version,
+            tok=self.token,
+            extra_content={
+                "initial_state": [
+                    {
+                        "type": EventTypes.JoinRules,
+                        "state_key": "",
+                        "content": {
+                            "join_rule": join_rule,
+                            **extra_content,
+                        },
+                    }
+                ]
+            },
+        )
+        self._add_child(self.space, room_id, self.token)
+        return room_id
+
+    def test_filtering(self):
+        """
+        Rooms should be properly filtered to only include rooms the user has access to.
+        """
+        user2 = self.register_user("user2", "pass")
+        token2 = self.login("user2", "pass")
+
+        # Create a few rooms which will have different properties.
+        public_room = self._create_room_with_join_rule(JoinRules.PUBLIC)
+        knock_room = self._create_room_with_join_rule(
+            JoinRules.KNOCK, room_version=RoomVersions.V7.identifier
+        )
+        not_invited_room = self._create_room_with_join_rule(JoinRules.INVITE)
+        invited_room = self._create_room_with_join_rule(JoinRules.INVITE)
+        self.helper.invite(invited_room, targ=user2, tok=self.token)
+        restricted_room = self._create_room_with_join_rule(
+            JoinRules.RESTRICTED,
+            room_version=RoomVersions.V8.identifier,
+            allow=[],
+        )
+        restricted_accessible_room = self._create_room_with_join_rule(
+            JoinRules.RESTRICTED,
+            room_version=RoomVersions.V8.identifier,
+            allow=[
+                {
+                    "type": RestrictedJoinRuleTypes.ROOM_MEMBERSHIP,
+                    "room_id": self.space,
+                    "via": [self.hs.hostname],
+                }
+            ],
+        )
+        world_readable_room = self._create_room_with_join_rule(JoinRules.INVITE)
+        self.helper.send_state(
+            world_readable_room,
+            event_type=EventTypes.RoomHistoryVisibility,
+            body={"history_visibility": HistoryVisibility.WORLD_READABLE},
+            tok=self.token,
+        )
+        joined_room = self._create_room_with_join_rule(JoinRules.INVITE)
+        self.helper.invite(joined_room, targ=user2, tok=self.token)
+        self.helper.join(joined_room, user2, tok=token2)
+
+        # Join the space.
+        self.helper.join(self.space, user2, tok=token2)
+        result = self.get_success(self.handler.get_space_summary(user2, self.space))
+        expected = [
+            (
+                self.space,
+                [
+                    self.room,
+                    public_room,
+                    knock_room,
+                    not_invited_room,
+                    invited_room,
+                    restricted_room,
+                    restricted_accessible_room,
+                    world_readable_room,
+                    joined_room,
+                ],
+            ),
+            (self.room, ()),
+            (public_room, ()),
+            (knock_room, ()),
+            (invited_room, ()),
+            (restricted_accessible_room, ()),
+            (world_readable_room, ()),
+            (joined_room, ()),
+        ]
+        self._assert_rooms(result, expected)
+
+        result = self.get_success(self.handler.get_room_hierarchy(user2, self.space))
+        self._assert_hierarchy(result, expected)
+
+    def test_complex_space(self):
+        """
+        Create a "complex" space to see how it handles things like loops and subspaces.
+        """
+        # Create an inaccessible room.
+        user2 = self.register_user("user2", "pass")
+        token2 = self.login("user2", "pass")
+        room2 = self.helper.create_room_as(user2, is_public=False, tok=token2)
+        # This is a bit odd as "user" is adding a room they don't know about, but
+        # it works for the tests.
+        self._add_child(self.space, room2, self.token)
+
+        # Create a subspace under the space with an additional room in it.
+        subspace = self.helper.create_room_as(
+            self.user,
+            tok=self.token,
+            extra_content={
+                "creation_content": {EventContentFields.ROOM_TYPE: RoomTypes.SPACE}
+            },
+        )
+        subroom = self.helper.create_room_as(self.user, tok=self.token)
+        self._add_child(self.space, subspace, token=self.token)
+        self._add_child(subspace, subroom, token=self.token)
+        # Also add the two rooms from the space into this subspace (causing loops).
+        self._add_child(subspace, self.room, token=self.token)
+        self._add_child(subspace, room2, self.token)
+
+        result = self.get_success(self.handler.get_space_summary(self.user, self.space))
+
+        # The result should include each room a single time and each link.
+        expected = [
+            (self.space, [self.room, room2, subspace]),
+            (self.room, ()),
+            (subspace, [subroom, self.room, room2]),
+            (subroom, ()),
+        ]
+        self._assert_rooms(result, expected)
+
+        result = self.get_success(
+            self.handler.get_room_hierarchy(self.user, self.space)
+        )
+        self._assert_hierarchy(result, expected)
+
+    def test_pagination(self):
+        """Test simple pagination works."""
+        room_ids = []
+        for i in range(1, 10):
+            room = self.helper.create_room_as(self.user, tok=self.token)
+            self._add_child(self.space, room, self.token, order=str(i))
+            room_ids.append(room)
+        # The room created initially doesn't have an order, so comes last.
+        room_ids.append(self.room)
+
+        result = self.get_success(
+            self.handler.get_room_hierarchy(self.user, self.space, limit=7)
+        )
+        # The result should have the space and all of the links, plus some of the
+        # rooms and a pagination token.
+        expected: List[Tuple[str, Iterable[str]]] = [(self.space, room_ids)]
+        expected += [(room_id, ()) for room_id in room_ids[:6]]
+        self._assert_hierarchy(result, expected)
+        self.assertIn("next_batch", result)
+
+        # Check the next page.
+        result = self.get_success(
+            self.handler.get_room_hierarchy(
+                self.user, self.space, limit=5, from_token=result["next_batch"]
+            )
+        )
+        # The result should have the space and the room in it, along with a link
+        # from space -> room.
+        expected = [(room_id, ()) for room_id in room_ids[6:]]
+        self._assert_hierarchy(result, expected)
+        self.assertNotIn("next_batch", result)
+
+    def test_invalid_pagination_token(self):
+        """An invalid pagination token, or changing other parameters, shoudl be rejected."""
+        room_ids = []
+        for i in range(1, 10):
+            room = self.helper.create_room_as(self.user, tok=self.token)
+            self._add_child(self.space, room, self.token, order=str(i))
+            room_ids.append(room)
+        # The room created initially doesn't have an order, so comes last.
+        room_ids.append(self.room)
+
+        result = self.get_success(
+            self.handler.get_room_hierarchy(self.user, self.space, limit=7)
+        )
+        self.assertIn("next_batch", result)
+
+        # Changing the room ID, suggested-only, or max-depth causes an error.
+        self.get_failure(
+            self.handler.get_room_hierarchy(
+                self.user, self.room, from_token=result["next_batch"]
+            ),
+            SynapseError,
+        )
+        self.get_failure(
+            self.handler.get_room_hierarchy(
+                self.user,
+                self.space,
+                suggested_only=True,
+                from_token=result["next_batch"],
+            ),
+            SynapseError,
+        )
+        self.get_failure(
+            self.handler.get_room_hierarchy(
+                self.user, self.space, max_depth=0, from_token=result["next_batch"]
+            ),
+            SynapseError,
+        )
+
+        # An invalid token is ignored.
+        self.get_failure(
+            self.handler.get_room_hierarchy(self.user, self.space, from_token="foo"),
+            SynapseError,
+        )
+
+    def test_max_depth(self):
+        """Create a deep tree to test the max depth against."""
+        spaces = [self.space]
+        rooms = [self.room]
+        for _ in range(5):
+            spaces.append(
+                self.helper.create_room_as(
+                    self.user,
+                    tok=self.token,
+                    extra_content={
+                        "creation_content": {
+                            EventContentFields.ROOM_TYPE: RoomTypes.SPACE
+                        }
+                    },
+                )
+            )
+            self._add_child(spaces[-2], spaces[-1], self.token)
+            rooms.append(self.helper.create_room_as(self.user, tok=self.token))
+            self._add_child(spaces[-1], rooms[-1], self.token)
+
+        # Test just the space itself.
+        result = self.get_success(
+            self.handler.get_room_hierarchy(self.user, self.space, max_depth=0)
+        )
+        expected: List[Tuple[str, Iterable[str]]] = [(spaces[0], [rooms[0], spaces[1]])]
+        self._assert_hierarchy(result, expected)
+
+        # A single additional layer.
+        result = self.get_success(
+            self.handler.get_room_hierarchy(self.user, self.space, max_depth=1)
+        )
+        expected += [
+            (rooms[0], ()),
+            (spaces[1], [rooms[1], spaces[2]]),
+        ]
+        self._assert_hierarchy(result, expected)
+
+        # A few layers.
+        result = self.get_success(
+            self.handler.get_room_hierarchy(self.user, self.space, max_depth=3)
+        )
+        expected += [
+            (rooms[1], ()),
+            (spaces[2], [rooms[2], spaces[3]]),
+            (rooms[2], ()),
+            (spaces[3], [rooms[3], spaces[4]]),
+        ]
+        self._assert_hierarchy(result, expected)
+
+    def test_fed_complex(self):
+        """
+        Return data over federation and ensure that it is handled properly.
+        """
+        fed_hostname = self.hs.hostname + "2"
+        subspace = "#subspace:" + fed_hostname
+        subroom = "#subroom:" + fed_hostname
+
+        # Generate some good data, and some bad data:
+        #
+        # * Event *back* to the root room.
+        # * Unrelated events / rooms
+        # * Multiple levels of events (in a not-useful order, e.g. grandchild
+        #   events before child events).
+
+        # Note that these entries are brief, but should contain enough info.
+        requested_room_entry = _RoomEntry(
+            subspace,
+            {
+                "room_id": subspace,
+                "world_readable": True,
+                "room_type": RoomTypes.SPACE,
+            },
+            [
+                {
+                    "type": EventTypes.SpaceChild,
+                    "room_id": subspace,
+                    "state_key": subroom,
+                    "content": {"via": [fed_hostname]},
+                }
+            ],
+        )
+        child_room = {
+            "room_id": subroom,
+            "world_readable": True,
+        }
+
+        async def summarize_remote_room(
+            _self, room, suggested_only, max_children, exclude_rooms
+        ):
+            return [
+                requested_room_entry,
+                _RoomEntry(
+                    subroom,
+                    {
+                        "room_id": subroom,
+                        "world_readable": True,
+                    },
+                ),
+            ]
+
+        async def summarize_remote_room_hierarchy(_self, room, suggested_only):
+            return requested_room_entry, {subroom: child_room}, set()
+
+        # Add a room to the space which is on another server.
+        self._add_child(self.space, subspace, self.token)
+
+        with mock.patch(
+            "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room",
+            new=summarize_remote_room,
+        ):
+            result = self.get_success(
+                self.handler.get_space_summary(self.user, self.space)
+            )
+
+        expected = [
+            (self.space, [self.room, subspace]),
+            (self.room, ()),
+            (subspace, [subroom]),
+            (subroom, ()),
+        ]
+        self._assert_rooms(result, expected)
+
+        with mock.patch(
+            "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room_hierarchy",
+            new=summarize_remote_room_hierarchy,
+        ):
+            result = self.get_success(
+                self.handler.get_room_hierarchy(self.user, self.space)
+            )
+        self._assert_hierarchy(result, expected)
+
+    def test_fed_filtering(self):
+        """
+        Rooms returned over federation should be properly filtered to only include
+        rooms the user has access to.
+        """
+        fed_hostname = self.hs.hostname + "2"
+        subspace = "#subspace:" + fed_hostname
+
+        # Create a few rooms which will have different properties.
+        public_room = "#public:" + fed_hostname
+        knock_room = "#knock:" + fed_hostname
+        not_invited_room = "#not_invited:" + fed_hostname
+        invited_room = "#invited:" + fed_hostname
+        restricted_room = "#restricted:" + fed_hostname
+        restricted_accessible_room = "#restricted_accessible:" + fed_hostname
+        world_readable_room = "#world_readable:" + fed_hostname
+        joined_room = self.helper.create_room_as(self.user, tok=self.token)
+
+        # Poke an invite over federation into the database.
+        self._poke_fed_invite(invited_room, "@remote:" + fed_hostname)
+
+        # Note that these entries are brief, but should contain enough info.
+        children_rooms = (
+            (
+                public_room,
+                {
+                    "room_id": public_room,
+                    "world_readable": False,
+                    "join_rules": JoinRules.PUBLIC,
+                },
+            ),
+            (
+                knock_room,
+                {
+                    "room_id": knock_room,
+                    "world_readable": False,
+                    "join_rules": JoinRules.KNOCK,
+                },
+            ),
+            (
+                not_invited_room,
+                {
+                    "room_id": not_invited_room,
+                    "world_readable": False,
+                    "join_rules": JoinRules.INVITE,
+                },
+            ),
+            (
+                invited_room,
+                {
+                    "room_id": invited_room,
+                    "world_readable": False,
+                    "join_rules": JoinRules.INVITE,
+                },
+            ),
+            (
+                restricted_room,
+                {
+                    "room_id": restricted_room,
+                    "world_readable": False,
+                    "join_rules": JoinRules.RESTRICTED,
+                    "allowed_spaces": [],
+                },
+            ),
+            (
+                restricted_accessible_room,
+                {
+                    "room_id": restricted_accessible_room,
+                    "world_readable": False,
+                    "join_rules": JoinRules.RESTRICTED,
+                    "allowed_spaces": [self.room],
+                },
+            ),
+            (
+                world_readable_room,
+                {
+                    "room_id": world_readable_room,
+                    "world_readable": True,
+                    "join_rules": JoinRules.INVITE,
+                },
+            ),
+            (
+                joined_room,
+                {
+                    "room_id": joined_room,
+                    "world_readable": False,
+                    "join_rules": JoinRules.INVITE,
+                },
+            ),
+        )
+
+        subspace_room_entry = _RoomEntry(
+            subspace,
+            {
+                "room_id": subspace,
+                "world_readable": True,
+            },
+            # Place each room in the sub-space.
+            [
+                {
+                    "type": EventTypes.SpaceChild,
+                    "room_id": subspace,
+                    "state_key": room_id,
+                    "content": {"via": [fed_hostname]},
+                }
+                for room_id, _ in children_rooms
+            ],
+        )
+
+        async def summarize_remote_room(
+            _self, room, suggested_only, max_children, exclude_rooms
+        ):
+            return [subspace_room_entry] + [
+                # A copy is made of the room data since the allowed_spaces key
+                # is removed.
+                _RoomEntry(child_room[0], dict(child_room[1]))
+                for child_room in children_rooms
+            ]
+
+        async def summarize_remote_room_hierarchy(_self, room, suggested_only):
+            return subspace_room_entry, dict(children_rooms), set()
+
+        # Add a room to the space which is on another server.
+        self._add_child(self.space, subspace, self.token)
+
+        with mock.patch(
+            "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room",
+            new=summarize_remote_room,
+        ):
+            result = self.get_success(
+                self.handler.get_space_summary(self.user, self.space)
+            )
+
+        expected = [
+            (self.space, [self.room, subspace]),
+            (self.room, ()),
+            (
+                subspace,
+                [
+                    public_room,
+                    knock_room,
+                    not_invited_room,
+                    invited_room,
+                    restricted_room,
+                    restricted_accessible_room,
+                    world_readable_room,
+                    joined_room,
+                ],
+            ),
+            (public_room, ()),
+            (knock_room, ()),
+            (invited_room, ()),
+            (restricted_accessible_room, ()),
+            (world_readable_room, ()),
+            (joined_room, ()),
+        ]
+        self._assert_rooms(result, expected)
+
+        with mock.patch(
+            "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room_hierarchy",
+            new=summarize_remote_room_hierarchy,
+        ):
+            result = self.get_success(
+                self.handler.get_room_hierarchy(self.user, self.space)
+            )
+        self._assert_hierarchy(result, expected)
+
+    def test_fed_invited(self):
+        """
+        A room which the user was invited to should be included in the response.
+
+        This differs from test_fed_filtering in that the room itself is being
+        queried over federation, instead of it being included as a sub-room of
+        a space in the response.
+        """
+        fed_hostname = self.hs.hostname + "2"
+        fed_room = "#subroom:" + fed_hostname
+
+        # Poke an invite over federation into the database.
+        self._poke_fed_invite(fed_room, "@remote:" + fed_hostname)
+
+        fed_room_entry = _RoomEntry(
+            fed_room,
+            {
+                "room_id": fed_room,
+                "world_readable": False,
+                "join_rules": JoinRules.INVITE,
+            },
+        )
+
+        async def summarize_remote_room(
+            _self, room, suggested_only, max_children, exclude_rooms
+        ):
+            return [fed_room_entry]
+
+        async def summarize_remote_room_hierarchy(_self, room, suggested_only):
+            return fed_room_entry, {}, set()
+
+        # Add a room to the space which is on another server.
+        self._add_child(self.space, fed_room, self.token)
+
+        with mock.patch(
+            "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room",
+            new=summarize_remote_room,
+        ):
+            result = self.get_success(
+                self.handler.get_space_summary(self.user, self.space)
+            )
+
+        expected = [
+            (self.space, [self.room, fed_room]),
+            (self.room, ()),
+            (fed_room, ()),
+        ]
+        self._assert_rooms(result, expected)
+
+        with mock.patch(
+            "synapse.handlers.room_summary.RoomSummaryHandler._summarize_remote_room_hierarchy",
+            new=summarize_remote_room_hierarchy,
+        ):
+            result = self.get_success(
+                self.handler.get_room_hierarchy(self.user, self.space)
+            )
+        self._assert_hierarchy(result, expected)
+
+
+class RoomSummaryTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        admin.register_servlets_for_client_rest_resource,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs: HomeServer):
+        self.hs = hs
+        self.handler = self.hs.get_room_summary_handler()
+
+        # Create a user.
+        self.user = self.register_user("user", "pass")
+        self.token = self.login("user", "pass")
+
+        # Create a simple room.
+        self.room = self.helper.create_room_as(self.user, tok=self.token)
+        self.helper.send_state(
+            self.room,
+            event_type=EventTypes.JoinRules,
+            body={"join_rule": JoinRules.INVITE},
+            tok=self.token,
+        )
+
+    def test_own_room(self):
+        """Test a simple room created by the requester."""
+        result = self.get_success(self.handler.get_room_summary(self.user, self.room))
+        self.assertEqual(result.get("room_id"), self.room)
+
+    def test_visibility(self):
+        """A user not in a private room cannot get its summary."""
+        user2 = self.register_user("user2", "pass")
+        token2 = self.login("user2", "pass")
+
+        # The user cannot see the room.
+        self.get_failure(self.handler.get_room_summary(user2, self.room), NotFoundError)
+
+        # If the room is made world-readable it should return a result.
+        self.helper.send_state(
+            self.room,
+            event_type=EventTypes.RoomHistoryVisibility,
+            body={"history_visibility": HistoryVisibility.WORLD_READABLE},
+            tok=self.token,
+        )
+        result = self.get_success(self.handler.get_room_summary(user2, self.room))
+        self.assertEqual(result.get("room_id"), self.room)
+
+        # Make it not world-readable again and confirm it results in an error.
+        self.helper.send_state(
+            self.room,
+            event_type=EventTypes.RoomHistoryVisibility,
+            body={"history_visibility": HistoryVisibility.JOINED},
+            tok=self.token,
+        )
+        self.get_failure(self.handler.get_room_summary(user2, self.room), NotFoundError)
+
+        # If the room is made public it should return a result.
+        self.helper.send_state(
+            self.room,
+            event_type=EventTypes.JoinRules,
+            body={"join_rule": JoinRules.PUBLIC},
+            tok=self.token,
+        )
+        result = self.get_success(self.handler.get_room_summary(user2, self.room))
+        self.assertEqual(result.get("room_id"), self.room)
+
+        # Join the space, make it invite-only again and results should be returned.
+        self.helper.join(self.room, user2, tok=token2)
+        self.helper.send_state(
+            self.room,
+            event_type=EventTypes.JoinRules,
+            body={"join_rule": JoinRules.INVITE},
+            tok=self.token,
+        )
+        result = self.get_success(self.handler.get_room_summary(user2, self.room))
+        self.assertEqual(result.get("room_id"), self.room)
diff --git a/tests/handlers/test_saml.py b/tests/handlers/test_saml.py
index 30efd43b40..8cfc184fef 100644
--- a/tests/handlers/test_saml.py
+++ b/tests/handlers/test_saml.py
@@ -13,8 +13,7 @@
 #  limitations under the License.
 
 from typing import Optional
-
-from mock import Mock
+from unittest.mock import Mock
 
 import attr
 
diff --git a/tests/handlers/test_send_email.py b/tests/handlers/test_send_email.py
new file mode 100644
index 0000000000..6f77b1237c
--- /dev/null
+++ b/tests/handlers/test_send_email.py
@@ -0,0 +1,112 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import List, Tuple
+
+from zope.interface import implementer
+
+from twisted.internet import defer
+from twisted.internet.address import IPv4Address
+from twisted.internet.defer import ensureDeferred
+from twisted.mail import interfaces, smtp
+
+from tests.server import FakeTransport
+from tests.unittest import HomeserverTestCase
+
+
+@implementer(interfaces.IMessageDelivery)
+class _DummyMessageDelivery:
+    def __init__(self):
+        # (recipient, message) tuples
+        self.messages: List[Tuple[smtp.Address, bytes]] = []
+
+    def receivedHeader(self, helo, origin, recipients):
+        return None
+
+    def validateFrom(self, helo, origin):
+        return origin
+
+    def record_message(self, recipient: smtp.Address, message: bytes):
+        self.messages.append((recipient, message))
+
+    def validateTo(self, user: smtp.User):
+        return lambda: _DummyMessage(self, user)
+
+
+@implementer(interfaces.IMessageSMTP)
+class _DummyMessage:
+    """IMessageSMTP implementation which saves the message delivered to it
+    to the _DummyMessageDelivery object.
+    """
+
+    def __init__(self, delivery: _DummyMessageDelivery, user: smtp.User):
+        self._delivery = delivery
+        self._user = user
+        self._buffer: List[bytes] = []
+
+    def lineReceived(self, line):
+        self._buffer.append(line)
+
+    def eomReceived(self):
+        message = b"\n".join(self._buffer) + b"\n"
+        self._delivery.record_message(self._user.dest, message)
+        return defer.succeed(b"saved")
+
+    def connectionLost(self):
+        pass
+
+
+class SendEmailHandlerTestCase(HomeserverTestCase):
+    def test_send_email(self):
+        """Happy-path test that we can send email to a non-TLS server."""
+        h = self.hs.get_send_email_handler()
+        d = ensureDeferred(
+            h.send_email(
+                "foo@bar.com", "test subject", "Tests", "HTML content", "Text content"
+            )
+        )
+        # there should be an attempt to connect to localhost:25
+        self.assertEqual(len(self.reactor.tcpClients), 1)
+        (host, port, client_factory, _timeout, _bindAddress) = self.reactor.tcpClients[
+            0
+        ]
+        self.assertEqual(host, "localhost")
+        self.assertEqual(port, 25)
+
+        # wire it up to an SMTP server
+        message_delivery = _DummyMessageDelivery()
+        server_protocol = smtp.ESMTP()
+        server_protocol.delivery = message_delivery
+        # make sure that the server uses the test reactor to set timeouts
+        server_protocol.callLater = self.reactor.callLater  # type: ignore[assignment]
+
+        client_protocol = client_factory.buildProtocol(None)
+        client_protocol.makeConnection(FakeTransport(server_protocol, self.reactor))
+        server_protocol.makeConnection(
+            FakeTransport(
+                client_protocol,
+                self.reactor,
+                peer_address=IPv4Address("TCP", "127.0.0.1", 1234),
+            )
+        )
+
+        # the message should now get delivered
+        self.get_success(d, by=0.1)
+
+        # check it arrived
+        self.assertEqual(len(message_delivery.messages), 1)
+        user, msg = message_delivery.messages.pop()
+        self.assertEqual(str(user), "foo@bar.com")
+        self.assertIn(b"Subject: test subject", msg)
diff --git a/tests/handlers/test_stats.py b/tests/handlers/test_stats.py
index 0229f58315..0673053ab0 100644
--- a/tests/handlers/test_stats.py
+++ b/tests/handlers/test_stats.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +13,7 @@
 # limitations under the License.
 
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.storage.databases.main import stats
 
 from tests import unittest
@@ -95,16 +94,12 @@ async def get_all_room_state(self):
     def _get_current_stats(self, stats_type, stat_id):
         table, id_col = stats.TYPE_TO_TABLE[stats_type]
 
-        cols = list(stats.ABSOLUTE_STATS_FIELDS[stats_type]) + list(
-            stats.PER_SLICE_FIELDS[stats_type]
-        )
-
-        end_ts = self.store.quantise_stats_time(self.reactor.seconds() * 1000)
+        cols = list(stats.ABSOLUTE_STATS_FIELDS[stats_type])
 
         return self.get_success(
             self.store.db_pool.simple_select_one(
-                table + "_historical",
-                {id_col: stat_id, end_ts: end_ts},
+                table + "_current",
+                {id_col: stat_id},
                 cols,
                 allow_none=True,
             )
@@ -163,115 +158,6 @@ def test_initial_room(self):
         self.assertEqual(len(r), 1)
         self.assertEqual(r[0]["topic"], "foo")
 
-    def test_initial_earliest_token(self):
-        """
-        Ingestion via notify_new_event will ignore tokens that the background
-        update have already processed.
-        """
-
-        self.reactor.advance(86401)
-
-        self.hs.config.stats_enabled = False
-        self.handler.stats_enabled = False
-
-        u1 = self.register_user("u1", "pass")
-        u1_token = self.login("u1", "pass")
-
-        u2 = self.register_user("u2", "pass")
-        u2_token = self.login("u2", "pass")
-
-        u3 = self.register_user("u3", "pass")
-        u3_token = self.login("u3", "pass")
-
-        room_1 = self.helper.create_room_as(u1, tok=u1_token)
-        self.helper.send_state(
-            room_1, event_type="m.room.topic", body={"topic": "foo"}, tok=u1_token
-        )
-
-        # Begin the ingestion by creating the temp tables. This will also store
-        # the position that the deltas should begin at, once they take over.
-        self.hs.config.stats_enabled = True
-        self.handler.stats_enabled = True
-        self.store.db_pool.updates._all_done = False
-        self.get_success(
-            self.store.db_pool.simple_update_one(
-                table="stats_incremental_position",
-                keyvalues={},
-                updatevalues={"stream_id": 0},
-            )
-        )
-
-        self.get_success(
-            self.store.db_pool.simple_insert(
-                "background_updates",
-                {"update_name": "populate_stats_prepare", "progress_json": "{}"},
-            )
-        )
-
-        while not self.get_success(
-            self.store.db_pool.updates.has_completed_background_updates()
-        ):
-            self.get_success(
-                self.store.db_pool.updates.do_next_background_update(100), by=0.1
-            )
-
-        # Now, before the table is actually ingested, add some more events.
-        self.helper.invite(room=room_1, src=u1, targ=u2, tok=u1_token)
-        self.helper.join(room=room_1, user=u2, tok=u2_token)
-
-        # orig_delta_processor = self.store.
-
-        # Now do the initial ingestion.
-        self.get_success(
-            self.store.db_pool.simple_insert(
-                "background_updates",
-                {"update_name": "populate_stats_process_rooms", "progress_json": "{}"},
-            )
-        )
-        self.get_success(
-            self.store.db_pool.simple_insert(
-                "background_updates",
-                {
-                    "update_name": "populate_stats_cleanup",
-                    "progress_json": "{}",
-                    "depends_on": "populate_stats_process_rooms",
-                },
-            )
-        )
-
-        self.store.db_pool.updates._all_done = False
-        while not self.get_success(
-            self.store.db_pool.updates.has_completed_background_updates()
-        ):
-            self.get_success(
-                self.store.db_pool.updates.do_next_background_update(100), by=0.1
-            )
-
-        self.reactor.advance(86401)
-
-        # Now add some more events, triggering ingestion. Because of the stream
-        # position being set to before the events sent in the middle, a simpler
-        # implementation would reprocess those events, and say there were four
-        # users, not three.
-        self.helper.invite(room=room_1, src=u1, targ=u3, tok=u1_token)
-        self.helper.join(room=room_1, user=u3, tok=u3_token)
-
-        # self.handler.notify_new_event()
-
-        # We need to let the delta processor advance…
-        self.reactor.advance(10 * 60)
-
-        # Get the slices! There should be two -- day 1, and day 2.
-        r = self.get_success(self.store.get_statistics_for_subject("room", room_1, 0))
-
-        self.assertEqual(len(r), 2)
-
-        # The oldest has 2 joined members
-        self.assertEqual(r[-1]["joined_members"], 2)
-
-        # The newest has 3
-        self.assertEqual(r[0]["joined_members"], 3)
-
     def test_create_user(self):
         """
         When we create a user, it should have statistics already ready.
@@ -303,22 +189,6 @@ def test_create_room(self):
         self.assertIsNotNone(r1stats)
         self.assertIsNotNone(r2stats)
 
-        # contains the default things you'd expect in a fresh room
-        self.assertEqual(
-            r1stats["total_events"],
-            EXPT_NUM_STATE_EVTS_IN_FRESH_PUBLIC_ROOM,
-            "Wrong number of total_events in new room's stats!"
-            " You may need to update this if more state events are added to"
-            " the room creation process.",
-        )
-        self.assertEqual(
-            r2stats["total_events"],
-            EXPT_NUM_STATE_EVTS_IN_FRESH_PRIVATE_ROOM,
-            "Wrong number of total_events in new room's stats!"
-            " You may need to update this if more state events are added to"
-            " the room creation process.",
-        )
-
         self.assertEqual(
             r1stats["current_state_events"], EXPT_NUM_STATE_EVTS_IN_FRESH_PUBLIC_ROOM
         )
@@ -334,24 +204,6 @@ def test_create_room(self):
         self.assertEqual(r2stats["invited_members"], 0)
         self.assertEqual(r2stats["banned_members"], 0)
 
-    def test_send_message_increments_total_events(self):
-        """
-        When we send a message, it increments total_events.
-        """
-
-        self._perform_background_initial_update()
-
-        u1 = self.register_user("u1", "pass")
-        u1token = self.login("u1", "pass")
-        r1 = self.helper.create_room_as(u1, tok=u1token)
-        r1stats_ante = self._get_current_stats("room", r1)
-
-        self.helper.send(r1, "hiss", tok=u1token)
-
-        r1stats_post = self._get_current_stats("room", r1)
-
-        self.assertEqual(r1stats_post["total_events"] - r1stats_ante["total_events"], 1)
-
     def test_updating_profile_information_does_not_increase_joined_members_count(self):
         """
         Check that the joined_members count does not increase when a user changes their
@@ -385,7 +237,7 @@ def test_updating_profile_information_does_not_increase_joined_members_count(sel
 
     def test_send_state_event_nonoverwriting(self):
         """
-        When we send a non-overwriting state event, it increments total_events AND current_state_events
+        When we send a non-overwriting state event, it increments current_state_events
         """
 
         self._perform_background_initial_update()
@@ -406,44 +258,14 @@ def test_send_state_event_nonoverwriting(self):
 
         r1stats_post = self._get_current_stats("room", r1)
 
-        self.assertEqual(r1stats_post["total_events"] - r1stats_ante["total_events"], 1)
         self.assertEqual(
             r1stats_post["current_state_events"] - r1stats_ante["current_state_events"],
             1,
         )
 
-    def test_send_state_event_overwriting(self):
-        """
-        When we send an overwriting state event, it increments total_events ONLY
-        """
-
-        self._perform_background_initial_update()
-
-        u1 = self.register_user("u1", "pass")
-        u1token = self.login("u1", "pass")
-        r1 = self.helper.create_room_as(u1, tok=u1token)
-
-        self.helper.send_state(
-            r1, "cat.hissing", {"value": True}, tok=u1token, state_key="tabby"
-        )
-
-        r1stats_ante = self._get_current_stats("room", r1)
-
-        self.helper.send_state(
-            r1, "cat.hissing", {"value": False}, tok=u1token, state_key="tabby"
-        )
-
-        r1stats_post = self._get_current_stats("room", r1)
-
-        self.assertEqual(r1stats_post["total_events"] - r1stats_ante["total_events"], 1)
-        self.assertEqual(
-            r1stats_post["current_state_events"] - r1stats_ante["current_state_events"],
-            0,
-        )
-
     def test_join_first_time(self):
         """
-        When a user joins a room for the first time, total_events, current_state_events and
+        When a user joins a room for the first time, current_state_events and
         joined_members should increase by exactly 1.
         """
 
@@ -462,7 +284,6 @@ def test_join_first_time(self):
 
         r1stats_post = self._get_current_stats("room", r1)
 
-        self.assertEqual(r1stats_post["total_events"] - r1stats_ante["total_events"], 1)
         self.assertEqual(
             r1stats_post["current_state_events"] - r1stats_ante["current_state_events"],
             1,
@@ -473,7 +294,7 @@ def test_join_first_time(self):
 
     def test_join_after_leave(self):
         """
-        When a user joins a room after being previously left, total_events and
+        When a user joins a room after being previously left,
         joined_members should increase by exactly 1.
         current_state_events should not increase.
         left_members should decrease by exactly 1.
@@ -497,7 +318,6 @@ def test_join_after_leave(self):
 
         r1stats_post = self._get_current_stats("room", r1)
 
-        self.assertEqual(r1stats_post["total_events"] - r1stats_ante["total_events"], 1)
         self.assertEqual(
             r1stats_post["current_state_events"] - r1stats_ante["current_state_events"],
             0,
@@ -511,7 +331,7 @@ def test_join_after_leave(self):
 
     def test_invited(self):
         """
-        When a user invites another user, current_state_events, total_events and
+        When a user invites another user, current_state_events and
         invited_members should increase by exactly 1.
         """
 
@@ -529,7 +349,6 @@ def test_invited(self):
 
         r1stats_post = self._get_current_stats("room", r1)
 
-        self.assertEqual(r1stats_post["total_events"] - r1stats_ante["total_events"], 1)
         self.assertEqual(
             r1stats_post["current_state_events"] - r1stats_ante["current_state_events"],
             1,
@@ -540,7 +359,7 @@ def test_invited(self):
 
     def test_join_after_invite(self):
         """
-        When a user joins a room after being invited, total_events and
+        When a user joins a room after being invited and
         joined_members should increase by exactly 1.
         current_state_events should not increase.
         invited_members should decrease by exactly 1.
@@ -563,7 +382,6 @@ def test_join_after_invite(self):
 
         r1stats_post = self._get_current_stats("room", r1)
 
-        self.assertEqual(r1stats_post["total_events"] - r1stats_ante["total_events"], 1)
         self.assertEqual(
             r1stats_post["current_state_events"] - r1stats_ante["current_state_events"],
             0,
@@ -577,7 +395,7 @@ def test_join_after_invite(self):
 
     def test_left(self):
         """
-        When a user leaves a room after joining, total_events and
+        When a user leaves a room after joining and
         left_members should increase by exactly 1.
         current_state_events should not increase.
         joined_members should decrease by exactly 1.
@@ -600,7 +418,6 @@ def test_left(self):
 
         r1stats_post = self._get_current_stats("room", r1)
 
-        self.assertEqual(r1stats_post["total_events"] - r1stats_ante["total_events"], 1)
         self.assertEqual(
             r1stats_post["current_state_events"] - r1stats_ante["current_state_events"],
             0,
@@ -614,7 +431,7 @@ def test_left(self):
 
     def test_banned(self):
         """
-        When a user is banned from a room after joining, total_events and
+        When a user is banned from a room after joining and
         left_members should increase by exactly 1.
         current_state_events should not increase.
         banned_members should decrease by exactly 1.
@@ -637,7 +454,6 @@ def test_banned(self):
 
         r1stats_post = self._get_current_stats("room", r1)
 
-        self.assertEqual(r1stats_post["total_events"] - r1stats_ante["total_events"], 1)
         self.assertEqual(
             r1stats_post["current_state_events"] - r1stats_ante["current_state_events"],
             0,
diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py
index e62586142e..84f05f6c58 100644
--- a/tests/handlers/test_sync.py
+++ b/tests/handlers/test_sync.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,7 +22,7 @@
 
 
 class SyncTestCase(tests.unittest.HomeserverTestCase):
-    """ Tests Sync Handler. """
+    """Tests Sync Handler."""
 
     def prepare(self, reactor, clock, hs):
         self.hs = hs
@@ -37,7 +36,7 @@ def prepare(self, reactor, clock, hs):
     def test_wait_for_sync_for_user_auth_blocking(self):
         user_id1 = "@user1:test"
         user_id2 = "@user2:test"
-        sync_config = self._generate_sync_config(user_id1)
+        sync_config = generate_sync_config(user_id1)
         requester = create_requester(user_id1)
 
         self.reactor.advance(100)  # So we get not 0 time
@@ -60,7 +59,7 @@ def test_wait_for_sync_for_user_auth_blocking(self):
 
         self.auth_blocking._hs_disabled = False
 
-        sync_config = self._generate_sync_config(user_id2)
+        sync_config = generate_sync_config(user_id2)
         requester = create_requester(user_id2)
 
         e = self.get_failure(
@@ -69,11 +68,12 @@ def test_wait_for_sync_for_user_auth_blocking(self):
         )
         self.assertEquals(e.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
 
-    def _generate_sync_config(self, user_id):
-        return SyncConfig(
-            user=UserID(user_id.split(":")[0][1:], user_id.split(":")[1]),
-            filter_collection=DEFAULT_FILTER_COLLECTION,
-            is_guest=False,
-            request_key="request_key",
-            device_id="device_id",
-        )
+
+def generate_sync_config(user_id: str) -> SyncConfig:
+    return SyncConfig(
+        user=UserID(user_id.split(":")[0][1:], user_id.split(":")[1]),
+        filter_collection=DEFAULT_FILTER_COLLECTION,
+        is_guest=False,
+        request_key="request_key",
+        device_id="device_id",
+    )
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index 24e7138196..fa3cff598e 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,8 +15,7 @@
 
 import json
 from typing import Dict
-
-from mock import ANY, Mock, call
+from unittest.mock import ANY, Mock, call
 
 from twisted.internet import defer
 from twisted.web.resource import Resource
@@ -40,6 +38,9 @@
 # Test room id
 ROOM_ID = "a-room"
 
+# Room we're not in
+OTHER_ROOM_ID = "another-room"
+
 
 def _expect_edu_transaction(edu_type, content, origin="test"):
     return {
@@ -91,14 +92,8 @@ def prepare(self, reactor, clock, hs):
         self.event_source = hs.get_event_sources().sources["typing"]
 
         self.datastore = hs.get_datastore()
-        retry_timings_res = {
-            "destination": "",
-            "retry_last_ts": 0,
-            "retry_interval": 0,
-            "failure_ts": None,
-        }
         self.datastore.get_destination_retry_timings = Mock(
-            return_value=defer.succeed(retry_timings_res)
+            return_value=defer.succeed(None)
         )
 
         self.datastore.get_device_updates_by_remote = Mock(
@@ -123,6 +118,11 @@ async def check_user_in_room(room_id, user_id):
 
         hs.get_auth().check_user_in_room = check_user_in_room
 
+        async def check_host_in_room(room_id, server_name):
+            return room_id == ROOM_ID
+
+        hs.get_event_auth_handler().check_host_in_room = check_host_in_room
+
         def get_joined_hosts_for_room(room_id):
             return {member.domain for member in self.room_members}
 
@@ -252,6 +252,35 @@ def test_started_typing_remote_recv(self):
             ],
         )
 
+    def test_started_typing_remote_recv_not_in_room(self):
+        self.room_members = [U_APPLE, U_ONION]
+
+        self.assertEquals(self.event_source.get_current_key(), 0)
+
+        channel = self.make_request(
+            "PUT",
+            "/_matrix/federation/v1/send/1000000",
+            _make_edu_transaction_json(
+                "m.typing",
+                content={
+                    "room_id": OTHER_ROOM_ID,
+                    "user_id": U_ONION.to_string(),
+                    "typing": True,
+                },
+            ),
+            federation_auth_origin=b"farm",
+        )
+        self.assertEqual(channel.code, 200)
+
+        self.on_new_event.assert_not_called()
+
+        self.assertEquals(self.event_source.get_current_key(), 0)
+        events = self.get_success(
+            self.event_source.get_new_events(room_ids=[OTHER_ROOM_ID], from_key=0)
+        )
+        self.assertEquals(events[0], [])
+        self.assertEquals(events[1], 0)
+
     @override_config({"send_federation": True})
     def test_stopped_typing(self):
         self.room_members = [U_APPLE, U_BANANA, U_ONION]
diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py
index dbe68bb058..1504b4d38f 100644
--- a/tests/handlers/test_user_directory.py
+++ b/tests/handlers/test_user_directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,15 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
 import synapse.rest.admin
 from synapse.api.constants import EventTypes, RoomEncryptionAlgorithms, UserTypes
 from synapse.api.room_versions import RoomVersion, RoomVersions
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import account, account_validity, user_directory
+from synapse.rest.client import account, account_validity, login, room, user_directory
 from synapse.storage.roommember import ProfileInfo
 
 from tests import unittest
@@ -313,15 +311,13 @@ def test_spam_checker(self):
         s = self.get_success(self.handler.search_users(u1, "user2", 10))
         self.assertEqual(len(s["results"]), 1)
 
+        async def allow_all(user_profile):
+            # Allow all users.
+            return False
+
         # Configure a spam checker that does not filter any users.
         spam_checker = self.hs.get_spam_checker()
-
-        class AllowAll:
-            async def check_username_for_spam(self, user_profile):
-                # Allow all users.
-                return False
-
-        spam_checker.spam_checkers = [AllowAll()]
+        spam_checker._check_username_for_spam_callbacks = [allow_all]
 
         # The results do not change:
         # We get one search result when searching for user2 by user1.
@@ -329,12 +325,11 @@ async def check_username_for_spam(self, user_profile):
         self.assertEqual(len(s["results"]), 1)
 
         # Configure a spam checker that filters all users.
-        class BlockAll:
-            async def check_username_for_spam(self, user_profile):
-                # All users are spammy.
-                return True
+        async def block_all(user_profile):
+            # All users are spammy.
+            return True
 
-        spam_checker.spam_checkers = [BlockAll()]
+        spam_checker._check_username_for_spam_callbacks = [block_all]
 
         # User1 now gets no search results for any of the other users.
         s = self.get_success(self.handler.search_users(u1, "user2", 10))
@@ -695,7 +690,7 @@ class UserInfoTestCase(unittest.FederatingHomeserverTestCase):
         login.register_servlets,
         synapse.rest.admin.register_servlets_for_client_rest_resource,
         account_validity.register_servlets,
-        synapse.rest.client.v2_alpha.user_directory.register_servlets,
+        user_directory.register_servlets,
         account.register_servlets,
     ]
 
diff --git a/tests/http/__init__.py b/tests/http/__init__.py
index 3e5a856584..e74f7f5b48 100644
--- a/tests/http/__init__.py
+++ b/tests/http/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/http/federation/__init__.py b/tests/http/federation/__init__.py
index 1453d04571..743fb9904a 100644
--- a/tests/http/federation/__init__.py
+++ b/tests/http/federation/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py
index 3972abb038..992d8f94fd 100644
--- a/tests/http/federation/test_matrix_federation_agent.py
+++ b/tests/http/federation/test_matrix_federation_agent.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,9 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import base64
 import logging
-
-from mock import Mock
+import os
+from typing import Iterable, Optional
+from unittest.mock import Mock, patch
 
 import treq
 from netaddr import IPSet
@@ -23,11 +24,12 @@
 
 from twisted.internet import defer
 from twisted.internet._sslverify import ClientTLSOptions, OpenSSLCertificateOptions
+from twisted.internet.interfaces import IProtocolFactory
 from twisted.internet.protocol import Factory
-from twisted.protocols.tls import TLSMemoryBIOFactory
+from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
 from twisted.web._newclient import ResponseNeverReceived
 from twisted.web.client import Agent
-from twisted.web.http import HTTPChannel
+from twisted.web.http import HTTPChannel, Request
 from twisted.web.http_headers import Headers
 from twisted.web.iweb import IPolicyForHTTPS
 
@@ -50,24 +52,6 @@
 
 logger = logging.getLogger(__name__)
 
-test_server_connection_factory = None
-
-
-def get_connection_factory():
-    # this needs to happen once, but not until we are ready to run the first test
-    global test_server_connection_factory
-    if test_server_connection_factory is None:
-        test_server_connection_factory = TestServerTLSConnectionFactory(
-            sanlist=[
-                b"DNS:testserv",
-                b"DNS:target-server",
-                b"DNS:xn--bcher-kva.com",
-                b"IP:1.2.3.4",
-                b"IP:::1",
-            ]
-        )
-    return test_server_connection_factory
-
 
 # Once Async Mocks or lambdas are supported this can go away.
 def generate_resolve_service(result):
@@ -101,24 +85,38 @@ def setUp(self):
             had_well_known_cache=self.had_well_known_cache,
         )
 
-        self.agent = MatrixFederationAgent(
-            reactor=self.reactor,
-            tls_client_options_factory=FederationPolicyForHTTPS(config),
-            user_agent="test-agent",  # Note that this is unused since _well_known_resolver is provided.
-            ip_blacklist=IPSet(),
-            _srv_resolver=self.mock_resolver,
-            _well_known_resolver=self.well_known_resolver,
-        )
-
-    def _make_connection(self, client_factory, expected_sni):
+    def _make_connection(
+        self,
+        client_factory: IProtocolFactory,
+        ssl: bool = True,
+        expected_sni: bytes = None,
+        tls_sanlist: Optional[Iterable[bytes]] = None,
+    ) -> HTTPChannel:
         """Builds a test server, and completes the outgoing client connection
+        Args:
+            client_factory: the the factory that the
+                application is trying to use to make the outbound connection. We will
+                invoke it to build the client Protocol
+
+            ssl: If true, we will expect an ssl connection and wrap
+                server_factory with a TLSMemoryBIOFactory
+                False is set only for when proxy expect http connection.
+                Otherwise federation requests use always https.
+
+            expected_sni: the expected SNI value
+
+            tls_sanlist: list of SAN entries for the TLS cert presented by the server.
 
         Returns:
-            HTTPChannel: the test server
+            the server Protocol returned by server_factory
         """
 
         # build the test server
-        server_tls_protocol = _build_test_server(get_connection_factory())
+        server_factory = _get_test_protocol_factory()
+        if ssl:
+            server_factory = _wrap_server_factory_for_tls(server_factory, tls_sanlist)
+
+        server_protocol = server_factory.buildProtocol(None)
 
         # now, tell the client protocol factory to build the client protocol (it will be a
         # _WrappingProtocol, around a TLSMemoryBIOProtocol, around an
@@ -129,35 +127,39 @@ def _make_connection(self, client_factory, expected_sni):
         # stubbing that out here.
         client_protocol = client_factory.buildProtocol(None)
         client_protocol.makeConnection(
-            FakeTransport(server_tls_protocol, self.reactor, client_protocol)
+            FakeTransport(server_protocol, self.reactor, client_protocol)
         )
 
-        # tell the server tls protocol to send its stuff back to the client, too
-        server_tls_protocol.makeConnection(
-            FakeTransport(client_protocol, self.reactor, server_tls_protocol)
+        # tell the server protocol to send its stuff back to the client, too
+        server_protocol.makeConnection(
+            FakeTransport(client_protocol, self.reactor, server_protocol)
         )
 
-        # grab a hold of the TLS connection, in case it gets torn down
-        server_tls_connection = server_tls_protocol._tlsConnection
-
-        # fish the test server back out of the server-side TLS protocol.
-        http_protocol = server_tls_protocol.wrappedProtocol
+        if ssl:
+            # fish the test server back out of the server-side TLS protocol.
+            http_protocol = server_protocol.wrappedProtocol
+            # grab a hold of the TLS connection, in case it gets torn down
+            tls_connection = server_protocol._tlsConnection
+        else:
+            http_protocol = server_protocol
+            tls_connection = None
 
-        # give the reactor a pump to get the TLS juices flowing.
-        self.reactor.pump((0.1,))
+        # give the reactor a pump to get the TLS juices flowing (if needed)
+        self.reactor.advance(0)
 
         # check the SNI
-        server_name = server_tls_connection.get_servername()
-        self.assertEqual(
-            server_name,
-            expected_sni,
-            "Expected SNI %s but got %s" % (expected_sni, server_name),
-        )
+        if expected_sni is not None:
+            server_name = tls_connection.get_servername()
+            self.assertEqual(
+                server_name,
+                expected_sni,
+                f"Expected SNI {expected_sni!s} but got {server_name!s}",
+            )
 
         return http_protocol
 
     @defer.inlineCallbacks
-    def _make_get_request(self, uri):
+    def _make_get_request(self, uri: bytes):
         """
         Sends a simple GET request via the agent, and checks its logcontext management
         """
@@ -180,17 +182,21 @@ def _make_get_request(self, uri):
                 _check_logcontext(context)
 
     def _handle_well_known_connection(
-        self, client_factory, expected_sni, content, response_headers={}
-    ):
+        self,
+        client_factory: IProtocolFactory,
+        expected_sni: bytes,
+        content: bytes,
+        response_headers: Optional[dict] = None,
+    ) -> HTTPChannel:
         """Handle an outgoing HTTPs connection: wire it up to a server, check that the
         request is for a .well-known, and send the response.
 
         Args:
-            client_factory (IProtocolFactory): outgoing connection
-            expected_sni (bytes): SNI that we expect the outgoing connection to send
-            content (bytes): content to send back as the .well-known
+            client_factory: outgoing connection
+            expected_sni: SNI that we expect the outgoing connection to send
+            content: content to send back as the .well-known
         Returns:
-            HTTPChannel: server impl
+            server impl
         """
         # make the connection for .well-known
         well_known_server = self._make_connection(
@@ -202,10 +208,15 @@ def _handle_well_known_connection(
         self.assertEqual(
             request.requestHeaders.getRawHeaders(b"user-agent"), [b"test-agent"]
         )
-        self._send_well_known_response(request, content, headers=response_headers)
+        self._send_well_known_response(request, content, headers=response_headers or {})
         return well_known_server
 
-    def _send_well_known_response(self, request, content, headers={}):
+    def _send_well_known_response(
+        self,
+        request: Request,
+        content: bytes,
+        headers: Optional[dict] = None,
+    ):
         """Check that an incoming request looks like a valid .well-known request, and
         send back the response.
         """
@@ -213,17 +224,44 @@ def _send_well_known_response(self, request, content, headers={}):
         self.assertEqual(request.path, b"/.well-known/matrix/server")
         self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"testserv"])
         # send back a response
-        for k, v in headers.items():
+        for k, v in (headers or {}).items():
             request.setHeader(k, v)
         request.write(content)
         request.finish()
 
         self.reactor.pump((0.1,))
 
-    def test_get(self):
+    def _make_agent(self) -> MatrixFederationAgent:
         """
-        happy-path test of a GET request with an explicit port
+        If a proxy server is set, the MatrixFederationAgent must be created again
+        because it is created too early during setUp
         """
+        return MatrixFederationAgent(
+            reactor=self.reactor,
+            tls_client_options_factory=self.tls_factory,
+            user_agent="test-agent",  # Note that this is unused since _well_known_resolver is provided.
+            ip_whitelist=IPSet(),
+            ip_blacklist=IPSet(),
+            _srv_resolver=self.mock_resolver,
+            _well_known_resolver=self.well_known_resolver,
+        )
+
+    def test_get(self):
+        """happy-path test of a GET request with an explicit port"""
+        self._do_get()
+
+    @patch.dict(
+        os.environ,
+        {"https_proxy": "proxy.com", "no_proxy": "testserv"},
+    )
+    def test_get_bypass_proxy(self):
+        """test of a GET request with an explicit port and bypass proxy"""
+        self._do_get()
+
+    def _do_get(self):
+        """test of a GET request with an explicit port"""
+        self.agent = self._make_agent()
+
         self.reactor.lookups["testserv"] = "1.2.3.4"
         test_d = self._make_get_request(b"matrix://testserv:8448/foo/bar")
 
@@ -267,6 +305,182 @@ def test_get(self):
         # that should give us a Response object
         self.assertEqual(response.code, 200)
 
+        # Send the body
+        request.write(b'{ "a": 1 }')
+        request.finish()
+
+        self.reactor.pump((0.1,))
+
+        # check it can be read
+        json = self.successResultOf(treq.json_content(response))
+        self.assertEqual(json, {"a": 1})
+
+    @patch.dict(
+        os.environ, {"https_proxy": "http://proxy.com", "no_proxy": "unused.com"}
+    )
+    def test_get_via_http_proxy(self):
+        """test for federation request through a http proxy"""
+        self._do_get_via_proxy(expect_proxy_ssl=False, expected_auth_credentials=None)
+
+    @patch.dict(
+        os.environ,
+        {"https_proxy": "http://user:pass@proxy.com", "no_proxy": "unused.com"},
+    )
+    def test_get_via_http_proxy_with_auth(self):
+        """test for federation request through a http proxy with authentication"""
+        self._do_get_via_proxy(
+            expect_proxy_ssl=False, expected_auth_credentials=b"user:pass"
+        )
+
+    @patch.dict(
+        os.environ, {"https_proxy": "https://proxy.com", "no_proxy": "unused.com"}
+    )
+    def test_get_via_https_proxy(self):
+        """test for federation request through a https proxy"""
+        self._do_get_via_proxy(expect_proxy_ssl=True, expected_auth_credentials=None)
+
+    @patch.dict(
+        os.environ,
+        {"https_proxy": "https://user:pass@proxy.com", "no_proxy": "unused.com"},
+    )
+    def test_get_via_https_proxy_with_auth(self):
+        """test for federation request through a https proxy with authentication"""
+        self._do_get_via_proxy(
+            expect_proxy_ssl=True, expected_auth_credentials=b"user:pass"
+        )
+
+    def _do_get_via_proxy(
+        self,
+        expect_proxy_ssl: bool = False,
+        expected_auth_credentials: Optional[bytes] = None,
+    ):
+        """Send a https federation request via an agent and check that it is correctly
+            received at the proxy and client. The proxy can use either http or https.
+        Args:
+            expect_proxy_ssl: True if we expect the request to connect to the proxy via https.
+            expected_auth_credentials: credentials we expect to be presented to authenticate at the proxy
+        """
+        self.agent = self._make_agent()
+
+        self.reactor.lookups["testserv"] = "1.2.3.4"
+        self.reactor.lookups["proxy.com"] = "9.9.9.9"
+        test_d = self._make_get_request(b"matrix://testserv:8448/foo/bar")
+
+        # Nothing happened yet
+        self.assertNoResult(test_d)
+
+        # Make sure treq is trying to connect
+        clients = self.reactor.tcpClients
+        self.assertEqual(len(clients), 1)
+        (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+        # make sure we are connecting to the proxy
+        self.assertEqual(host, "9.9.9.9")
+        self.assertEqual(port, 1080)
+
+        # make a test server to act as the proxy, and wire up the client
+        proxy_server = self._make_connection(
+            client_factory,
+            ssl=expect_proxy_ssl,
+            tls_sanlist=[b"DNS:proxy.com"] if expect_proxy_ssl else None,
+            expected_sni=b"proxy.com" if expect_proxy_ssl else None,
+        )
+
+        assert isinstance(proxy_server, HTTPChannel)
+
+        # now there should be a pending CONNECT request
+        self.assertEqual(len(proxy_server.requests), 1)
+
+        request = proxy_server.requests[0]
+        self.assertEqual(request.method, b"CONNECT")
+        self.assertEqual(request.path, b"testserv:8448")
+
+        # Check whether auth credentials have been supplied to the proxy
+        proxy_auth_header_values = request.requestHeaders.getRawHeaders(
+            b"Proxy-Authorization"
+        )
+
+        if expected_auth_credentials is not None:
+            # Compute the correct header value for Proxy-Authorization
+            encoded_credentials = base64.b64encode(expected_auth_credentials)
+            expected_header_value = b"Basic " + encoded_credentials
+
+            # Validate the header's value
+            self.assertIn(expected_header_value, proxy_auth_header_values)
+        else:
+            # Check that the Proxy-Authorization header has not been supplied to the proxy
+            self.assertIsNone(proxy_auth_header_values)
+
+        # tell the proxy server not to close the connection
+        proxy_server.persistent = True
+
+        request.finish()
+
+        # now we make another test server to act as the upstream HTTP server.
+        server_ssl_protocol = _wrap_server_factory_for_tls(
+            _get_test_protocol_factory()
+        ).buildProtocol(None)
+
+        # Tell the HTTP server to send outgoing traffic back via the proxy's transport.
+        proxy_server_transport = proxy_server.transport
+        server_ssl_protocol.makeConnection(proxy_server_transport)
+
+        # ... and replace the protocol on the proxy's transport with the
+        # TLSMemoryBIOProtocol for the test server, so that incoming traffic
+        # to the proxy gets sent over to the HTTP(s) server.
+
+        # See also comment at `_do_https_request_via_proxy`
+        # in ../test_proxyagent.py for more details
+        if expect_proxy_ssl:
+            assert isinstance(proxy_server_transport, TLSMemoryBIOProtocol)
+            proxy_server_transport.wrappedProtocol = server_ssl_protocol
+        else:
+            assert isinstance(proxy_server_transport, FakeTransport)
+            client_protocol = proxy_server_transport.other
+            c2s_transport = client_protocol.transport
+            c2s_transport.other = server_ssl_protocol
+
+        self.reactor.advance(0)
+
+        server_name = server_ssl_protocol._tlsConnection.get_servername()
+        expected_sni = b"testserv"
+        self.assertEqual(
+            server_name,
+            expected_sni,
+            f"Expected SNI {expected_sni!s} but got {server_name!s}",
+        )
+
+        # now there should be a pending request
+        http_server = server_ssl_protocol.wrappedProtocol
+        self.assertEqual(len(http_server.requests), 1)
+
+        request = http_server.requests[0]
+        self.assertEqual(request.method, b"GET")
+        self.assertEqual(request.path, b"/foo/bar")
+        self.assertEqual(
+            request.requestHeaders.getRawHeaders(b"host"), [b"testserv:8448"]
+        )
+        self.assertEqual(
+            request.requestHeaders.getRawHeaders(b"user-agent"), [b"test-agent"]
+        )
+        # Check that the destination server DID NOT receive proxy credentials
+        self.assertIsNone(request.requestHeaders.getRawHeaders(b"Proxy-Authorization"))
+        content = request.content.read()
+        self.assertEqual(content, b"")
+
+        # Deferred is still without a result
+        self.assertNoResult(test_d)
+
+        # send the headers
+        request.responseHeaders.setRawHeaders(b"Content-Type", [b"application/json"])
+        request.write("")
+
+        self.reactor.pump((0.1,))
+
+        response = self.successResultOf(test_d)
+
+        # that should give us a Response object
+        self.assertEqual(response.code, 200)
+
         # Send the body
         request.write('{ "a": 1 }'.encode("ascii"))
         request.finish()
@@ -281,6 +495,8 @@ def test_get_ip_address(self):
         """
         Test the behaviour when the server name contains an explicit IP (with no port)
         """
+        self.agent = self._make_agent()
+
         # there will be a getaddrinfo on the IP
         self.reactor.lookups["1.2.3.4"] = "1.2.3.4"
 
@@ -315,6 +531,7 @@ def test_get_ipv6_address(self):
         Test the behaviour when the server name contains an explicit IPv6 address
         (with no port)
         """
+        self.agent = self._make_agent()
 
         # there will be a getaddrinfo on the IP
         self.reactor.lookups["::1"] = "::1"
@@ -350,6 +567,7 @@ def test_get_ipv6_address_with_port(self):
         Test the behaviour when the server name contains an explicit IPv6 address
         (with explicit port)
         """
+        self.agent = self._make_agent()
 
         # there will be a getaddrinfo on the IP
         self.reactor.lookups["::1"] = "::1"
@@ -384,6 +602,8 @@ def test_get_hostname_bad_cert(self):
         """
         Test the behaviour when the certificate on the server doesn't match the hostname
         """
+        self.agent = self._make_agent()
+
         self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
         self.reactor.lookups["testserv1"] = "1.2.3.4"
 
@@ -436,6 +656,8 @@ def test_get_ip_address_bad_cert(self):
         Test the behaviour when the server name contains an explicit IP, but
         the server cert doesn't cover it
         """
+        self.agent = self._make_agent()
+
         # there will be a getaddrinfo on the IP
         self.reactor.lookups["1.2.3.5"] = "1.2.3.5"
 
@@ -466,6 +688,7 @@ def test_get_no_srv_no_well_known(self):
         """
         Test the behaviour when the server name has no port, no SRV, and no well-known
         """
+        self.agent = self._make_agent()
 
         self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
         self.reactor.lookups["testserv"] = "1.2.3.4"
@@ -519,6 +742,7 @@ def test_get_no_srv_no_well_known(self):
 
     def test_get_well_known(self):
         """Test the behaviour when the .well-known delegates elsewhere"""
+        self.agent = self._make_agent()
 
         self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
         self.reactor.lookups["testserv"] = "1.2.3.4"
@@ -582,6 +806,8 @@ def test_get_well_known_redirect(self):
         """Test the behaviour when the server name has no port and no SRV record, but
         the .well-known has a 300 redirect
         """
+        self.agent = self._make_agent()
+
         self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
         self.reactor.lookups["testserv"] = "1.2.3.4"
         self.reactor.lookups["target-server"] = "1::f"
@@ -670,6 +896,7 @@ def test_get_invalid_well_known(self):
         """
         Test the behaviour when the server name has an *invalid* well-known (and no SRV)
         """
+        self.agent = self._make_agent()
 
         self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
         self.reactor.lookups["testserv"] = "1.2.3.4"
@@ -738,6 +965,7 @@ def test_get_well_known_unsigned_cert(self):
             reactor=self.reactor,
             tls_client_options_factory=tls_factory,
             user_agent=b"test-agent",  # This is unused since _well_known_resolver is passed below.
+            ip_whitelist=IPSet(),
             ip_blacklist=IPSet(),
             _srv_resolver=self.mock_resolver,
             _well_known_resolver=WellKnownResolver(
@@ -775,6 +1003,8 @@ def test_get_hostname_srv(self):
         """
         Test the behaviour when there is a single SRV record
         """
+        self.agent = self._make_agent()
+
         self.mock_resolver.resolve_service.side_effect = generate_resolve_service(
             [Server(host=b"srvtarget", port=8443)]
         )
@@ -815,6 +1045,8 @@ def test_get_well_known_srv(self):
         """Test the behaviour when the .well-known redirects to a place where there
         is a SRV.
         """
+        self.agent = self._make_agent()
+
         self.reactor.lookups["testserv"] = "1.2.3.4"
         self.reactor.lookups["srvtarget"] = "5.6.7.8"
 
@@ -871,6 +1103,7 @@ def test_get_well_known_srv(self):
 
     def test_idna_servername(self):
         """test the behaviour when the server name has idna chars in"""
+        self.agent = self._make_agent()
 
         self.mock_resolver.resolve_service.side_effect = generate_resolve_service([])
 
@@ -932,6 +1165,7 @@ def test_idna_servername(self):
 
     def test_idna_srv_target(self):
         """test the behaviour when the target of a SRV record has idna chars"""
+        self.agent = self._make_agent()
 
         self.mock_resolver.resolve_service.side_effect = generate_resolve_service(
             [Server(host=b"xn--trget-3qa.com", port=8443)]  # târget.com
@@ -1135,6 +1369,8 @@ def test_well_known_too_large(self):
 
     def test_srv_fallbacks(self):
         """Test that other SRV results are tried if the first one fails."""
+        self.agent = self._make_agent()
+
         self.mock_resolver.resolve_service.side_effect = generate_resolve_service(
             [
                 Server(host=b"target.com", port=8443),
@@ -1261,34 +1497,49 @@ def _check_logcontext(context):
         raise AssertionError("Expected logcontext %s but was %s" % (context, current))
 
 
-def _build_test_server(connection_creator):
-    """Construct a test server
-
-    This builds an HTTP channel, wrapped with a TLSMemoryBIOProtocol
-
+def _wrap_server_factory_for_tls(
+    factory: IProtocolFactory, sanlist: Iterable[bytes] = None
+) -> IProtocolFactory:
+    """Wrap an existing Protocol Factory with a test TLSMemoryBIOFactory
+    The resultant factory will create a TLS server which presents a certificate
+    signed by our test CA, valid for the domains in `sanlist`
     Args:
-        connection_creator (IOpenSSLServerConnectionCreator): thing to build
-            SSL connections
-        sanlist (list[bytes]): list of the SAN entries for the cert returned
-            by the server
+        factory: protocol factory to wrap
+        sanlist: list of domains the cert should be valid for
+    Returns:
+        interfaces.IProtocolFactory
+    """
+    if sanlist is None:
+        sanlist = [
+            b"DNS:testserv",
+            b"DNS:target-server",
+            b"DNS:xn--bcher-kva.com",
+            b"IP:1.2.3.4",
+            b"IP:::1",
+        ]
+
+    connection_creator = TestServerTLSConnectionFactory(sanlist=sanlist)
+    return TLSMemoryBIOFactory(
+        connection_creator, isClient=False, wrappedFactory=factory
+    )
 
+
+def _get_test_protocol_factory() -> IProtocolFactory:
+    """Get a protocol Factory which will build an HTTPChannel
     Returns:
-        TLSMemoryBIOProtocol
+        interfaces.IProtocolFactory
     """
     server_factory = Factory.forProtocol(HTTPChannel)
+
     # Request.finish expects the factory to have a 'log' method.
     server_factory.log = _log_request
 
-    server_tls_factory = TLSMemoryBIOFactory(
-        connection_creator, isClient=False, wrappedFactory=server_factory
-    )
-
-    return server_tls_factory.buildProtocol(None)
+    return server_factory
 
 
-def _log_request(request):
+def _log_request(request: str):
     """Implements Factory.log, which is expected by Request.finish"""
-    logger.info("Completed request %s", request)
+    logger.info(f"Completed request {request}")
 
 
 @implementer(IPolicyForHTTPS)
diff --git a/tests/http/federation/test_srv_resolver.py b/tests/http/federation/test_srv_resolver.py
index fee2985d35..c49be33b9f 100644
--- a/tests/http/federation/test_srv_resolver.py
+++ b/tests/http/federation/test_srv_resolver.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 #
@@ -14,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
diff --git a/tests/http/test_additional_resource.py b/tests/http/test_additional_resource.py
index 453391a5a5..768c2ba4ea 100644
--- a/tests/http/test_additional_resource.py
+++ b/tests/http/test_additional_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/http/test_client.py b/tests/http/test_client.py
index 0ce181a51e..7e2f2a01cc 100644
--- a/tests/http/test_client.py
+++ b/tests/http/test_client.py
@@ -13,8 +13,7 @@
 #  limitations under the License.
 
 from io import BytesIO
-
-from mock import Mock
+from unittest.mock import Mock
 
 from netaddr import IPSet
 
diff --git a/tests/http/test_endpoint.py b/tests/http/test_endpoint.py
index d06ea518ce..1f9a2f9b1d 100644
--- a/tests/http/test_endpoint.py
+++ b/tests/http/test_endpoint.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py
index 9c52c8fdca..d9a8b077d3 100644
--- a/tests/http/test_fedclient.py
+++ b/tests/http/test_fedclient.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from netaddr import IPSet
 from parameterized import parameterized
@@ -27,6 +26,7 @@
 
 from synapse.api.errors import RequestSendFailed
 from synapse.http.matrixfederationclient import (
+    MAX_RESPONSE_SIZE,
     MatrixFederationHttpClient,
     MatrixFederationRequest,
 )
@@ -102,7 +102,7 @@ def do_request():
         self.assertNoResult(test_d)
 
         # Send it the HTTP response
-        res_json = '{ "a": 1 }'.encode("ascii")
+        res_json = b'{ "a": 1 }'
         protocol.dataReceived(
             b"HTTP/1.1 200 OK\r\n"
             b"Server: Fake\r\n"
@@ -339,10 +339,8 @@ def test_timeout_reading_body(self, method_name: str):
 
         # Send it the HTTP response
         client.dataReceived(
-            (
-                b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"
-                b"Server: Fake\r\n\r\n"
-            )
+            b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"
+            b"Server: Fake\r\n\r\n"
         )
 
         # Push by enough to time it out
@@ -561,3 +559,61 @@ def test_json_error(self, return_value):
 
         f = self.failureResultOf(test_d)
         self.assertIsInstance(f.value, RequestSendFailed)
+
+    def test_too_big(self):
+        """
+        Test what happens if a huge response is returned from the remote endpoint.
+        """
+
+        test_d = defer.ensureDeferred(self.cl.get_json("testserv:8008", "foo/bar"))
+
+        self.pump()
+
+        # Nothing happened yet
+        self.assertNoResult(test_d)
+
+        # Make sure treq is trying to connect
+        clients = self.reactor.tcpClients
+        self.assertEqual(len(clients), 1)
+        (host, port, factory, _timeout, _bindAddress) = clients[0]
+        self.assertEqual(host, "1.2.3.4")
+        self.assertEqual(port, 8008)
+
+        # complete the connection and wire it up to a fake transport
+        protocol = factory.buildProtocol(None)
+        transport = StringTransport()
+        protocol.makeConnection(transport)
+
+        # that should have made it send the request to the transport
+        self.assertRegex(transport.value(), b"^GET /foo/bar")
+        self.assertRegex(transport.value(), b"Host: testserv:8008")
+
+        # Deferred is still without a result
+        self.assertNoResult(test_d)
+
+        # Send it a huge HTTP response
+        protocol.dataReceived(
+            b"HTTP/1.1 200 OK\r\n"
+            b"Server: Fake\r\n"
+            b"Content-Type: application/json\r\n"
+            b"\r\n"
+        )
+
+        self.pump()
+
+        # should still be waiting
+        self.assertNoResult(test_d)
+
+        sent = 0
+        chunk_size = 1024 * 512
+        while not test_d.called:
+            protocol.dataReceived(b"a" * chunk_size)
+            sent += chunk_size
+            self.assertLessEqual(sent, MAX_RESPONSE_SIZE)
+
+        self.assertEqual(sent, MAX_RESPONSE_SIZE)
+
+        f = self.failureResultOf(test_d)
+        self.assertIsInstance(f.value, RequestSendFailed)
+
+        self.assertTrue(transport.disconnecting)
diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py
index 3ea8b5bec7..2db77c6a73 100644
--- a/tests/http/test_proxyagent.py
+++ b/tests/http/test_proxyagent.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,19 +14,23 @@
 import base64
 import logging
 import os
-from typing import Optional
+from typing import Iterable, Optional
 from unittest.mock import patch
 
 import treq
 from netaddr import IPSet
+from parameterized import parameterized
 
 from twisted.internet import interfaces  # noqa: F401
+from twisted.internet.endpoints import HostnameEndpoint, _WrapperEndpoint
+from twisted.internet.interfaces import IProtocol, IProtocolFactory
 from twisted.internet.protocol import Factory
-from twisted.protocols.tls import TLSMemoryBIOFactory
+from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
 from twisted.web.http import HTTPChannel
 
 from synapse.http.client import BlacklistingReactorWrapper
-from synapse.http.proxyagent import ProxyAgent
+from synapse.http.connectproxyclient import ProxyCredentials
+from synapse.http.proxyagent import ProxyAgent, parse_proxy
 
 from tests.http import TestServerTLSConnectionFactory, get_test_https_policy
 from tests.server import FakeTransport, ThreadedMemoryReactorClock
@@ -38,33 +41,208 @@
 HTTPFactory = Factory.forProtocol(HTTPChannel)
 
 
+class ProxyParserTests(TestCase):
+    """
+    Values for test
+    [
+        proxy_string,
+        expected_scheme,
+        expected_hostname,
+        expected_port,
+        expected_credentials,
+    ]
+    """
+
+    @parameterized.expand(
+        [
+            # host
+            [b"localhost", b"http", b"localhost", 1080, None],
+            [b"localhost:9988", b"http", b"localhost", 9988, None],
+            # host+scheme
+            [b"https://localhost", b"https", b"localhost", 1080, None],
+            [b"https://localhost:1234", b"https", b"localhost", 1234, None],
+            # ipv4
+            [b"1.2.3.4", b"http", b"1.2.3.4", 1080, None],
+            [b"1.2.3.4:9988", b"http", b"1.2.3.4", 9988, None],
+            # ipv4+scheme
+            [b"https://1.2.3.4", b"https", b"1.2.3.4", 1080, None],
+            [b"https://1.2.3.4:9988", b"https", b"1.2.3.4", 9988, None],
+            # ipv6 - without brackets is broken
+            # [
+            #     b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
+            #     b"http",
+            #     b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
+            #     1080,
+            #     None,
+            # ],
+            # [
+            #     b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
+            #     b"http",
+            #     b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
+            #     1080,
+            #     None,
+            # ],
+            # [b"::1", b"http", b"::1", 1080, None],
+            # [b"::ffff:0.0.0.0", b"http", b"::ffff:0.0.0.0", 1080, None],
+            # ipv6 - with brackets
+            [
+                b"[2001:0db8:85a3:0000:0000:8a2e:0370:effe]",
+                b"http",
+                b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
+                1080,
+                None,
+            ],
+            [
+                b"[2001:0db8:85a3:0000:0000:8a2e:0370:1234]",
+                b"http",
+                b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
+                1080,
+                None,
+            ],
+            [b"[::1]", b"http", b"::1", 1080, None],
+            [b"[::ffff:0.0.0.0]", b"http", b"::ffff:0.0.0.0", 1080, None],
+            # ipv6+port
+            [
+                b"[2001:0db8:85a3:0000:0000:8a2e:0370:effe]:9988",
+                b"http",
+                b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
+                9988,
+                None,
+            ],
+            [
+                b"[2001:0db8:85a3:0000:0000:8a2e:0370:1234]:9988",
+                b"http",
+                b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
+                9988,
+                None,
+            ],
+            [b"[::1]:9988", b"http", b"::1", 9988, None],
+            [b"[::ffff:0.0.0.0]:9988", b"http", b"::ffff:0.0.0.0", 9988, None],
+            # ipv6+scheme
+            [
+                b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:effe]",
+                b"https",
+                b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
+                1080,
+                None,
+            ],
+            [
+                b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:1234]",
+                b"https",
+                b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
+                1080,
+                None,
+            ],
+            [b"https://[::1]", b"https", b"::1", 1080, None],
+            [b"https://[::ffff:0.0.0.0]", b"https", b"::ffff:0.0.0.0", 1080, None],
+            # ipv6+scheme+port
+            [
+                b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:effe]:9988",
+                b"https",
+                b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
+                9988,
+                None,
+            ],
+            [
+                b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:1234]:9988",
+                b"https",
+                b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
+                9988,
+                None,
+            ],
+            [b"https://[::1]:9988", b"https", b"::1", 9988, None],
+            # with credentials
+            [
+                b"https://user:pass@1.2.3.4:9988",
+                b"https",
+                b"1.2.3.4",
+                9988,
+                b"user:pass",
+            ],
+            [b"user:pass@1.2.3.4:9988", b"http", b"1.2.3.4", 9988, b"user:pass"],
+            [
+                b"https://user:pass@proxy.local:9988",
+                b"https",
+                b"proxy.local",
+                9988,
+                b"user:pass",
+            ],
+            [
+                b"user:pass@proxy.local:9988",
+                b"http",
+                b"proxy.local",
+                9988,
+                b"user:pass",
+            ],
+        ]
+    )
+    def test_parse_proxy(
+        self,
+        proxy_string: bytes,
+        expected_scheme: bytes,
+        expected_hostname: bytes,
+        expected_port: int,
+        expected_credentials: Optional[bytes],
+    ):
+        """
+        Tests that a given proxy URL will be broken into the components.
+        Args:
+            proxy_string: The proxy connection string.
+            expected_scheme: Expected value of proxy scheme.
+            expected_hostname: Expected value of proxy hostname.
+            expected_port: Expected value of proxy port.
+            expected_credentials: Expected value of credentials.
+                Must be in form '<username>:<password>' or None
+        """
+        proxy_cred = None
+        if expected_credentials:
+            proxy_cred = ProxyCredentials(expected_credentials)
+        self.assertEqual(
+            (
+                expected_scheme,
+                expected_hostname,
+                expected_port,
+                proxy_cred,
+            ),
+            parse_proxy(proxy_string),
+        )
+
+
 class MatrixFederationAgentTests(TestCase):
     def setUp(self):
         self.reactor = ThreadedMemoryReactorClock()
 
     def _make_connection(
-        self, client_factory, server_factory, ssl=False, expected_sni=None
-    ):
+        self,
+        client_factory: IProtocolFactory,
+        server_factory: IProtocolFactory,
+        ssl: bool = False,
+        expected_sni: Optional[bytes] = None,
+        tls_sanlist: Optional[Iterable[bytes]] = None,
+    ) -> IProtocol:
         """Builds a test server, and completes the outgoing client connection
 
         Args:
-            client_factory (interfaces.IProtocolFactory): the the factory that the
+            client_factory: the the factory that the
                 application is trying to use to make the outbound connection. We will
                 invoke it to build the client Protocol
 
-            server_factory (interfaces.IProtocolFactory): a factory to build the
+            server_factory: a factory to build the
                 server-side protocol
 
-            ssl (bool): If true, we will expect an ssl connection and wrap
+            ssl: If true, we will expect an ssl connection and wrap
                 server_factory with a TLSMemoryBIOFactory
 
-            expected_sni (bytes|None): the expected SNI value
+            expected_sni: the expected SNI value
+
+            tls_sanlist: list of SAN entries for the TLS cert presented by the server.
+                 Defaults to [b'DNS:test.com']
 
         Returns:
-            IProtocol: the server Protocol returned by server_factory
+            the server Protocol returned by server_factory
         """
         if ssl:
-            server_factory = _wrap_server_factory_for_tls(server_factory)
+            server_factory = _wrap_server_factory_for_tls(server_factory, tls_sanlist)
 
         server_protocol = server_factory.buildProtocol(None)
 
@@ -99,22 +277,28 @@ def _make_connection(
             self.assertEqual(
                 server_name,
                 expected_sni,
-                "Expected SNI %s but got %s" % (expected_sni, server_name),
+                f"Expected SNI {expected_sni!s} but got {server_name!s}",
             )
 
         return http_protocol
 
-    def _test_request_direct_connection(self, agent, scheme, hostname, path):
+    def _test_request_direct_connection(
+        self,
+        agent: ProxyAgent,
+        scheme: bytes,
+        hostname: bytes,
+        path: bytes,
+    ):
         """Runs a test case for a direct connection not going through a proxy.
 
         Args:
-            agent (ProxyAgent): the proxy agent being tested
+            agent: the proxy agent being tested
 
-            scheme (bytes): expected to be either "http" or "https"
+            scheme: expected to be either "http" or "https"
 
-            hostname (bytes): the hostname to connect to in the test
+            hostname: the hostname to connect to in the test
 
-            path (bytes): the path to connect to in the test
+            path: the path to connect to in the test
         """
         is_https = scheme == b"https"
 
@@ -206,7 +390,98 @@ def test_https_request_via_no_proxy_star(self):
 
     @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "unused.com"})
     def test_http_request_via_proxy(self):
-        agent = ProxyAgent(self.reactor, use_proxy=True)
+        """
+        Tests that requests can be made through a proxy.
+        """
+        self._do_http_request_via_proxy(
+            expect_proxy_ssl=False, expected_auth_credentials=None
+        )
+
+    @patch.dict(
+        os.environ,
+        {"http_proxy": "bob:pinkponies@proxy.com:8888", "no_proxy": "unused.com"},
+    )
+    def test_http_request_via_proxy_with_auth(self):
+        """
+        Tests that authenticated requests can be made through a proxy.
+        """
+        self._do_http_request_via_proxy(
+            expect_proxy_ssl=False, expected_auth_credentials=b"bob:pinkponies"
+        )
+
+    @patch.dict(
+        os.environ, {"http_proxy": "https://proxy.com:8888", "no_proxy": "unused.com"}
+    )
+    def test_http_request_via_https_proxy(self):
+        self._do_http_request_via_proxy(
+            expect_proxy_ssl=True, expected_auth_credentials=None
+        )
+
+    @patch.dict(
+        os.environ,
+        {
+            "http_proxy": "https://bob:pinkponies@proxy.com:8888",
+            "no_proxy": "unused.com",
+        },
+    )
+    def test_http_request_via_https_proxy_with_auth(self):
+        self._do_http_request_via_proxy(
+            expect_proxy_ssl=True, expected_auth_credentials=b"bob:pinkponies"
+        )
+
+    @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"})
+    def test_https_request_via_proxy(self):
+        """Tests that TLS-encrypted requests can be made through a proxy"""
+        self._do_https_request_via_proxy(
+            expect_proxy_ssl=False, expected_auth_credentials=None
+        )
+
+    @patch.dict(
+        os.environ,
+        {"https_proxy": "bob:pinkponies@proxy.com", "no_proxy": "unused.com"},
+    )
+    def test_https_request_via_proxy_with_auth(self):
+        """Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
+        self._do_https_request_via_proxy(
+            expect_proxy_ssl=False, expected_auth_credentials=b"bob:pinkponies"
+        )
+
+    @patch.dict(
+        os.environ, {"https_proxy": "https://proxy.com", "no_proxy": "unused.com"}
+    )
+    def test_https_request_via_https_proxy(self):
+        """Tests that TLS-encrypted requests can be made through a proxy"""
+        self._do_https_request_via_proxy(
+            expect_proxy_ssl=True, expected_auth_credentials=None
+        )
+
+    @patch.dict(
+        os.environ,
+        {"https_proxy": "https://bob:pinkponies@proxy.com", "no_proxy": "unused.com"},
+    )
+    def test_https_request_via_https_proxy_with_auth(self):
+        """Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
+        self._do_https_request_via_proxy(
+            expect_proxy_ssl=True, expected_auth_credentials=b"bob:pinkponies"
+        )
+
+    def _do_http_request_via_proxy(
+        self,
+        expect_proxy_ssl: bool = False,
+        expected_auth_credentials: Optional[bytes] = None,
+    ):
+        """Send a http request via an agent and check that it is correctly received at
+            the proxy. The proxy can use either http or https.
+        Args:
+            expect_proxy_ssl: True if we expect the request to connect via https to proxy
+            expected_auth_credentials: credentials to authenticate at proxy
+        """
+        if expect_proxy_ssl:
+            agent = ProxyAgent(
+                self.reactor, use_proxy=True, contextFactory=get_test_https_policy()
+            )
+        else:
+            agent = ProxyAgent(self.reactor, use_proxy=True)
 
         self.reactor.lookups["proxy.com"] = "1.2.3.5"
         d = agent.request(b"GET", b"http://test.com")
@@ -220,7 +495,11 @@ def test_http_request_via_proxy(self):
 
         # make a test server, and wire up the client
         http_server = self._make_connection(
-            client_factory, _get_test_protocol_factory()
+            client_factory,
+            _get_test_protocol_factory(),
+            ssl=expect_proxy_ssl,
+            tls_sanlist=[b"DNS:proxy.com"] if expect_proxy_ssl else None,
+            expected_sni=b"proxy.com" if expect_proxy_ssl else None,
         )
 
         # the FakeTransport is async, so we need to pump the reactor
@@ -230,6 +509,23 @@ def test_http_request_via_proxy(self):
         self.assertEqual(len(http_server.requests), 1)
 
         request = http_server.requests[0]
+
+        # Check whether auth credentials have been supplied to the proxy
+        proxy_auth_header_values = request.requestHeaders.getRawHeaders(
+            b"Proxy-Authorization"
+        )
+
+        if expected_auth_credentials is not None:
+            # Compute the correct header value for Proxy-Authorization
+            encoded_credentials = base64.b64encode(expected_auth_credentials)
+            expected_header_value = b"Basic " + encoded_credentials
+
+            # Validate the header's value
+            self.assertIn(expected_header_value, proxy_auth_header_values)
+        else:
+            # Check that the Proxy-Authorization header has not been supplied to the proxy
+            self.assertIsNone(proxy_auth_header_values)
+
         self.assertEqual(request.method, b"GET")
         self.assertEqual(request.path, b"http://test.com")
         self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"])
@@ -242,23 +538,17 @@ def test_http_request_via_proxy(self):
         body = self.successResultOf(treq.content(resp))
         self.assertEqual(body, b"result")
 
-    @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"})
-    def test_https_request_via_proxy(self):
-        """Tests that TLS-encrypted requests can be made through a proxy"""
-        self._do_https_request_via_proxy(auth_credentials=None)
-
-    @patch.dict(
-        os.environ,
-        {"https_proxy": "bob:pinkponies@proxy.com", "no_proxy": "unused.com"},
-    )
-    def test_https_request_via_proxy_with_auth(self):
-        """Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
-        self._do_https_request_via_proxy(auth_credentials="bob:pinkponies")
-
     def _do_https_request_via_proxy(
         self,
-        auth_credentials: Optional[str] = None,
+        expect_proxy_ssl: bool = False,
+        expected_auth_credentials: Optional[bytes] = None,
     ):
+        """Send a https request via an agent and check that it is correctly received at
+            the proxy and client. The proxy can use either http or https.
+        Args:
+            expect_proxy_ssl: True if we expect the request to connect via https to proxy
+            expected_auth_credentials: credentials to authenticate at proxy
+        """
         agent = ProxyAgent(
             self.reactor,
             contextFactory=get_test_https_policy(),
@@ -275,18 +565,15 @@ def _do_https_request_via_proxy(
         self.assertEqual(host, "1.2.3.5")
         self.assertEqual(port, 1080)
 
-        # make a test HTTP server, and wire up the client
+        # make a test server to act as the proxy, and wire up the client
         proxy_server = self._make_connection(
-            client_factory, _get_test_protocol_factory()
+            client_factory,
+            _get_test_protocol_factory(),
+            ssl=expect_proxy_ssl,
+            tls_sanlist=[b"DNS:proxy.com"] if expect_proxy_ssl else None,
+            expected_sni=b"proxy.com" if expect_proxy_ssl else None,
         )
-
-        # fish the transports back out so that we can do the old switcheroo
-        s2c_transport = proxy_server.transport
-        client_protocol = s2c_transport.other
-        c2s_transport = client_protocol.transport
-
-        # the FakeTransport is async, so we need to pump the reactor
-        self.reactor.advance(0)
+        assert isinstance(proxy_server, HTTPChannel)
 
         # now there should be a pending CONNECT request
         self.assertEqual(len(proxy_server.requests), 1)
@@ -300,9 +587,9 @@ def _do_https_request_via_proxy(
             b"Proxy-Authorization"
         )
 
-        if auth_credentials is not None:
+        if expected_auth_credentials is not None:
             # Compute the correct header value for Proxy-Authorization
-            encoded_credentials = base64.b64encode(b"bob:pinkponies")
+            encoded_credentials = base64.b64encode(expected_auth_credentials)
             expected_header_value = b"Basic " + encoded_credentials
 
             # Validate the header's value
@@ -314,31 +601,49 @@ def _do_https_request_via_proxy(
         # tell the proxy server not to close the connection
         proxy_server.persistent = True
 
-        # this just stops the http Request trying to do a chunked response
-        # request.setHeader(b"Content-Length", b"0")
         request.finish()
 
-        # now we can replace the proxy channel with a new, SSL-wrapped HTTP channel
-        ssl_factory = _wrap_server_factory_for_tls(_get_test_protocol_factory())
-        ssl_protocol = ssl_factory.buildProtocol(None)
-        http_server = ssl_protocol.wrappedProtocol
+        # now we make another test server to act as the upstream HTTP server.
+        server_ssl_protocol = _wrap_server_factory_for_tls(
+            _get_test_protocol_factory()
+        ).buildProtocol(None)
 
-        ssl_protocol.makeConnection(
-            FakeTransport(client_protocol, self.reactor, ssl_protocol)
-        )
-        c2s_transport.other = ssl_protocol
+        # Tell the HTTP server to send outgoing traffic back via the proxy's transport.
+        proxy_server_transport = proxy_server.transport
+        server_ssl_protocol.makeConnection(proxy_server_transport)
+
+        # ... and replace the protocol on the proxy's transport with the
+        # TLSMemoryBIOProtocol for the test server, so that incoming traffic
+        # to the proxy gets sent over to the HTTP(s) server.
+        #
+        # This needs a bit of gut-wrenching, which is different depending on whether
+        # the proxy is using TLS or not.
+        #
+        # (an alternative, possibly more elegant, approach would be to use a custom
+        # Protocol to implement the proxy, which starts out by forwarding to an
+        # HTTPChannel (to implement the CONNECT command) and can then be switched
+        # into a mode where it forwards its traffic to another Protocol.)
+        if expect_proxy_ssl:
+            assert isinstance(proxy_server_transport, TLSMemoryBIOProtocol)
+            proxy_server_transport.wrappedProtocol = server_ssl_protocol
+        else:
+            assert isinstance(proxy_server_transport, FakeTransport)
+            client_protocol = proxy_server_transport.other
+            c2s_transport = client_protocol.transport
+            c2s_transport.other = server_ssl_protocol
 
         self.reactor.advance(0)
 
-        server_name = ssl_protocol._tlsConnection.get_servername()
+        server_name = server_ssl_protocol._tlsConnection.get_servername()
         expected_sni = b"test.com"
         self.assertEqual(
             server_name,
             expected_sni,
-            "Expected SNI %s but got %s" % (expected_sni, server_name),
+            f"Expected SNI {expected_sni!s} but got {server_name!s}",
         )
 
         # now there should be a pending request
+        http_server = server_ssl_protocol.wrappedProtocol
         self.assertEqual(len(http_server.requests), 1)
 
         request = http_server.requests[0]
@@ -472,7 +777,7 @@ def test_https_request_via_uppercase_proxy_with_blacklist(self):
         self.assertEqual(
             server_name,
             expected_sni,
-            "Expected SNI %s but got %s" % (expected_sni, server_name),
+            f"Expected SNI {expected_sni!s} but got {server_name!s}",
         )
 
         # now there should be a pending request
@@ -491,16 +796,48 @@ def test_https_request_via_uppercase_proxy_with_blacklist(self):
         body = self.successResultOf(treq.content(resp))
         self.assertEqual(body, b"result")
 
+    @patch.dict(os.environ, {"http_proxy": "proxy.com:8888"})
+    def test_proxy_with_no_scheme(self):
+        http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
+        self.assertIsInstance(http_proxy_agent.http_proxy_endpoint, HostnameEndpoint)
+        self.assertEqual(http_proxy_agent.http_proxy_endpoint._hostStr, "proxy.com")
+        self.assertEqual(http_proxy_agent.http_proxy_endpoint._port, 8888)
+
+    @patch.dict(os.environ, {"http_proxy": "socks://proxy.com:8888"})
+    def test_proxy_with_unsupported_scheme(self):
+        with self.assertRaises(ValueError):
+            ProxyAgent(self.reactor, use_proxy=True)
+
+    @patch.dict(os.environ, {"http_proxy": "http://proxy.com:8888"})
+    def test_proxy_with_http_scheme(self):
+        http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
+        self.assertIsInstance(http_proxy_agent.http_proxy_endpoint, HostnameEndpoint)
+        self.assertEqual(http_proxy_agent.http_proxy_endpoint._hostStr, "proxy.com")
+        self.assertEqual(http_proxy_agent.http_proxy_endpoint._port, 8888)
+
+    @patch.dict(os.environ, {"http_proxy": "https://proxy.com:8888"})
+    def test_proxy_with_https_scheme(self):
+        https_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
+        self.assertIsInstance(https_proxy_agent.http_proxy_endpoint, _WrapperEndpoint)
+        self.assertEqual(
+            https_proxy_agent.http_proxy_endpoint._wrappedEndpoint._hostStr, "proxy.com"
+        )
+        self.assertEqual(
+            https_proxy_agent.http_proxy_endpoint._wrappedEndpoint._port, 8888
+        )
+
 
-def _wrap_server_factory_for_tls(factory, sanlist=None):
+def _wrap_server_factory_for_tls(
+    factory: IProtocolFactory, sanlist: Iterable[bytes] = None
+) -> IProtocolFactory:
     """Wrap an existing Protocol Factory with a test TLSMemoryBIOFactory
 
     The resultant factory will create a TLS server which presents a certificate
     signed by our test CA, valid for the domains in `sanlist`
 
     Args:
-        factory (interfaces.IProtocolFactory): protocol factory to wrap
-        sanlist (iterable[bytes]): list of domains the cert should be valid for
+        factory: protocol factory to wrap
+        sanlist: list of domains the cert should be valid for
 
     Returns:
         interfaces.IProtocolFactory
@@ -514,7 +851,7 @@ def _wrap_server_factory_for_tls(factory, sanlist=None):
     )
 
 
-def _get_test_protocol_factory():
+def _get_test_protocol_factory() -> IProtocolFactory:
     """Get a protocol Factory which will build an HTTPChannel
 
     Returns:
@@ -528,6 +865,6 @@ def _get_test_protocol_factory():
     return server_factory
 
 
-def _log_request(request):
+def _log_request(request: str):
     """Implements Factory.log, which is expected by Request.finish"""
-    logger.info("Completed request %s", request)
+    logger.info(f"Completed request {request}")
diff --git a/tests/http/test_servlet.py b/tests/http/test_servlet.py
index 45089158ce..a80bfb9f4e 100644
--- a/tests/http/test_servlet.py
+++ b/tests/http/test_servlet.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +13,7 @@
 # limitations under the License.
 import json
 from io import BytesIO
-
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.api.errors import SynapseError
 from synapse.http.servlet import (
diff --git a/tests/http/test_simple_client.py b/tests/http/test_simple_client.py
index a1cf0862d4..c85a3665c1 100644
--- a/tests/http/test_simple_client.py
+++ b/tests/http/test_simple_client.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
 
 from netaddr import IPSet
 
diff --git a/tests/http/test_site.py b/tests/http/test_site.py
new file mode 100644
index 0000000000..8c13b4f693
--- /dev/null
+++ b/tests/http/test_site.py
@@ -0,0 +1,83 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet.address import IPv6Address
+from twisted.test.proto_helpers import StringTransport
+
+from synapse.app.homeserver import SynapseHomeServer
+
+from tests.unittest import HomeserverTestCase
+
+
+class SynapseRequestTestCase(HomeserverTestCase):
+    def make_homeserver(self, reactor, clock):
+        return self.setup_test_homeserver(homeserver_to_use=SynapseHomeServer)
+
+    def test_large_request(self):
+        """overlarge HTTP requests should be rejected"""
+        self.hs.start_listening()
+
+        # find the HTTP server which is configured to listen on port 0
+        (port, factory, _backlog, interface) = self.reactor.tcpServers[0]
+        self.assertEqual(interface, "::")
+        self.assertEqual(port, 0)
+
+        # as a control case, first send a regular request.
+
+        # complete the connection and wire it up to a fake transport
+        client_address = IPv6Address("TCP", "::1", "2345")
+        protocol = factory.buildProtocol(client_address)
+        transport = StringTransport()
+        protocol.makeConnection(transport)
+
+        protocol.dataReceived(
+            b"POST / HTTP/1.1\r\n"
+            b"Connection: close\r\n"
+            b"Transfer-Encoding: chunked\r\n"
+            b"\r\n"
+            b"0\r\n"
+            b"\r\n"
+        )
+
+        while not transport.disconnecting:
+            self.reactor.advance(1)
+
+        # we should get a 404
+        self.assertRegex(transport.value().decode(), r"^HTTP/1\.1 404 ")
+
+        # now send an oversized request
+        protocol = factory.buildProtocol(client_address)
+        transport = StringTransport()
+        protocol.makeConnection(transport)
+
+        protocol.dataReceived(
+            b"POST / HTTP/1.1\r\n"
+            b"Connection: close\r\n"
+            b"Transfer-Encoding: chunked\r\n"
+            b"\r\n"
+        )
+
+        # we deliberately send all the data in one big chunk, to ensure that
+        # twisted isn't buffering the data in the chunked transfer decoder.
+        # we start with the chunk size, in hex. (We won't actually send this much)
+        protocol.dataReceived(b"10000000\r\n")
+        sent = 0
+        while not transport.disconnected:
+            self.assertLess(sent, 0x10000000, "connection did not drop")
+            protocol.dataReceived(b"\0" * 1024)
+            sent += 1024
+
+        # default max upload size is 50M, so it should drop on the next buffer after
+        # that.
+        self.assertEqual(sent, 50 * 1024 * 1024 + 1024)
diff --git a/tests/logging/__init__.py b/tests/logging/__init__.py
index a58d51441c..1acf5666a8 100644
--- a/tests/logging/__init__.py
+++ b/tests/logging/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/logging/test_remote_handler.py b/tests/logging/test_remote_handler.py
index 4bc27a1d7d..b0d046fe00 100644
--- a/tests/logging/test_remote_handler.py
+++ b/tests/logging/test_remote_handler.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/logging/test_terse_json.py b/tests/logging/test_terse_json.py
index 48a74e2eee..1160716929 100644
--- a/tests/logging/test_terse_json.py
+++ b/tests/logging/test_terse_json.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,15 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import json
 import logging
-from io import StringIO
+from io import BytesIO, StringIO
+from unittest.mock import Mock, patch
+
+from twisted.web.server import Request
 
+from synapse.http.site import SynapseRequest
 from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter
 from synapse.logging.context import LoggingContext, LoggingContextFilter
 
 from tests.logging import LoggerCleanupMixin
+from tests.server import FakeChannel
 from tests.unittest import TestCase
 
 
@@ -120,7 +123,7 @@ def test_with_context(self):
         handler.addFilter(LoggingContextFilter())
         logger = self.get_logger(handler)
 
-        with LoggingContext(request="test"):
+        with LoggingContext("name"):
             logger.info("Hello there, %s!", "wally")
 
         log = self.get_log_line()
@@ -134,4 +137,63 @@ def test_with_context(self):
         ]
         self.assertCountEqual(log.keys(), expected_log_keys)
         self.assertEqual(log["log"], "Hello there, wally!")
-        self.assertEqual(log["request"], "test")
+        self.assertEqual(log["request"], "name")
+
+    def test_with_request_context(self):
+        """
+        Information from the logging context request should be added to the JSON response.
+        """
+        handler = logging.StreamHandler(self.output)
+        handler.setFormatter(JsonFormatter())
+        handler.addFilter(LoggingContextFilter())
+        logger = self.get_logger(handler)
+
+        # A full request isn't needed here.
+        site = Mock(spec=["site_tag", "server_version_string", "getResourceFor"])
+        site.site_tag = "test-site"
+        site.server_version_string = "Server v1"
+        request = SynapseRequest(FakeChannel(site, None))
+        # Call requestReceived to finish instantiating the object.
+        request.content = BytesIO()
+        # Partially skip some of the internal processing of SynapseRequest.
+        request._started_processing = Mock()
+        request.request_metrics = Mock(spec=["name"])
+        with patch.object(Request, "render"):
+            request.requestReceived(b"POST", b"/_matrix/client/versions", b"1.1")
+
+        # Also set the requester to ensure the processing works.
+        request.requester = "@foo:test"
+
+        with LoggingContext(
+            request.get_request_id(), parent_context=request.logcontext
+        ):
+            logger.info("Hello there, %s!", "wally")
+
+        log = self.get_log_line()
+
+        # The terse logger includes additional request information, if possible.
+        expected_log_keys = [
+            "log",
+            "level",
+            "namespace",
+            "request",
+            "ip_address",
+            "site_tag",
+            "requester",
+            "authenticated_entity",
+            "method",
+            "url",
+            "protocol",
+            "user_agent",
+        ]
+        self.assertCountEqual(log.keys(), expected_log_keys)
+        self.assertEqual(log["log"], "Hello there, wally!")
+        self.assertTrue(log["request"].startswith("POST-"))
+        self.assertEqual(log["ip_address"], "127.0.0.1")
+        self.assertEqual(log["site_tag"], "test-site")
+        self.assertEqual(log["requester"], "@foo:test")
+        self.assertEqual(log["authenticated_entity"], "@foo:test")
+        self.assertEqual(log["method"], "POST")
+        self.assertEqual(log["url"], "/_matrix/client/versions")
+        self.assertEqual(log["protocol"], "1.1")
+        self.assertEqual(log["user_agent"], "")
diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py
index edacd1b566..7dd519cd44 100644
--- a/tests/module_api/test_api.py
+++ b/tests/module_api/test_api.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,14 +11,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
 
+from twisted.internet import defer
+
+from synapse.api.constants import EduTypes
 from synapse.events import EventBase
+from synapse.federation.units import Transaction
+from synapse.handlers.presence import UserPresenceState
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, presence, room
 from synapse.types import create_requester
 
-from tests.unittest import HomeserverTestCase
+from tests.events.test_presence_router import send_presence_update, sync_presence
+from tests.replication._base import BaseMultiWorkerStreamTestCase
+from tests.test_utils.event_injection import inject_member_event
+from tests.unittest import HomeserverTestCase, override_config
+from tests.utils import USE_POSTGRES_FOR_TESTS
 
 
 class ModuleApiTestCase(HomeserverTestCase):
@@ -27,12 +35,19 @@ class ModuleApiTestCase(HomeserverTestCase):
         admin.register_servlets,
         login.register_servlets,
         room.register_servlets,
+        presence.register_servlets,
     ]
 
     def prepare(self, reactor, clock, homeserver):
         self.store = homeserver.get_datastore()
         self.module_api = homeserver.get_module_api()
         self.event_creation_handler = homeserver.get_event_creation_handler()
+        self.sync_handler = homeserver.get_sync_handler()
+
+    def make_homeserver(self, reactor, clock):
+        return self.setup_test_homeserver(
+            federation_transport_client=Mock(spec=["send_transaction"]),
+        )
 
     def test_can_register_user(self):
         """Tests that an external module can register a user"""
@@ -64,6 +79,16 @@ def test_can_register_user(self):
         displayname = self.get_success(self.store.get_profile_displayname("bob"))
         self.assertEqual(displayname, "Bobberino")
 
+    def test_get_userinfo_by_id(self):
+        user_id = self.register_user("alice", "1234")
+        found_user = self.get_success(self.module_api.get_userinfo_by_id(user_id))
+        self.assertEqual(found_user.user_id.to_string(), user_id)
+        self.assertIdentical(found_user.is_admin, False)
+
+    def test_get_userinfo_by_id__no_user_found(self):
+        found_user = self.get_success(self.module_api.get_userinfo_by_id("@alice:test"))
+        self.assertIsNone(found_user)
+
     def test_sending_events_into_room(self):
         """Tests that a module can send events into a room"""
         # Mock out create_and_send_nonmember_event to check whether events are being sent
@@ -85,9 +110,9 @@ def test_sending_events_into_room(self):
             "content": content,
             "sender": user_id,
         }
-        event = self.get_success(
+        event: EventBase = self.get_success(
             self.module_api.create_and_send_event_into_room(event_dict)
-        )  # type: EventBase
+        )
         self.assertEqual(event.sender, user_id)
         self.assertEqual(event.type, "m.room.message")
         self.assertEqual(event.room_id, room_id)
@@ -121,9 +146,9 @@ def test_sending_events_into_room(self):
             "sender": user_id,
             "state_key": "",
         }
-        event = self.get_success(
+        event: EventBase = self.get_success(
             self.module_api.create_and_send_event_into_room(event_dict)
-        )  # type: EventBase
+        )
         self.assertEqual(event.sender, user_id)
         self.assertEqual(event.type, "m.room.power_levels")
         self.assertEqual(event.room_id, room_id)
@@ -205,3 +230,286 @@ def test_public_rooms(self):
             )
         )
         self.assertFalse(is_in_public_rooms)
+
+    def test_send_local_online_presence_to(self):
+        # Test sending local online presence to users from the main process
+        _test_sending_local_online_presence_to_local_user(self, test_with_workers=False)
+
+    @override_config({"send_federation": True})
+    def test_send_local_online_presence_to_federation(self):
+        """Tests that send_local_presence_to_users sends local online presence to remote users."""
+        # Create a user who will send presence updates
+        self.presence_sender_id = self.register_user("presence_sender1", "monkey")
+        self.presence_sender_tok = self.login("presence_sender1", "monkey")
+
+        # And a room they're a part of
+        room_id = self.helper.create_room_as(
+            self.presence_sender_id,
+            tok=self.presence_sender_tok,
+        )
+
+        # Mark them as online
+        send_presence_update(
+            self,
+            self.presence_sender_id,
+            self.presence_sender_tok,
+            "online",
+            "I'm online!",
+        )
+
+        # Make up a remote user to send presence to
+        remote_user_id = "@far_away_person:island"
+
+        # Create a join membership event for the remote user into the room.
+        # This allows presence information to flow from one user to the other.
+        self.get_success(
+            inject_member_event(
+                self.hs,
+                room_id,
+                sender=remote_user_id,
+                target=remote_user_id,
+                membership="join",
+            )
+        )
+
+        # The remote user would have received the existing room members' presence
+        # when they joined the room.
+        #
+        # Thus we reset the mock, and try sending online local user
+        # presence again
+        self.hs.get_federation_transport_client().send_transaction.reset_mock()
+
+        # Broadcast local user online presence
+        self.get_success(
+            self.module_api.send_local_online_presence_to([remote_user_id])
+        )
+
+        # Check that a presence update was sent as part of a federation transaction
+        found_update = False
+        calls = (
+            self.hs.get_federation_transport_client().send_transaction.call_args_list
+        )
+        for call in calls:
+            call_args = call[0]
+            federation_transaction: Transaction = call_args[0]
+
+            # Get the sent EDUs in this transaction
+            edus = federation_transaction.get_dict()["edus"]
+
+            for edu in edus:
+                # Make sure we're only checking presence-type EDUs
+                if edu["edu_type"] != EduTypes.Presence:
+                    continue
+
+                # EDUs can contain multiple presence updates
+                for presence_update in edu["content"]["push"]:
+                    if presence_update["user_id"] == self.presence_sender_id:
+                        found_update = True
+
+        self.assertTrue(found_update)
+
+
+class ModuleApiWorkerTestCase(BaseMultiWorkerStreamTestCase):
+    """For testing ModuleApi functionality in a multi-worker setup"""
+
+    # Testing stream ID replication from the main to worker processes requires postgres
+    # (due to needing `MultiWriterIdGenerator`).
+    if not USE_POSTGRES_FOR_TESTS:
+        skip = "Requires Postgres"
+
+    servlets = [
+        admin.register_servlets,
+        login.register_servlets,
+        room.register_servlets,
+        presence.register_servlets,
+    ]
+
+    def default_config(self):
+        conf = super().default_config()
+        conf["redis"] = {"enabled": "true"}
+        conf["stream_writers"] = {"presence": ["presence_writer"]}
+        conf["instance_map"] = {
+            "presence_writer": {"host": "testserv", "port": 1001},
+        }
+        return conf
+
+    def prepare(self, reactor, clock, homeserver):
+        self.module_api = homeserver.get_module_api()
+        self.sync_handler = homeserver.get_sync_handler()
+
+    def test_send_local_online_presence_to_workers(self):
+        # Test sending local online presence to users from a worker process
+        _test_sending_local_online_presence_to_local_user(self, test_with_workers=True)
+
+
+def _test_sending_local_online_presence_to_local_user(
+    test_case: HomeserverTestCase, test_with_workers: bool = False
+):
+    """Tests that send_local_presence_to_users sends local online presence to local users.
+
+    This simultaneously tests two different usecases:
+        * Testing that this method works when either called from a worker or the main process.
+            - We test this by calling this method from both a TestCase that runs in monolith mode, and one that
+              runs with a main and generic_worker.
+        * Testing that multiple devices syncing simultaneously will all receive a snapshot of local,
+            online presence - but only once per device.
+
+    Args:
+        test_with_workers: If True, this method will call ModuleApi.send_local_online_presence_to on a
+            worker process. The test users will still sync with the main process. The purpose of testing
+            with a worker is to check whether a Synapse module running on a worker can inform other workers/
+            the main process that they should include additional presence when a user next syncs.
+    """
+    if test_with_workers:
+        # Create a worker process to make module_api calls against
+        worker_hs = test_case.make_worker_hs(
+            "synapse.app.generic_worker", {"worker_name": "presence_writer"}
+        )
+
+    # Create a user who will send presence updates
+    test_case.presence_receiver_id = test_case.register_user(
+        "presence_receiver1", "monkey"
+    )
+    test_case.presence_receiver_tok = test_case.login("presence_receiver1", "monkey")
+
+    # And another user that will send presence updates out
+    test_case.presence_sender_id = test_case.register_user("presence_sender2", "monkey")
+    test_case.presence_sender_tok = test_case.login("presence_sender2", "monkey")
+
+    # Put them in a room together so they will receive each other's presence updates
+    room_id = test_case.helper.create_room_as(
+        test_case.presence_receiver_id,
+        tok=test_case.presence_receiver_tok,
+    )
+    test_case.helper.join(
+        room_id, test_case.presence_sender_id, tok=test_case.presence_sender_tok
+    )
+
+    # Presence sender comes online
+    send_presence_update(
+        test_case,
+        test_case.presence_sender_id,
+        test_case.presence_sender_tok,
+        "online",
+        "I'm online!",
+    )
+
+    # Presence receiver should have received it
+    presence_updates, sync_token = sync_presence(
+        test_case, test_case.presence_receiver_id
+    )
+    test_case.assertEqual(len(presence_updates), 1)
+
+    presence_update: UserPresenceState = presence_updates[0]
+    test_case.assertEqual(presence_update.user_id, test_case.presence_sender_id)
+    test_case.assertEqual(presence_update.state, "online")
+
+    if test_with_workers:
+        # Replicate the current sync presence token from the main process to the worker process.
+        # We need to do this so that the worker process knows the current presence stream ID to
+        # insert into the database when we call ModuleApi.send_local_online_presence_to.
+        test_case.replicate()
+
+    # Syncing again should result in no presence updates
+    presence_updates, sync_token = sync_presence(
+        test_case, test_case.presence_receiver_id, sync_token
+    )
+    test_case.assertEqual(len(presence_updates), 0)
+
+    # We do an (initial) sync with a second "device" now, getting a new sync token.
+    # We'll use this in a moment.
+    _, sync_token_second_device = sync_presence(
+        test_case, test_case.presence_receiver_id
+    )
+
+    # Determine on which process (main or worker) to call ModuleApi.send_local_online_presence_to on
+    if test_with_workers:
+        module_api_to_use = worker_hs.get_module_api()
+    else:
+        module_api_to_use = test_case.module_api
+
+    # Trigger sending local online presence. We expect this information
+    # to be saved to the database where all processes can access it.
+    # Note that we're syncing via the master.
+    d = module_api_to_use.send_local_online_presence_to(
+        [
+            test_case.presence_receiver_id,
+        ]
+    )
+    d = defer.ensureDeferred(d)
+
+    if test_with_workers:
+        # In order for the required presence_set_state replication request to occur between the
+        # worker and main process, we need to pump the reactor. Otherwise, the coordinator that
+        # reads the request on the main process won't do so, and the request will time out.
+        while not d.called:
+            test_case.reactor.advance(0.1)
+
+    test_case.get_success(d)
+
+    # The presence receiver should have received online presence again.
+    presence_updates, sync_token = sync_presence(
+        test_case, test_case.presence_receiver_id, sync_token
+    )
+    test_case.assertEqual(len(presence_updates), 1)
+
+    presence_update: UserPresenceState = presence_updates[0]
+    test_case.assertEqual(presence_update.user_id, test_case.presence_sender_id)
+    test_case.assertEqual(presence_update.state, "online")
+
+    # We attempt to sync with the second sync token we received above - just to check that
+    # multiple syncing devices will each receive the necessary online presence.
+    presence_updates, sync_token_second_device = sync_presence(
+        test_case, test_case.presence_receiver_id, sync_token_second_device
+    )
+    test_case.assertEqual(len(presence_updates), 1)
+
+    presence_update: UserPresenceState = presence_updates[0]
+    test_case.assertEqual(presence_update.user_id, test_case.presence_sender_id)
+    test_case.assertEqual(presence_update.state, "online")
+
+    # However, if we now sync with either "device", we won't receive another burst of online presence
+    # until the API is called again sometime in the future
+    presence_updates, sync_token = sync_presence(
+        test_case, test_case.presence_receiver_id, sync_token
+    )
+
+    # Now we check that we don't receive *offline* updates using ModuleApi.send_local_online_presence_to.
+
+    # Presence sender goes offline
+    send_presence_update(
+        test_case,
+        test_case.presence_sender_id,
+        test_case.presence_sender_tok,
+        "offline",
+        "I slink back into the darkness.",
+    )
+
+    # Presence receiver should have received the updated, offline state
+    presence_updates, sync_token = sync_presence(
+        test_case, test_case.presence_receiver_id, sync_token
+    )
+    test_case.assertEqual(len(presence_updates), 1)
+
+    # Now trigger sending local online presence.
+    d = module_api_to_use.send_local_online_presence_to(
+        [
+            test_case.presence_receiver_id,
+        ]
+    )
+    d = defer.ensureDeferred(d)
+
+    if test_with_workers:
+        # In order for the required presence_set_state replication request to occur between the
+        # worker and main process, we need to pump the reactor. Otherwise, the coordinator that
+        # reads the request on the main process won't do so, and the request will time out.
+        while not d.called:
+            test_case.reactor.advance(0.1)
+
+    test_case.get_success(d)
+
+    # Presence receiver should *not* have received offline state
+    presence_updates, sync_token = sync_presence(
+        test_case, test_case.presence_receiver_id, sync_token
+    )
+    test_case.assertEqual(len(presence_updates), 0)
diff --git a/tests/push/test_email.py b/tests/push/test_email.py
index 941cf42429..e0a3342088 100644
--- a/tests/push/test_email.py
+++ b/tests/push/test_email.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,7 +21,7 @@
 
 import synapse.rest.admin
 from synapse.api.errors import Codes, SynapseError
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 
 from tests.unittest import HomeserverTestCase
 
@@ -46,14 +45,6 @@ class EmailPusherTests(HomeserverTestCase):
 
     def make_homeserver(self, reactor, clock):
 
-        # List[Tuple[Deferred, args, kwargs]]
-        self.email_attempts = []
-
-        def sendmail(*args, **kwargs):
-            d = Deferred()
-            self.email_attempts.append((d, args, kwargs))
-            return d
-
         config = self.default_config()
         config["email"] = {
             "enable_notifs": True,
@@ -76,7 +67,17 @@ def sendmail(*args, **kwargs):
         config["public_baseurl"] = "aaa"
         config["start_pushers"] = True
 
-        hs = self.setup_test_homeserver(config=config, sendmail=sendmail)
+        hs = self.setup_test_homeserver(config=config)
+
+        # List[Tuple[Deferred, args, kwargs]]
+        self.email_attempts = []
+
+        def sendmail(*args, **kwargs):
+            d = Deferred()
+            self.email_attempts.append((d, args, kwargs))
+            return d
+
+        hs.get_send_email_handler()._sendmail = sendmail
 
         return hs
 
diff --git a/tests/push/test_http.py b/tests/push/test_http.py
index a3b304d316..402eceb6a3 100644
--- a/tests/push/test_http.py
+++ b/tests/push/test_http.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,15 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet.defer import Deferred
 
 import synapse.rest.admin
 from synapse.logging.context import make_deferred_yieldable
 from synapse.push import PusherConfigException
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import receipts
+from synapse.rest.client import login, receipts, room
 
 from tests.unittest import HomeserverTestCase, override_config
 
diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py
index 4a841f5bb8..a52e89e407 100644
--- a/tests/push/test_push_rule_evaluator.py
+++ b/tests/push/test_push_rule_evaluator.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Any, Dict
+
 from synapse.api.room_versions import RoomVersions
 from synapse.events import FrozenEvent
 from synapse.push import push_rule_evaluator
@@ -67,6 +68,170 @@ def test_display_name(self):
         # A display name with spaces should work fine.
         self.assertTrue(evaluator.matches(condition, "@user:test", "foo bar"))
 
+    def _assert_matches(
+        self, condition: Dict[str, Any], content: Dict[str, Any], msg=None
+    ) -> None:
+        evaluator = self._get_evaluator(content)
+        self.assertTrue(evaluator.matches(condition, "@user:test", "display_name"), msg)
+
+    def _assert_not_matches(
+        self, condition: Dict[str, Any], content: Dict[str, Any], msg=None
+    ) -> None:
+        evaluator = self._get_evaluator(content)
+        self.assertFalse(
+            evaluator.matches(condition, "@user:test", "display_name"), msg
+        )
+
+    def test_event_match_body(self):
+        """Check that event_match conditions on content.body work as expected"""
+
+        # if the key is `content.body`, the pattern matches substrings.
+
+        # non-wildcards should match
+        condition = {
+            "kind": "event_match",
+            "key": "content.body",
+            "pattern": "foobaz",
+        }
+        self._assert_matches(
+            condition,
+            {"body": "aaa FoobaZ zzz"},
+            "patterns should match and be case-insensitive",
+        )
+        self._assert_not_matches(
+            condition,
+            {"body": "aa xFoobaZ yy"},
+            "pattern should only match at word boundaries",
+        )
+        self._assert_not_matches(
+            condition,
+            {"body": "aa foobazx yy"},
+            "pattern should only match at word boundaries",
+        )
+
+        # wildcards should match
+        condition = {
+            "kind": "event_match",
+            "key": "content.body",
+            "pattern": "f?o*baz",
+        }
+
+        self._assert_matches(
+            condition,
+            {"body": "aaa FoobarbaZ zzz"},
+            "* should match string and pattern should be case-insensitive",
+        )
+        self._assert_matches(
+            condition, {"body": "aa foobaz yy"}, "* should match 0 characters"
+        )
+        self._assert_not_matches(
+            condition, {"body": "aa fobbaz yy"}, "? should not match 0 characters"
+        )
+        self._assert_not_matches(
+            condition, {"body": "aa fiiobaz yy"}, "? should not match 2 characters"
+        )
+        self._assert_not_matches(
+            condition,
+            {"body": "aa xfooxbaz yy"},
+            "pattern should only match at word boundaries",
+        )
+        self._assert_not_matches(
+            condition,
+            {"body": "aa fooxbazx yy"},
+            "pattern should only match at word boundaries",
+        )
+
+        # test backslashes
+        condition = {
+            "kind": "event_match",
+            "key": "content.body",
+            "pattern": r"f\oobaz",
+        }
+        self._assert_matches(
+            condition,
+            {"body": r"F\oobaz"},
+            "backslash should match itself",
+        )
+        condition = {
+            "kind": "event_match",
+            "key": "content.body",
+            "pattern": r"f\?obaz",
+        }
+        self._assert_matches(
+            condition,
+            {"body": r"F\oobaz"},
+            r"? after \ should match any character",
+        )
+
+    def test_event_match_non_body(self):
+        """Check that event_match conditions on other keys work as expected"""
+
+        # if the key is anything other than 'content.body', the pattern must match the
+        # whole value.
+
+        # non-wildcards should match
+        condition = {
+            "kind": "event_match",
+            "key": "content.value",
+            "pattern": "foobaz",
+        }
+        self._assert_matches(
+            condition,
+            {"value": "FoobaZ"},
+            "patterns should match and be case-insensitive",
+        )
+        self._assert_not_matches(
+            condition,
+            {"value": "xFoobaZ"},
+            "pattern should only match at the start/end of the value",
+        )
+        self._assert_not_matches(
+            condition,
+            {"value": "FoobaZz"},
+            "pattern should only match at the start/end of the value",
+        )
+
+        # wildcards should match
+        condition = {
+            "kind": "event_match",
+            "key": "content.value",
+            "pattern": "f?o*baz",
+        }
+        self._assert_matches(
+            condition,
+            {"value": "FoobarbaZ"},
+            "* should match string and pattern should be case-insensitive",
+        )
+        self._assert_matches(
+            condition, {"value": "foobaz"}, "* should match 0 characters"
+        )
+        self._assert_not_matches(
+            condition, {"value": "fobbaz"}, "? should not match 0 characters"
+        )
+        self._assert_not_matches(
+            condition, {"value": "fiiobaz"}, "? should not match 2 characters"
+        )
+        self._assert_not_matches(
+            condition,
+            {"value": "xfooxbaz"},
+            "pattern should only match at the start/end of the value",
+        )
+        self._assert_not_matches(
+            condition,
+            {"value": "fooxbazx"},
+            "pattern should only match at the start/end of the value",
+        )
+        self._assert_not_matches(
+            condition,
+            {"value": "x\nfooxbaz"},
+            "pattern should not match after a newline",
+        )
+        self._assert_not_matches(
+            condition,
+            {"value": "fooxbaz\nx"},
+            "pattern should not match before a newline",
+        )
+
     def test_no_body(self):
         """Not having a body shouldn't break the evaluator."""
         evaluator = self._get_evaluator({})
diff --git a/tests/replication/__init__.py b/tests/replication/__init__.py
index b7df13c9ee..f43a360a80 100644
--- a/tests/replication/__init__.py
+++ b/tests/replication/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/replication/_base.py b/tests/replication/_base.py
index 1d4a592862..e9fd991718 100644
--- a/tests/replication/_base.py
+++ b/tests/replication/_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,22 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import Any, Callable, Dict, List, Optional, Tuple, Type
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
-from twisted.internet.interfaces import IConsumer, IPullProducer, IReactorTime
 from twisted.internet.protocol import Protocol
-from twisted.internet.task import LoopingCall
-from twisted.web.http import HTTPChannel
 from twisted.web.resource import Resource
-from twisted.web.server import Request, Site
 
-from synapse.app.generic_worker import (
-    GenericWorkerReplicationHandler,
-    GenericWorkerServer,
-)
+from synapse.app.generic_worker import GenericWorkerServer
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseRequest, SynapseSite
 from synapse.replication.http import ReplicationRestResource
+from synapse.replication.tcp.client import ReplicationDataHandler
 from synapse.replication.tcp.handler import ReplicationCommandHandler
 from synapse.replication.tcp.protocol import ClientReplicationStreamProtocol
 from synapse.replication.tcp.resource import (
@@ -36,7 +29,6 @@
     ServerReplicationStreamProtocol,
 )
 from synapse.server import HomeServer
-from synapse.util import Clock
 
 from tests import unittest
 from tests.server import FakeTransport
@@ -61,9 +53,9 @@ def prepare(self, reactor, clock, hs):
         # build a replication server
         server_factory = ReplicationStreamProtocolFactory(hs)
         self.streamer = hs.get_replication_streamer()
-        self.server = server_factory.buildProtocol(
+        self.server: ServerReplicationStreamProtocol = server_factory.buildProtocol(
             None
-        )  # type: ServerReplicationStreamProtocol
+        )
 
         # Make a new HomeServer object for the worker
         self.reactor.lookups["testserv"] = "1.2.3.4"
@@ -157,7 +149,19 @@ def handle_http_replication_attempt(self) -> SynapseRequest:
         client_protocol = client_factory.buildProtocol(None)
 
         # Set up the server side protocol
-        channel = _PushHTTPChannel(self.reactor, SynapseRequest, self.site)
+        channel = self.site.buildProtocol(None)
+
+        # hook into the channel's request factory so that we can keep a record
+        # of the requests
+        requests: List[SynapseRequest] = []
+        real_request_factory = channel.requestFactory
+
+        def request_factory(*args, **kwargs):
+            request = real_request_factory(*args, **kwargs)
+            requests.append(request)
+            return request
+
+        channel.requestFactory = request_factory
 
         # Connect client to server and vice versa.
         client_to_server_transport = FakeTransport(
@@ -179,7 +183,10 @@ def handle_http_replication_attempt(self) -> SynapseRequest:
         server_to_client_transport.loseConnection()
         client_to_server_transport.loseConnection()
 
-        return channel.request
+        # there should have been exactly one request
+        self.assertEqual(len(requests), 1)
+
+        return requests[0]
 
     def assert_request_is_get_repl_stream_updates(
         self, request: SynapseRequest, stream_name: str
@@ -188,7 +195,7 @@ def assert_request_is_get_repl_stream_updates(
         fetching updates for given stream.
         """
 
-        path = request.path  # type: bytes  # type: ignore
+        path: bytes = request.path  # type: ignore
         self.assertRegex(
             path,
             br"^/_synapse/replication/get_repl_stream_updates/%s/[^/]+$"
@@ -205,7 +212,7 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
     unlike `BaseStreamTestCase`.
     """
 
-    servlets = []  # type: List[Callable[[HomeServer, JsonResource], None]]
+    servlets: List[Callable[[HomeServer, JsonResource], None]] = []
 
     def setUp(self):
         super().setUp()
@@ -266,7 +273,7 @@ def create_test_resource(self):
         return resource
 
     def make_worker_hs(
-        self, worker_app: str, extra_config: dict = {}, **kwargs
+        self, worker_app: str, extra_config: Optional[dict] = None, **kwargs
     ) -> HomeServer:
         """Make a new worker HS instance, correctly connecting replcation
         stream to the master HS.
@@ -283,7 +290,7 @@ def make_worker_hs(
 
         config = self._get_worker_hs_config()
         config["worker_app"] = worker_app
-        config.update(extra_config)
+        config.update(extra_config or {})
 
         worker_hs = self.setup_test_homeserver(
             homeserver_to_use=GenericWorkerServer,
@@ -352,6 +359,8 @@ def make_worker_hs(
             config=worker_hs.config.server.listeners[0],
             resource=resource,
             server_version_string="1",
+            max_request_body_size=4096,
+            reactor=self.reactor,
         )
 
         if worker_hs.config.redis.redis_enabled:
@@ -389,7 +398,7 @@ def _handle_http_replication_attempt(self, hs, repl_port):
         client_protocol = client_factory.buildProtocol(None)
 
         # Set up the server side protocol
-        channel = _PushHTTPChannel(self.reactor, SynapseRequest, self._hs_to_site[hs])
+        channel = self._hs_to_site[hs].buildProtocol(None)
 
         # Connect client to server and vice versa.
         client_to_server_transport = FakeTransport(
@@ -432,14 +441,14 @@ def connect_any_redis_attempts(self):
             server_protocol.makeConnection(server_to_client_transport)
 
 
-class TestReplicationDataHandler(GenericWorkerReplicationHandler):
+class TestReplicationDataHandler(ReplicationDataHandler):
     """Drop-in for ReplicationDataHandler which just collects RDATA rows"""
 
     def __init__(self, hs: HomeServer):
         super().__init__(hs)
 
         # list of received (stream_name, token, row) tuples
-        self.received_rdata_rows = []  # type: List[Tuple[str, int, Any]]
+        self.received_rdata_rows: List[Tuple[str, int, Any]] = []
 
     async def on_rdata(self, stream_name, instance_name, token, rows):
         await super().on_rdata(stream_name, instance_name, token, rows)
@@ -447,112 +456,6 @@ async def on_rdata(self, stream_name, instance_name, token, rows):
             self.received_rdata_rows.append((stream_name, token, r))
 
 
-class _PushHTTPChannel(HTTPChannel):
-    """A HTTPChannel that wraps pull producers to push producers.
-
-    This is a hack to get around the fact that HTTPChannel transparently wraps a
-    pull producer (which is what Synapse uses to reply to requests) with
-    `_PullToPush` to convert it to a push producer. Unfortunately `_PullToPush`
-    uses the standard reactor rather than letting us use our test reactor, which
-    makes it very hard to test.
-    """
-
-    def __init__(
-        self, reactor: IReactorTime, request_factory: Type[Request], site: Site
-    ):
-        super().__init__()
-        self.reactor = reactor
-        self.requestFactory = request_factory
-        self.site = site
-
-        self._pull_to_push_producer = None  # type: Optional[_PullToPushProducer]
-
-    def registerProducer(self, producer, streaming):
-        # Convert pull producers to push producer.
-        if not streaming:
-            self._pull_to_push_producer = _PullToPushProducer(
-                self.reactor, producer, self
-            )
-            producer = self._pull_to_push_producer
-
-        super().registerProducer(producer, True)
-
-    def unregisterProducer(self):
-        if self._pull_to_push_producer:
-            # We need to manually stop the _PullToPushProducer.
-            self._pull_to_push_producer.stop()
-
-    def checkPersistence(self, request, version):
-        """Check whether the connection can be re-used"""
-        # We hijack this to always say no for ease of wiring stuff up in
-        # `handle_http_replication_attempt`.
-        request.responseHeaders.setRawHeaders(b"connection", [b"close"])
-        return False
-
-    def requestDone(self, request):
-        # Store the request for inspection.
-        self.request = request
-        super().requestDone(request)
-
-
-class _PullToPushProducer:
-    """A push producer that wraps a pull producer."""
-
-    def __init__(
-        self, reactor: IReactorTime, producer: IPullProducer, consumer: IConsumer
-    ):
-        self._clock = Clock(reactor)
-        self._producer = producer
-        self._consumer = consumer
-
-        # While running we use a looping call with a zero delay to call
-        # resumeProducing on given producer.
-        self._looping_call = None  # type: Optional[LoopingCall]
-
-        # We start writing next reactor tick.
-        self._start_loop()
-
-    def _start_loop(self):
-        """Start the looping call to"""
-
-        if not self._looping_call:
-            # Start a looping call which runs every tick.
-            self._looping_call = self._clock.looping_call(self._run_once, 0)
-
-    def stop(self):
-        """Stops calling resumeProducing."""
-        if self._looping_call:
-            self._looping_call.stop()
-            self._looping_call = None
-
-    def pauseProducing(self):
-        """Implements IPushProducer"""
-        self.stop()
-
-    def resumeProducing(self):
-        """Implements IPushProducer"""
-        self._start_loop()
-
-    def stopProducing(self):
-        """Implements IPushProducer"""
-        self.stop()
-        self._producer.stopProducing()
-
-    def _run_once(self):
-        """Calls resumeProducing on producer once."""
-
-        try:
-            self._producer.resumeProducing()
-        except Exception:
-            logger.exception("Failed to call resumeProducing")
-            try:
-                self._consumer.unregisterProducer()
-            except Exception:
-                pass
-
-            self.stopProducing()
-
-
 class FakeRedisPubSubServer:
     """A fake Redis server for pub/sub."""
 
@@ -581,7 +484,7 @@ def buildProtocol(self, addr):
 class FakeRedisPubSubProtocol(Protocol):
     """A connection from a client talking to the fake Redis server."""
 
-    transport = None  # type: Optional[FakeTransport]
+    transport: Optional[FakeTransport] = None
 
     def __init__(self, server: FakeRedisPubSubServer):
         self._server = server
@@ -647,12 +550,12 @@ def encode(self, obj):
         if obj is None:
             return "$-1\r\n"
         if isinstance(obj, str):
-            return "${len}\r\n{str}\r\n".format(len=len(obj), str=obj)
+            return f"${len(obj)}\r\n{obj}\r\n"
         if isinstance(obj, int):
-            return ":{val}\r\n".format(val=obj)
+            return f":{obj}\r\n"
         if isinstance(obj, (list, tuple)):
             items = "".join(self.encode(a) for a in obj)
-            return "*{len}\r\n{items}".format(len=len(obj), items=items)
+            return f"*{len(obj)}\r\n{items}"
 
         raise Exception("Unrecognized type for encoding redis: %r: %r", type(obj), obj)
 
diff --git a/tests/replication/slave/__init__.py b/tests/replication/slave/__init__.py
index b7df13c9ee..f43a360a80 100644
--- a/tests/replication/slave/__init__.py
+++ b/tests/replication/slave/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/replication/slave/storage/__init__.py b/tests/replication/slave/storage/__init__.py
index b7df13c9ee..f43a360a80 100644
--- a/tests/replication/slave/storage/__init__.py
+++ b/tests/replication/slave/storage/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/replication/slave/storage/_base.py b/tests/replication/slave/storage/_base.py
index 56497b8476..83e89383f6 100644
--- a/tests/replication/slave/storage/_base.py
+++ b/tests/replication/slave/storage/_base.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from tests.replication._base import BaseStreamTestCase
 
diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py
index 0ceb0f935c..db80a0bdbd 100644
--- a/tests/replication/slave/storage/test_events.py
+++ b/tests/replication/slave/storage/test_events.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+from typing import Iterable, Optional
 
 from canonicaljson import encode_canonical_json
 
@@ -332,15 +333,18 @@ def build_event(
         room_id=ROOM_ID,
         type="m.room.message",
         key=None,
-        internal={},
+        internal: Optional[dict] = None,
         depth=None,
-        prev_events=[],
-        auth_events=[],
-        prev_state=[],
+        prev_events: Optional[list] = None,
+        auth_events: Optional[list] = None,
+        prev_state: Optional[list] = None,
         redacts=None,
-        push_actions=[],
-        **content
+        push_actions: Iterable = frozenset(),
+        **content,
     ):
+        prev_events = prev_events or []
+        auth_events = auth_events or []
+        prev_state = prev_state or []
 
         if depth is None:
             depth = self.event_id
@@ -369,7 +373,7 @@ def build_event(
         if redacts is not None:
             event_dict["redacts"] = redacts
 
-        event = make_event_from_dict(event_dict, internal_metadata_dict=internal)
+        event = make_event_from_dict(event_dict, internal_metadata_dict=internal or {})
 
         self.event_id += 1
         state_handler = self.hs.get_state_handler()
diff --git a/tests/replication/tcp/__init__.py b/tests/replication/tcp/__init__.py
index 1453d04571..743fb9904a 100644
--- a/tests/replication/tcp/__init__.py
+++ b/tests/replication/tcp/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/replication/tcp/streams/__init__.py b/tests/replication/tcp/streams/__init__.py
index 1453d04571..743fb9904a 100644
--- a/tests/replication/tcp/streams/__init__.py
+++ b/tests/replication/tcp/streams/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/replication/tcp/streams/test_account_data.py b/tests/replication/tcp/streams/test_account_data.py
index 153634d4ee..cdd052001b 100644
--- a/tests/replication/tcp/streams/test_account_data.py
+++ b/tests/replication/tcp/streams/test_account_data.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/replication/tcp/streams/test_events.py b/tests/replication/tcp/streams/test_events.py
index 77856fc304..f198a94887 100644
--- a/tests/replication/tcp/streams/test_events.py
+++ b/tests/replication/tcp/streams/test_events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,7 +24,7 @@
     EventsStreamRow,
 )
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 
 from tests.replication._base import BaseStreamTestCase
 from tests.test_utils.event_injection import inject_event, inject_member_event
@@ -136,9 +135,9 @@ def test_update_function_huge_state_change(self):
         )
 
         # this is the point in the DAG where we make a fork
-        fork_point = self.get_success(
+        fork_point: List[str] = self.get_success(
             self.hs.get_datastore().get_latest_event_ids_in_room(self.room_id)
-        )  # type: List[str]
+        )
 
         events = [
             self._inject_state_event(sender=OTHER_USER)
@@ -239,8 +238,8 @@ def test_update_function_huge_state_change(self):
         self.assertEqual(row.data.event_id, pl_event.event_id)
 
         # the state rows are unsorted
-        state_rows = []  # type: List[EventsStreamCurrentStateRow]
-        for stream_name, token, row in received_rows:
+        state_rows: List[EventsStreamCurrentStateRow] = []
+        for stream_name, _, row in received_rows:
             self.assertEqual("events", stream_name)
             self.assertIsInstance(row, EventsStreamRow)
             self.assertEqual(row.type, "state")
@@ -291,11 +290,11 @@ def test_update_function_state_row_limit(self):
         )
 
         # this is the point in the DAG where we make a fork
-        fork_point = self.get_success(
+        fork_point: List[str] = self.get_success(
             self.hs.get_datastore().get_latest_event_ids_in_room(self.room_id)
-        )  # type: List[str]
+        )
 
-        events = []  # type: List[EventBase]
+        events: List[EventBase] = []
         for user in user_ids:
             events.extend(
                 self._inject_state_event(sender=user) for _ in range(STATES_PER_USER)
@@ -356,8 +355,8 @@ def test_update_function_state_row_limit(self):
             self.assertEqual(row.data.event_id, pl_events[i].event_id)
 
             # the state rows are unsorted
-            state_rows = []  # type: List[EventsStreamCurrentStateRow]
-            for j in range(STATES_PER_USER + 1):
+            state_rows: List[EventsStreamCurrentStateRow] = []
+            for _ in range(STATES_PER_USER + 1):
                 stream_name, token, row = received_rows.pop(0)
                 self.assertEqual("events", stream_name)
                 self.assertIsInstance(row, EventsStreamRow)
diff --git a/tests/replication/tcp/streams/test_federation.py b/tests/replication/tcp/streams/test_federation.py
index aa4bf1c7e3..ffec06a0d6 100644
--- a/tests/replication/tcp/streams/test_federation.py
+++ b/tests/replication/tcp/streams/test_federation.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/replication/tcp/streams/test_receipts.py b/tests/replication/tcp/streams/test_receipts.py
index 56b062ecc1..38e292c1ab 100644
--- a/tests/replication/tcp/streams/test_receipts.py
+++ b/tests/replication/tcp/streams/test_receipts.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +14,7 @@
 
 # type: ignore
 
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.replication.tcp.streams._base import ReceiptsStream
 
@@ -44,7 +43,7 @@ def test_receipt(self):
         stream_name, _, token, rdata_rows = self.test_handler.on_rdata.call_args[0]
         self.assertEqual(stream_name, "receipts")
         self.assertEqual(1, len(rdata_rows))
-        row = rdata_rows[0]  # type: ReceiptsStream.ReceiptsStreamRow
+        row: ReceiptsStream.ReceiptsStreamRow = rdata_rows[0]
         self.assertEqual("!room:blue", row.room_id)
         self.assertEqual("m.read", row.receipt_type)
         self.assertEqual(USER_ID, row.user_id)
@@ -76,7 +75,7 @@ def test_receipt(self):
         self.assertEqual(token, 3)
         self.assertEqual(1, len(rdata_rows))
 
-        row = rdata_rows[0]  # type: ReceiptsStream.ReceiptsStreamRow
+        row: ReceiptsStream.ReceiptsStreamRow = rdata_rows[0]
         self.assertEqual("!room2:blue", row.room_id)
         self.assertEqual("m.read", row.receipt_type)
         self.assertEqual(USER_ID, row.user_id)
diff --git a/tests/replication/tcp/streams/test_typing.py b/tests/replication/tcp/streams/test_typing.py
index ca49d4dd3a..3ff5afc6e5 100644
--- a/tests/replication/tcp/streams/test_typing.py
+++ b/tests/replication/tcp/streams/test_typing.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.handlers.typing import RoomMember
 from synapse.replication.tcp.streams import TypingStream
@@ -48,7 +47,7 @@ def test_typing(self):
         stream_name, _, token, rdata_rows = self.test_handler.on_rdata.call_args[0]
         self.assertEqual(stream_name, "typing")
         self.assertEqual(1, len(rdata_rows))
-        row = rdata_rows[0]  # type: TypingStream.TypingStreamRow
+        row: TypingStream.TypingStreamRow = rdata_rows[0]
         self.assertEqual(ROOM_ID, row.room_id)
         self.assertEqual([USER_ID], row.user_ids)
 
@@ -103,7 +102,7 @@ def test_reset(self):
         stream_name, _, token, rdata_rows = self.test_handler.on_rdata.call_args[0]
         self.assertEqual(stream_name, "typing")
         self.assertEqual(1, len(rdata_rows))
-        row = rdata_rows[0]  # type: TypingStream.TypingStreamRow
+        row: TypingStream.TypingStreamRow = rdata_rows[0]
         self.assertEqual(ROOM_ID, row.room_id)
         self.assertEqual([USER_ID], row.user_ids)
 
diff --git a/tests/replication/tcp/test_commands.py b/tests/replication/tcp/test_commands.py
index 60c10a441a..cca7ebb719 100644
--- a/tests/replication/tcp/test_commands.py
+++ b/tests/replication/tcp/test_commands.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/replication/tcp/test_remote_server_up.py b/tests/replication/tcp/test_remote_server_up.py
index 1fe9d5b4d0..262c35cef3 100644
--- a/tests/replication/tcp/test_remote_server_up.py
+++ b/tests/replication/tcp/test_remote_server_up.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/replication/test_auth.py b/tests/replication/test_auth.py
index f8fd8a843c..43a16bb141 100644
--- a/tests/replication/test_auth.py
+++ b/tests/replication/test_auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +13,7 @@
 # limitations under the License.
 import logging
 
-from synapse.rest.client.v2_alpha import register
+from synapse.rest.client import register
 
 from tests.replication._base import BaseMultiWorkerStreamTestCase
 from tests.server import FakeChannel, make_request
diff --git a/tests/replication/test_client_reader_shard.py b/tests/replication/test_client_reader_shard.py
index 5da1d5dc4d..995097d72c 100644
--- a/tests/replication/test_client_reader_shard.py
+++ b/tests/replication/test_client_reader_shard.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +13,7 @@
 # limitations under the License.
 import logging
 
-from synapse.rest.client.v2_alpha import register
+from synapse.rest.client import register
 
 from tests.replication._base import BaseMultiWorkerStreamTestCase
 from tests.server import make_request
diff --git a/tests/replication/test_federation_ack.py b/tests/replication/test_federation_ack.py
index 0d9e3bb11d..04a869e295 100644
--- a/tests/replication/test_federation_ack.py
+++ b/tests/replication/test_federation_ack.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
+from unittest import mock
 
 from synapse.app.generic_worker import GenericWorkerServer
 from synapse.replication.tcp.commands import FederationAckCommand
diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py
index 2f2d117858..af5dfca752 100644
--- a/tests/replication/test_federation_sender_shard.py
+++ b/tests/replication/test_federation_sender_shard.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.api.constants import EventTypes, Membership
 from synapse.events.builder import EventBuilderFactory
 from synapse.rest.admin import register_servlets_for_client_rest_resource
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.types import UserID, create_requester
 
 from tests.replication._base import BaseMultiWorkerStreamTestCase
@@ -226,9 +224,11 @@ def create_room_with_remote_server(self, user, token, remote_server="other_serve
         }
 
         builder = factory.for_room_version(room_version, event_dict)
-        join_event = self.get_success(builder.build(prev_event_ids, None))
+        join_event = self.get_success(
+            builder.build(prev_event_ids=prev_event_ids, auth_event_ids=None)
+        )
 
-        self.get_success(federation.on_send_join_request(remote_server, join_event))
+        self.get_success(federation.on_send_membership_event(remote_server, join_event))
         self.replicate()
 
         return room
diff --git a/tests/replication/test_multi_media_repo.py b/tests/replication/test_multi_media_repo.py
index b0800f9840..ac419f0db3 100644
--- a/tests/replication/test_multi_media_repo.py
+++ b/tests/replication/test_multi_media_repo.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,7 +22,7 @@
 from twisted.web.server import Request
 
 from synapse.rest import admin
-from synapse.rest.client.v1 import login
+from synapse.rest.client import login
 from synapse.server import HomeServer
 
 from tests.http import TestServerTLSConnectionFactory, get_test_ca_cert_file
@@ -32,7 +31,7 @@
 
 logger = logging.getLogger(__name__)
 
-test_server_connection_factory = None  # type: Optional[TestServerTLSConnectionFactory]
+test_server_connection_factory: Optional[TestServerTLSConnectionFactory] = None
 
 
 class MediaRepoShardTestCase(BaseMultiWorkerStreamTestCase):
@@ -71,7 +70,7 @@ def _get_media_req(
             self.reactor,
             FakeSite(resource),
             "GET",
-            "/{}/{}".format(target, media_id),
+            f"/{target}/{media_id}",
             shorthand=False,
             access_token=self.access_token,
             await_result=False,
@@ -114,7 +113,7 @@ def _get_media_req(
         self.assertEqual(request.method, b"GET")
         self.assertEqual(
             request.path,
-            "/_matrix/media/r0/download/{}/{}".format(target, media_id).encode("utf-8"),
+            f"/_matrix/media/r0/download/{target}/{media_id}".encode("utf-8"),
         )
         self.assertEqual(
             request.requestHeaders.getRawHeaders(b"host"), [target.encode("utf-8")]
diff --git a/tests/replication/test_pusher_shard.py b/tests/replication/test_pusher_shard.py
index ab2988a6ba..4094a75f36 100644
--- a/tests/replication/test_pusher_shard.py
+++ b/tests/replication/test_pusher_shard.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 
 from tests.replication._base import BaseMultiWorkerStreamTestCase
 
diff --git a/tests/replication/test_sharded_event_persister.py b/tests/replication/test_sharded_event_persister.py
index c9b773fbd2..0a6e4795ee 100644
--- a/tests/replication/test_sharded_event_persister.py
+++ b/tests/replication/test_sharded_event_persister.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,13 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-
-from mock import patch
+from unittest.mock import patch
 
 from synapse.api.room_versions import RoomVersion
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import sync
+from synapse.rest.client import login, room, sync
 
 from tests.replication._base import BaseMultiWorkerStreamTestCase
 from tests.server import make_request
@@ -32,7 +29,7 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase):
     """Checks event persisting sharding works"""
 
     # Event persister sharding requires postgres (due to needing
-    # `MutliWriterIdGenerator`).
+    # `MultiWriterIdGenerator`).
     if not USE_POSTGRES_FOR_TESTS:
         skip = "Requires Postgres"
 
@@ -213,7 +210,7 @@ def test_vector_clock_token(self):
             self.reactor,
             sync_hs_site,
             "GET",
-            "/sync?since={}".format(next_batch),
+            f"/sync?since={next_batch}",
             access_token=access_token,
         )
 
@@ -243,7 +240,7 @@ def test_vector_clock_token(self):
             self.reactor,
             sync_hs_site,
             "GET",
-            "/sync?since={}".format(vector_clock_token),
+            f"/sync?since={vector_clock_token}",
             access_token=access_token,
         )
 
@@ -268,7 +265,7 @@ def test_vector_clock_token(self):
             self.reactor,
             sync_hs_site,
             "GET",
-            "/sync?since={}".format(next_batch),
+            f"/sync?since={next_batch}",
             access_token=access_token,
         )
 
diff --git a/tests/rest/__init__.py b/tests/rest/__init__.py
index fe0ac3f8e9..629e2df74a 100644
--- a/tests/rest/__init__.py
+++ b/tests/rest/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/rest/admin/__init__.py b/tests/rest/admin/__init__.py
index 1453d04571..743fb9904a 100644
--- a/tests/rest/admin/__init__.py
+++ b/tests/rest/admin/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py
index 057e27372e..bfa638fb4b 100644
--- a/tests/rest/admin/test_admin.py
+++ b/tests/rest/admin/test_admin.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,8 +16,7 @@
 import os
 import urllib.parse
 from binascii import unhexlify
-
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet.defer import Deferred
 
@@ -26,8 +24,7 @@
 from synapse.http.server import JsonResource
 from synapse.logging.context import make_deferred_yieldable
 from synapse.rest.admin import VersionServlet
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import groups
+from synapse.rest.client import groups, login, room
 
 from tests import unittest
 from tests.server import FakeSite, make_request
@@ -68,7 +65,7 @@ def test_delete_group(self):
         # Create a new group
         channel = self.make_request(
             "POST",
-            "/create_group".encode("ascii"),
+            b"/create_group",
             access_token=self.admin_user_tok,
             content={"localpart": "test"},
         )
@@ -131,9 +128,7 @@ def _check_group(self, group_id, expect_code):
 
     def _get_groups_user_is_in(self, access_token):
         """Returns the list of groups the user is in (given their access token)"""
-        channel = self.make_request(
-            "GET", "/joined_groups".encode("ascii"), access_token=access_token
-        )
+        channel = self.make_request("GET", b"/joined_groups", access_token=access_token)
 
         self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
 
diff --git a/tests/rest/admin/test_device.py b/tests/rest/admin/test_device.py
index 2a1bcf1760..c4afe5c3d9 100644
--- a/tests/rest/admin/test_device.py
+++ b/tests/rest/admin/test_device.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Dirk Klimpel
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +17,7 @@
 
 import synapse.rest.admin
 from synapse.api.errors import Codes
-from synapse.rest.client.v1 import login
+from synapse.rest.client import login
 
 from tests import unittest
 
@@ -431,7 +430,7 @@ def test_get_devices(self):
         """
         # Create devices
         number_devices = 5
-        for n in range(number_devices):
+        for _ in range(number_devices):
             self.login("user", "pass")
 
         # Get devices
@@ -548,7 +547,7 @@ def test_delete_devices(self):
 
         # Create devices
         number_devices = 5
-        for n in range(number_devices):
+        for _ in range(number_devices):
             self.login("user", "pass")
 
         # Get devices
diff --git a/tests/rest/admin/test_event_reports.py b/tests/rest/admin/test_event_reports.py
index e30ffe4fa0..e9ef89731f 100644
--- a/tests/rest/admin/test_event_reports.py
+++ b/tests/rest/admin/test_event_reports.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Dirk Klimpel
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,8 +16,7 @@
 
 import synapse.rest.admin
 from synapse.api.errors import Codes
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import report_event
+from synapse.rest.client import login, report_event, room
 
 from tests import unittest
 
@@ -49,23 +47,23 @@ def prepare(self, reactor, clock, hs):
         self.helper.join(self.room_id2, user=self.admin_user, tok=self.admin_user_tok)
 
         # Two rooms and two users. Every user sends and reports every room event
-        for i in range(5):
+        for _ in range(5):
             self._create_event_and_report(
                 room_id=self.room_id1,
                 user_tok=self.other_user_tok,
             )
-        for i in range(5):
+        for _ in range(5):
             self._create_event_and_report(
                 room_id=self.room_id2,
                 user_tok=self.other_user_tok,
             )
-        for i in range(5):
+        for _ in range(5):
             self._create_event_and_report(
                 room_id=self.room_id1,
                 user_tok=self.admin_user_tok,
             )
-        for i in range(5):
-            self._create_event_and_report(
+        for _ in range(5):
+            self._create_event_and_report_without_parameters(
                 room_id=self.room_id2,
                 user_tok=self.admin_user_tok,
             )
@@ -379,6 +377,19 @@ def _create_event_and_report(self, room_id, user_tok):
         )
         self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
 
+    def _create_event_and_report_without_parameters(self, room_id, user_tok):
+        """Create and report an event, but omit reason and score"""
+        resp = self.helper.send(room_id, tok=user_tok)
+        event_id = resp["event_id"]
+
+        channel = self.make_request(
+            "POST",
+            "rooms/%s/report/%s" % (room_id, event_id),
+            json.dumps({}),
+            access_token=user_tok,
+        )
+        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+
     def _check_fields(self, content):
         """Checks that all attributes are present in an event report"""
         for c in content:
diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py
index 31db472cd3..972d60570c 100644
--- a/tests/rest/admin/test_media.py
+++ b/tests/rest/admin/test_media.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Dirk Klimpel
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,9 +16,11 @@
 import os
 from binascii import unhexlify
 
+from parameterized import parameterized
+
 import synapse.rest.admin
 from synapse.api.errors import Codes
-from synapse.rest.client.v1 import login, profile, room
+from synapse.rest.client import login, profile, room
 from synapse.rest.media.v1.filepath import MediaFilePaths
 
 from tests import unittest
@@ -260,7 +261,7 @@ def test_missing_parameter(self):
         self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
         self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"])
         self.assertEqual(
-            "Missing integer query parameter b'before_ts'", channel.json_body["error"]
+            "Missing integer query parameter 'before_ts'", channel.json_body["error"]
         )
 
     def test_invalid_parameter(self):
@@ -302,7 +303,7 @@ def test_invalid_parameter(self):
         self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
         self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
         self.assertEqual(
-            "Boolean query parameter b'keep_profiles' must be one of ['true', 'false']",
+            "Boolean query parameter 'keep_profiles' must be one of ['true', 'false']",
             channel.json_body["error"],
         )
 
@@ -563,3 +564,228 @@ def _access_media(self, server_and_media_id, expect_success=True):
             )
             # Test that the file is deleted
             self.assertFalse(os.path.exists(local_path))
+
+
+class QuarantineMediaByIDTestCase(unittest.HomeserverTestCase):
+
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_media_repo,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        media_repo = hs.get_media_repository_resource()
+        self.store = hs.get_datastore()
+        self.server_name = hs.hostname
+
+        self.admin_user = self.register_user("admin", "pass", admin=True)
+        self.admin_user_tok = self.login("admin", "pass")
+
+        # Create media
+        upload_resource = media_repo.children[b"upload"]
+        # file size is 67 Byte
+        image_data = unhexlify(
+            b"89504e470d0a1a0a0000000d4948445200000001000000010806"
+            b"0000001f15c4890000000a49444154789c63000100000500010d"
+            b"0a2db40000000049454e44ae426082"
+        )
+
+        # Upload some media into the room
+        response = self.helper.upload_media(
+            upload_resource, image_data, tok=self.admin_user_tok, expect_code=200
+        )
+        # Extract media ID from the response
+        server_and_media_id = response["content_uri"][6:]  # Cut off 'mxc://'
+        self.media_id = server_and_media_id.split("/")[1]
+
+        self.url = "/_synapse/admin/v1/media/%s/%s/%s"
+
+    @parameterized.expand(["quarantine", "unquarantine"])
+    def test_no_auth(self, action: str):
+        """
+        Try to protect media without authentication.
+        """
+
+        channel = self.make_request(
+            "POST",
+            self.url % (action, self.server_name, self.media_id),
+            b"{}",
+        )
+
+        self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
+
+    @parameterized.expand(["quarantine", "unquarantine"])
+    def test_requester_is_no_admin(self, action: str):
+        """
+        If the user is not a server admin, an error is returned.
+        """
+        self.other_user = self.register_user("user", "pass")
+        self.other_user_token = self.login("user", "pass")
+
+        channel = self.make_request(
+            "POST",
+            self.url % (action, self.server_name, self.media_id),
+            access_token=self.other_user_token,
+        )
+
+        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
+
+    def test_quarantine_media(self):
+        """
+        Tests that quarantining and remove from quarantine a media is successfully
+        """
+
+        media_info = self.get_success(self.store.get_local_media(self.media_id))
+        self.assertFalse(media_info["quarantined_by"])
+
+        # quarantining
+        channel = self.make_request(
+            "POST",
+            self.url % ("quarantine", self.server_name, self.media_id),
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertFalse(channel.json_body)
+
+        media_info = self.get_success(self.store.get_local_media(self.media_id))
+        self.assertTrue(media_info["quarantined_by"])
+
+        # remove from quarantine
+        channel = self.make_request(
+            "POST",
+            self.url % ("unquarantine", self.server_name, self.media_id),
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertFalse(channel.json_body)
+
+        media_info = self.get_success(self.store.get_local_media(self.media_id))
+        self.assertFalse(media_info["quarantined_by"])
+
+    def test_quarantine_protected_media(self):
+        """
+        Tests that quarantining from protected media fails
+        """
+
+        # protect
+        self.get_success(self.store.mark_local_media_as_safe(self.media_id, safe=True))
+
+        # verify protection
+        media_info = self.get_success(self.store.get_local_media(self.media_id))
+        self.assertTrue(media_info["safe_from_quarantine"])
+
+        # quarantining
+        channel = self.make_request(
+            "POST",
+            self.url % ("quarantine", self.server_name, self.media_id),
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertFalse(channel.json_body)
+
+        # verify that is not in quarantine
+        media_info = self.get_success(self.store.get_local_media(self.media_id))
+        self.assertFalse(media_info["quarantined_by"])
+
+
+class ProtectMediaByIDTestCase(unittest.HomeserverTestCase):
+
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_media_repo,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        media_repo = hs.get_media_repository_resource()
+        self.store = hs.get_datastore()
+
+        self.admin_user = self.register_user("admin", "pass", admin=True)
+        self.admin_user_tok = self.login("admin", "pass")
+
+        # Create media
+        upload_resource = media_repo.children[b"upload"]
+        # file size is 67 Byte
+        image_data = unhexlify(
+            b"89504e470d0a1a0a0000000d4948445200000001000000010806"
+            b"0000001f15c4890000000a49444154789c63000100000500010d"
+            b"0a2db40000000049454e44ae426082"
+        )
+
+        # Upload some media into the room
+        response = self.helper.upload_media(
+            upload_resource, image_data, tok=self.admin_user_tok, expect_code=200
+        )
+        # Extract media ID from the response
+        server_and_media_id = response["content_uri"][6:]  # Cut off 'mxc://'
+        self.media_id = server_and_media_id.split("/")[1]
+
+        self.url = "/_synapse/admin/v1/media/%s/%s"
+
+    @parameterized.expand(["protect", "unprotect"])
+    def test_no_auth(self, action: str):
+        """
+        Try to protect media without authentication.
+        """
+
+        channel = self.make_request("POST", self.url % (action, self.media_id), b"{}")
+
+        self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
+
+    @parameterized.expand(["protect", "unprotect"])
+    def test_requester_is_no_admin(self, action: str):
+        """
+        If the user is not a server admin, an error is returned.
+        """
+        self.other_user = self.register_user("user", "pass")
+        self.other_user_token = self.login("user", "pass")
+
+        channel = self.make_request(
+            "POST",
+            self.url % (action, self.media_id),
+            access_token=self.other_user_token,
+        )
+
+        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
+
+    def test_protect_media(self):
+        """
+        Tests that protect and unprotect a media is successfully
+        """
+
+        media_info = self.get_success(self.store.get_local_media(self.media_id))
+        self.assertFalse(media_info["safe_from_quarantine"])
+
+        # protect
+        channel = self.make_request(
+            "POST",
+            self.url % ("protect", self.media_id),
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertFalse(channel.json_body)
+
+        media_info = self.get_success(self.store.get_local_media(self.media_id))
+        self.assertTrue(media_info["safe_from_quarantine"])
+
+        # unprotect
+        channel = self.make_request(
+            "POST",
+            self.url % ("unprotect", self.media_id),
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertFalse(channel.json_body)
+
+        media_info = self.get_success(self.store.get_local_media(self.media_id))
+        self.assertFalse(media_info["safe_from_quarantine"])
diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py
index b55160b70a..c9d4731017 100644
--- a/tests/rest/admin/test_room.py
+++ b/tests/rest/admin/test_room.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Dirk Klimpel
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,13 +15,14 @@
 import json
 import urllib.parse
 from typing import List, Optional
+from unittest.mock import Mock
 
-from mock import Mock
+from parameterized import parameterized_class
 
 import synapse.rest.admin
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import Codes
-from synapse.rest.client.v1 import directory, events, login, room
+from synapse.rest.client import directory, events, login, room
 
 from tests import unittest
 
@@ -146,6 +146,13 @@ def _assert_peek(self, room_id, expect_code):
         )
 
 
+@parameterized_class(
+    ("method", "url_template"),
+    [
+        ("POST", "/_synapse/admin/v1/rooms/%s/delete"),
+        ("DELETE", "/_synapse/admin/v1/rooms/%s"),
+    ],
+)
 class DeleteRoomTestCase(unittest.HomeserverTestCase):
     servlets = [
         synapse.rest.admin.register_servlets,
@@ -177,7 +184,7 @@ def prepare(self, reactor, clock, hs):
         self.room_id = self.helper.create_room_as(
             self.other_user, tok=self.other_user_tok
         )
-        self.url = "/_synapse/admin/v1/rooms/%s/delete" % self.room_id
+        self.url = self.url_template % self.room_id
 
     def test_requester_is_no_admin(self):
         """
@@ -185,7 +192,7 @@ def test_requester_is_no_admin(self):
         """
 
         channel = self.make_request(
-            "POST",
+            self.method,
             self.url,
             json.dumps({}),
             access_token=self.other_user_tok,
@@ -198,10 +205,10 @@ def test_room_does_not_exist(self):
         """
         Check that unknown rooms/server return error 404.
         """
-        url = "/_synapse/admin/v1/rooms/!unknown:test/delete"
+        url = self.url_template % "!unknown:test"
 
         channel = self.make_request(
-            "POST",
+            self.method,
             url,
             json.dumps({}),
             access_token=self.admin_user_tok,
@@ -214,10 +221,10 @@ def test_room_is_not_valid(self):
         """
         Check that invalid room names, return an error 400.
         """
-        url = "/_synapse/admin/v1/rooms/invalidroom/delete"
+        url = self.url_template % "invalidroom"
 
         channel = self.make_request(
-            "POST",
+            self.method,
             url,
             json.dumps({}),
             access_token=self.admin_user_tok,
@@ -236,7 +243,7 @@ def test_new_room_user_does_not_exist(self):
         body = json.dumps({"new_room_user_id": "@unknown:test"})
 
         channel = self.make_request(
-            "POST",
+            self.method,
             self.url,
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -255,7 +262,7 @@ def test_new_room_user_is_not_local(self):
         body = json.dumps({"new_room_user_id": "@not:exist.bla"})
 
         channel = self.make_request(
-            "POST",
+            self.method,
             self.url,
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -274,7 +281,7 @@ def test_block_is_not_bool(self):
         body = json.dumps({"block": "NotBool"})
 
         channel = self.make_request(
-            "POST",
+            self.method,
             self.url,
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -290,7 +297,7 @@ def test_purge_is_not_bool(self):
         body = json.dumps({"purge": "NotBool"})
 
         channel = self.make_request(
-            "POST",
+            self.method,
             self.url,
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -316,7 +323,7 @@ def test_purge_room_and_block(self):
         body = json.dumps({"block": True, "purge": True})
 
         channel = self.make_request(
-            "POST",
+            self.method,
             self.url.encode("ascii"),
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -349,7 +356,7 @@ def test_purge_room_and_not_block(self):
         body = json.dumps({"block": False, "purge": True})
 
         channel = self.make_request(
-            "POST",
+            self.method,
             self.url.encode("ascii"),
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -383,7 +390,7 @@ def test_block_room_and_not_purge(self):
         body = json.dumps({"block": False, "purge": False})
 
         channel = self.make_request(
-            "POST",
+            self.method,
             self.url.encode("ascii"),
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -428,10 +435,9 @@ def test_shutdown_room_consent(self):
         self._is_member(room_id=self.room_id, user_id=self.other_user)
 
         # Test that the admin can still send shutdown
-        url = "/_synapse/admin/v1/rooms/%s/delete" % self.room_id
         channel = self.make_request(
-            "POST",
-            url.encode("ascii"),
+            self.method,
+            self.url,
             json.dumps({"new_room_user_id": self.admin_user}),
             access_token=self.admin_user_tok,
         )
@@ -475,10 +481,9 @@ def test_shutdown_room_block_peek(self):
         self._is_member(room_id=self.room_id, user_id=self.other_user)
 
         # Test that the admin can still send shutdown
-        url = "/_synapse/admin/v1/rooms/%s/delete" % self.room_id
         channel = self.make_request(
-            "POST",
-            url.encode("ascii"),
+            self.method,
+            self.url,
             json.dumps({"new_room_user_id": self.admin_user}),
             access_token=self.admin_user_tok,
         )
@@ -530,7 +535,7 @@ def _is_purged(self, room_id):
                 )
             )
 
-            self.assertEqual(count, 0, msg="Rows not purged in {}".format(table))
+            self.assertEqual(count, 0, msg=f"Rows not purged in {table}")
 
     def _assert_peek(self, room_id, expect_code):
         """Assert that the admin user can (or cannot) peek into the room."""
@@ -594,7 +599,7 @@ def test_purge_room(self):
                 )
             )
 
-            self.assertEqual(count, 0, msg="Rows not purged in {}".format(table))
+            self.assertEqual(count, 0, msg=f"Rows not purged in {table}")
 
 
 class RoomTestCase(unittest.HomeserverTestCase):
@@ -617,7 +622,7 @@ def test_list_rooms(self):
         # Create 3 test rooms
         total_rooms = 3
         room_ids = []
-        for x in range(total_rooms):
+        for _ in range(total_rooms):
             room_id = self.helper.create_room_as(
                 self.admin_user, tok=self.admin_user_tok
             )
@@ -681,7 +686,7 @@ def test_list_rooms_pagination(self):
         # Create 5 test rooms
         total_rooms = 5
         room_ids = []
-        for x in range(total_rooms):
+        for _ in range(total_rooms):
             room_id = self.helper.create_room_as(
                 self.admin_user, tok=self.admin_user_tok
             )
@@ -1275,7 +1280,7 @@ def prepare(self, reactor, clock, homeserver):
         self.public_room_id = self.helper.create_room_as(
             self.creator, tok=self.creator_tok, is_public=True
         )
-        self.url = "/_synapse/admin/v1/join/{}".format(self.public_room_id)
+        self.url = f"/_synapse/admin/v1/join/{self.public_room_id}"
 
     def test_requester_is_no_admin(self):
         """
@@ -1415,7 +1420,7 @@ def test_join_private_room_if_not_member(self):
         private_room_id = self.helper.create_room_as(
             self.creator, tok=self.creator_tok, is_public=False
         )
-        url = "/_synapse/admin/v1/join/{}".format(private_room_id)
+        url = f"/_synapse/admin/v1/join/{private_room_id}"
         body = json.dumps({"user_id": self.second_user_id})
 
         channel = self.make_request(
@@ -1458,7 +1463,7 @@ def test_join_private_room_if_member(self):
 
         # Join user to room.
 
-        url = "/_synapse/admin/v1/join/{}".format(private_room_id)
+        url = f"/_synapse/admin/v1/join/{private_room_id}"
         body = json.dumps({"user_id": self.second_user_id})
 
         channel = self.make_request(
@@ -1488,7 +1493,7 @@ def test_join_private_room_if_owner(self):
         private_room_id = self.helper.create_room_as(
             self.admin_user, tok=self.admin_user_tok, is_public=False
         )
-        url = "/_synapse/admin/v1/join/{}".format(private_room_id)
+        url = f"/_synapse/admin/v1/join/{private_room_id}"
         body = json.dumps({"user_id": self.second_user_id})
 
         channel = self.make_request(
@@ -1579,7 +1584,7 @@ def test_context_as_admin(self):
             channel.json_body["event"]["event_id"], events[midway]["event_id"]
         )
 
-        for i, found_event in enumerate(channel.json_body["events_before"]):
+        for found_event in channel.json_body["events_before"]:
             for j, posted_event in enumerate(events):
                 if found_event["event_id"] == posted_event["event_id"]:
                     self.assertTrue(j < midway)
@@ -1587,7 +1592,7 @@ def test_context_as_admin(self):
             else:
                 self.fail("Event %s from events_before not found" % j)
 
-        for i, found_event in enumerate(channel.json_body["events_after"]):
+        for found_event in channel.json_body["events_after"]:
             for j, posted_event in enumerate(events):
                 if found_event["event_id"] == posted_event["event_id"]:
                     self.assertTrue(j > midway)
@@ -1628,7 +1633,7 @@ def test_public_room(self):
 
         channel = self.make_request(
             "POST",
-            "/_synapse/admin/v1/rooms/{}/make_room_admin".format(room_id),
+            f"/_synapse/admin/v1/rooms/{room_id}/make_room_admin",
             content={},
             access_token=self.admin_user_tok,
         )
@@ -1655,7 +1660,7 @@ def test_private_room(self):
 
         channel = self.make_request(
             "POST",
-            "/_synapse/admin/v1/rooms/{}/make_room_admin".format(room_id),
+            f"/_synapse/admin/v1/rooms/{room_id}/make_room_admin",
             content={},
             access_token=self.admin_user_tok,
         )
@@ -1681,7 +1686,7 @@ def test_other_user(self):
 
         channel = self.make_request(
             "POST",
-            "/_synapse/admin/v1/rooms/{}/make_room_admin".format(room_id),
+            f"/_synapse/admin/v1/rooms/{room_id}/make_room_admin",
             content={"user_id": self.second_user_id},
             access_token=self.admin_user_tok,
         )
@@ -1715,7 +1720,7 @@ def test_not_enough_power(self):
 
         channel = self.make_request(
             "POST",
-            "/_synapse/admin/v1/rooms/{}/make_room_admin".format(room_id),
+            f"/_synapse/admin/v1/rooms/{room_id}/make_room_admin",
             content={},
             access_token=self.admin_user_tok,
         )
@@ -1748,7 +1753,6 @@ def test_not_enough_power(self):
     "room_memberships",
     "room_stats_state",
     "room_stats_current",
-    "room_stats_historical",
     "room_stats_earliest_token",
     "rooms",
     "stream_ordering_to_exterm",
diff --git a/tests/rest/admin/test_statistics.py b/tests/rest/admin/test_statistics.py
index 1f1d11f527..5cd82209c4 100644
--- a/tests/rest/admin/test_statistics.py
+++ b/tests/rest/admin/test_statistics.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Dirk Klimpel
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,7 +18,7 @@
 
 import synapse.rest.admin
 from synapse.api.errors import Codes
-from synapse.rest.client.v1 import login
+from synapse.rest.client import login
 
 from tests import unittest
 
@@ -468,7 +467,7 @@ def _create_media(self, user_token: str, number_media: int):
             number_media: Number of media to be created for the user
         """
         upload_resource = self.media_repo.children[b"upload"]
-        for i in range(number_media):
+        for _ in range(number_media):
             # file size is 67 Byte
             image_data = unhexlify(
                 b"89504e470d0a1a0a0000000d4948445200000001000000010806"
diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py
index 79a05b519b..dfc0a0321b 100644
--- a/tests/rest/admin/test_user.py
+++ b/tests/rest/admin/test_user.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,19 +15,21 @@
 import hashlib
 import hmac
 import json
+import os
 import urllib.parse
 from binascii import unhexlify
 from typing import List, Optional
+from unittest.mock import Mock, patch
 
-from mock import Mock
+from parameterized import parameterized
 
 import synapse.rest.admin
 from synapse.api.constants import UserTypes
 from synapse.api.errors import Codes, HttpResponseException, ResourceLimitError
 from synapse.api.room_versions import RoomVersions
-from synapse.rest.client.v1 import login, logout, profile, room
-from synapse.rest.client.v2_alpha import devices, sync
-from synapse.types import JsonDict
+from synapse.rest.client import devices, login, logout, profile, room, sync
+from synapse.rest.media.v1.filepath import MediaFilePaths
+from synapse.types import JsonDict, UserID
 
 from tests import unittest
 from tests.server import FakeSite, make_request
@@ -56,8 +57,6 @@ def make_homeserver(self, reactor, clock):
         self.datastore = Mock(return_value=Mock())
         self.datastore.get_current_state_deltas = Mock(return_value=(0, []))
 
-        self.secrets = Mock()
-
         self.hs = self.setup_test_homeserver()
 
         self.hs.config.registration_shared_secret = "shared"
@@ -76,7 +75,7 @@ def test_disabled(self):
 
         channel = self.make_request("POST", self.url, b"{}")
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual(
             "Shared secret registration is not enabled", channel.json_body["error"]
         )
@@ -86,14 +85,13 @@ def test_get_nonce(self):
         Calling GET on the endpoint will return a randomised nonce, using the
         homeserver's secrets provider.
         """
-        secrets = Mock()
-        secrets.token_hex = Mock(return_value="abcd")
+        with patch("secrets.token_hex") as token_hex:
+            # Patch secrets.token_hex for the duration of this context
+            token_hex.return_value = "abcd"
 
-        self.hs.get_secrets = Mock(return_value=secrets)
-
-        channel = self.make_request("GET", self.url)
+            channel = self.make_request("GET", self.url)
 
-        self.assertEqual(channel.json_body, {"nonce": "abcd"})
+            self.assertEqual(channel.json_body, {"nonce": "abcd"})
 
     def test_expired_nonce(self):
         """
@@ -109,7 +107,7 @@ def test_expired_nonce(self):
         body = json.dumps({"nonce": nonce})
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("username must be specified", channel.json_body["error"])
 
         # 61 seconds
@@ -117,7 +115,7 @@ def test_expired_nonce(self):
 
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("unrecognised nonce", channel.json_body["error"])
 
     def test_register_incorrect_nonce(self):
@@ -171,7 +169,7 @@ def test_register_correct_nonce(self):
         )
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["user_id"])
 
     def test_nonce_reuse(self):
@@ -196,13 +194,13 @@ def test_nonce_reuse(self):
         )
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["user_id"])
 
         # Now, try and reuse it
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("unrecognised nonce", channel.json_body["error"])
 
     def test_missing_parts(self):
@@ -224,7 +222,7 @@ def nonce():
         body = json.dumps({})
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("nonce must be specified", channel.json_body["error"])
 
         #
@@ -235,28 +233,28 @@ def nonce():
         body = json.dumps({"nonce": nonce()})
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("username must be specified", channel.json_body["error"])
 
         # Must be a string
         body = json.dumps({"nonce": nonce(), "username": 1234})
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("Invalid username", channel.json_body["error"])
 
         # Must not have null bytes
         body = json.dumps({"nonce": nonce(), "username": "abcd\u0000"})
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("Invalid username", channel.json_body["error"])
 
         # Must not have null bytes
         body = json.dumps({"nonce": nonce(), "username": "a" * 1000})
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("Invalid username", channel.json_body["error"])
 
         #
@@ -267,28 +265,28 @@ def nonce():
         body = json.dumps({"nonce": nonce(), "username": "a"})
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("password must be specified", channel.json_body["error"])
 
         # Must be a string
         body = json.dumps({"nonce": nonce(), "username": "a", "password": 1234})
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("Invalid password", channel.json_body["error"])
 
         # Must not have null bytes
         body = json.dumps({"nonce": nonce(), "username": "a", "password": "abcd\u0000"})
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("Invalid password", channel.json_body["error"])
 
         # Super long
         body = json.dumps({"nonce": nonce(), "username": "a", "password": "A" * 1000})
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("Invalid password", channel.json_body["error"])
 
         #
@@ -306,7 +304,7 @@ def nonce():
         )
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual("Invalid user type", channel.json_body["error"])
 
     def test_displayname(self):
@@ -327,11 +325,11 @@ def test_displayname(self):
         )
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob1:test", channel.json_body["user_id"])
 
         channel = self.make_request("GET", "/profile/@bob1:test/displayname")
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("bob1", channel.json_body["displayname"])
 
         # displayname is None
@@ -353,11 +351,11 @@ def test_displayname(self):
         )
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob2:test", channel.json_body["user_id"])
 
         channel = self.make_request("GET", "/profile/@bob2:test/displayname")
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("bob2", channel.json_body["displayname"])
 
         # displayname is empty
@@ -379,7 +377,7 @@ def test_displayname(self):
         )
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob3:test", channel.json_body["user_id"])
 
         channel = self.make_request("GET", "/profile/@bob3:test/displayname")
@@ -404,11 +402,11 @@ def test_displayname(self):
         )
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob4:test", channel.json_body["user_id"])
 
         channel = self.make_request("GET", "/profile/@bob4:test/displayname")
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("Bob's Name", channel.json_body["displayname"])
 
     @override_config(
@@ -454,7 +452,7 @@ def test_register_mau_limit_reached(self):
         )
         channel = self.make_request("POST", self.url, body.encode("utf8"))
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["user_id"])
 
 
@@ -467,6 +465,8 @@ class UsersListTestCase(unittest.HomeserverTestCase):
     url = "/_synapse/admin/v2/users"
 
     def prepare(self, reactor, clock, hs):
+        self.store = hs.get_datastore()
+
         self.admin_user = self.register_user("admin", "pass", admin=True)
         self.admin_user_tok = self.login("admin", "pass")
 
@@ -476,7 +476,7 @@ def test_no_auth(self):
         """
         channel = self.make_request("GET", self.url, b"{}")
 
-        self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(401, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
 
     def test_requester_is_no_admin(self):
@@ -488,7 +488,7 @@ def test_requester_is_no_admin(self):
 
         channel = self.make_request("GET", self.url, access_token=other_user_token)
 
-        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(403, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
 
     def test_all_users(self):
@@ -500,11 +500,11 @@ def test_all_users(self):
         channel = self.make_request(
             "GET",
             self.url + "?deactivated=true",
-            b"{}",
+            {},
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(3, len(channel.json_body["users"]))
         self.assertEqual(3, channel.json_body["total"])
 
@@ -535,7 +535,7 @@ def _search_test(
             )
             channel = self.make_request(
                 "GET",
-                url.encode("ascii"),
+                url,
                 access_token=self.admin_user_tok,
             )
             self.assertEqual(expected_http_code, channel.code, msg=channel.json_body)
@@ -601,7 +601,7 @@ def test_invalid_parameter(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
 
         # negative from
@@ -611,7 +611,7 @@ def test_invalid_parameter(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
 
         # invalid guests
@@ -621,7 +621,7 @@ def test_invalid_parameter(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
 
         # invalid deactivated
@@ -631,7 +631,27 @@ def test_invalid_parameter(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
+
+        # unkown order_by
+        channel = self.make_request(
+            "GET",
+            self.url + "?order_by=bar",
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
+
+        # invalid search order
+        channel = self.make_request(
+            "GET",
+            self.url + "?dir=bar",
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
 
     def test_limit(self):
@@ -649,7 +669,7 @@ def test_limit(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_users)
         self.assertEqual(len(channel.json_body["users"]), 5)
         self.assertEqual(channel.json_body["next_token"], "5")
@@ -670,7 +690,7 @@ def test_from(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_users)
         self.assertEqual(len(channel.json_body["users"]), 15)
         self.assertNotIn("next_token", channel.json_body)
@@ -691,7 +711,7 @@ def test_limit_and_from(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_users)
         self.assertEqual(channel.json_body["next_token"], "15")
         self.assertEqual(len(channel.json_body["users"]), 10)
@@ -714,7 +734,7 @@ def test_next_token(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_users)
         self.assertEqual(len(channel.json_body["users"]), number_users)
         self.assertNotIn("next_token", channel.json_body)
@@ -727,7 +747,7 @@ def test_next_token(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_users)
         self.assertEqual(len(channel.json_body["users"]), number_users)
         self.assertNotIn("next_token", channel.json_body)
@@ -740,7 +760,7 @@ def test_next_token(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_users)
         self.assertEqual(len(channel.json_body["users"]), 19)
         self.assertEqual(channel.json_body["next_token"], "19")
@@ -754,11 +774,116 @@ def test_next_token(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_users)
         self.assertEqual(len(channel.json_body["users"]), 1)
         self.assertNotIn("next_token", channel.json_body)
 
+    def test_order_by(self):
+        """
+        Testing order list with parameter `order_by`
+        """
+
+        # make sure that the users do not have the same timestamps
+        self.reactor.advance(10)
+        user1 = self.register_user("user1", "pass1", admin=False, displayname="Name Z")
+        self.reactor.advance(10)
+        user2 = self.register_user("user2", "pass2", admin=False, displayname="Name Y")
+
+        # Modify user
+        self.get_success(self.store.set_user_deactivated_status(user1, True))
+        self.get_success(self.store.set_shadow_banned(UserID.from_string(user1), True))
+
+        # Set avatar URL to all users, that no user has a NULL value to avoid
+        # different sort order between SQlite and PostreSQL
+        self.get_success(self.store.set_profile_avatar_url("user1", "mxc://url3", 1))
+        self.get_success(self.store.set_profile_avatar_url("user2", "mxc://url2", 1))
+        self.get_success(self.store.set_profile_avatar_url("admin", "mxc://url1", 1))
+
+        # order by default (name)
+        self._order_test([self.admin_user, user1, user2], None)
+        self._order_test([self.admin_user, user1, user2], None, "f")
+        self._order_test([user2, user1, self.admin_user], None, "b")
+
+        # order by name
+        self._order_test([self.admin_user, user1, user2], "name")
+        self._order_test([self.admin_user, user1, user2], "name", "f")
+        self._order_test([user2, user1, self.admin_user], "name", "b")
+
+        # order by displayname
+        self._order_test([user2, user1, self.admin_user], "displayname")
+        self._order_test([user2, user1, self.admin_user], "displayname", "f")
+        self._order_test([self.admin_user, user1, user2], "displayname", "b")
+
+        # order by is_guest
+        # like sort by ascending name, as no guest user here
+        self._order_test([self.admin_user, user1, user2], "is_guest")
+        self._order_test([self.admin_user, user1, user2], "is_guest", "f")
+        self._order_test([self.admin_user, user1, user2], "is_guest", "b")
+
+        # order by admin
+        self._order_test([user1, user2, self.admin_user], "admin")
+        self._order_test([user1, user2, self.admin_user], "admin", "f")
+        self._order_test([self.admin_user, user1, user2], "admin", "b")
+
+        # order by deactivated
+        self._order_test([self.admin_user, user2, user1], "deactivated")
+        self._order_test([self.admin_user, user2, user1], "deactivated", "f")
+        self._order_test([user1, self.admin_user, user2], "deactivated", "b")
+
+        # order by user_type
+        # like sort by ascending name, as no special user type here
+        self._order_test([self.admin_user, user1, user2], "user_type")
+        self._order_test([self.admin_user, user1, user2], "user_type", "f")
+        self._order_test([self.admin_user, user1, user2], "is_guest", "b")
+
+        # order by shadow_banned
+        self._order_test([self.admin_user, user2, user1], "shadow_banned")
+        self._order_test([self.admin_user, user2, user1], "shadow_banned", "f")
+        self._order_test([user1, self.admin_user, user2], "shadow_banned", "b")
+
+        # order by avatar_url
+        self._order_test([self.admin_user, user2, user1], "avatar_url")
+        self._order_test([self.admin_user, user2, user1], "avatar_url", "f")
+        self._order_test([user1, user2, self.admin_user], "avatar_url", "b")
+
+        # order by creation_ts
+        self._order_test([self.admin_user, user1, user2], "creation_ts")
+        self._order_test([self.admin_user, user1, user2], "creation_ts", "f")
+        self._order_test([user2, user1, self.admin_user], "creation_ts", "b")
+
+    def _order_test(
+        self,
+        expected_user_list: List[str],
+        order_by: Optional[str],
+        dir: Optional[str] = None,
+    ):
+        """Request the list of users in a certain order. Assert that order is what
+        we expect
+        Args:
+            expected_user_list: The list of user_id in the order we expect to get
+                back from the server
+            order_by: The type of ordering to give the server
+            dir: The direction of ordering to give the server
+        """
+
+        url = self.url + "?deactivated=true&"
+        if order_by is not None:
+            url += "order_by=%s&" % (order_by,)
+        if dir is not None and dir in ("b", "f"):
+            url += "dir=%s" % (dir,)
+        channel = self.make_request(
+            "GET",
+            url,
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(channel.json_body["total"], len(expected_user_list))
+
+        returned_order = [row["name"] for row in channel.json_body["users"]]
+        self.assertEqual(expected_user_list, returned_order)
+        self._check_fields(channel.json_body["users"])
+
     def _check_fields(self, content: JsonDict):
         """Checks that the expected user attributes are present in content
         Args:
@@ -773,6 +898,7 @@ def _check_fields(self, content: JsonDict):
             self.assertIn("shadow_banned", u)
             self.assertIn("displayname", u)
             self.assertIn("avatar_url", u)
+            self.assertIn("creation_ts", u)
 
     def _create_users(self, number_users: int):
         """
@@ -825,7 +951,7 @@ def test_no_auth(self):
         """
         channel = self.make_request("POST", self.url, b"{}")
 
-        self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(401, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
 
     def test_requester_is_not_admin(self):
@@ -836,7 +962,7 @@ def test_requester_is_not_admin(self):
 
         channel = self.make_request("POST", url, access_token=self.other_user_token)
 
-        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(403, channel.code, msg=channel.json_body)
         self.assertEqual("You are not a server admin", channel.json_body["error"])
 
         channel = self.make_request(
@@ -846,7 +972,7 @@ def test_requester_is_not_admin(self):
             content=b"{}",
         )
 
-        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(403, channel.code, msg=channel.json_body)
         self.assertEqual("You are not a server admin", channel.json_body["error"])
 
     def test_user_does_not_exist(self):
@@ -876,7 +1002,7 @@ def test_erase_is_not_bool(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.BAD_JSON, channel.json_body["errcode"])
 
     def test_user_is_not_local(self):
@@ -892,7 +1018,7 @@ def test_user_is_not_local(self):
 
     def test_deactivate_user_erase_true(self):
         """
-        Test deactivating an user and set `erase` to `true`
+        Test deactivating a user and set `erase` to `true`
         """
 
         # Get user
@@ -902,24 +1028,22 @@ def test_deactivate_user_erase_true(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertEqual(False, channel.json_body["deactivated"])
         self.assertEqual("foo@bar.com", channel.json_body["threepids"][0]["address"])
         self.assertEqual("mxc://servername/mediaid", channel.json_body["avatar_url"])
         self.assertEqual("User1", channel.json_body["displayname"])
 
-        # Deactivate user
-        body = json.dumps({"erase": True})
-
+        # Deactivate and erase user
         channel = self.make_request(
             "POST",
             self.url,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content={"erase": True},
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
         # Get user
         channel = self.make_request(
@@ -928,7 +1052,7 @@ def test_deactivate_user_erase_true(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertEqual(True, channel.json_body["deactivated"])
         self.assertEqual(0, len(channel.json_body["threepids"]))
@@ -939,7 +1063,7 @@ def test_deactivate_user_erase_true(self):
 
     def test_deactivate_user_erase_false(self):
         """
-        Test deactivating an user and set `erase` to `false`
+        Test deactivating a user and set `erase` to `false`
         """
 
         # Get user
@@ -949,7 +1073,7 @@ def test_deactivate_user_erase_false(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertEqual(False, channel.json_body["deactivated"])
         self.assertEqual("foo@bar.com", channel.json_body["threepids"][0]["address"])
@@ -957,16 +1081,14 @@ def test_deactivate_user_erase_false(self):
         self.assertEqual("User1", channel.json_body["displayname"])
 
         # Deactivate user
-        body = json.dumps({"erase": False})
-
         channel = self.make_request(
             "POST",
             self.url,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content={"erase": False},
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
         # Get user
         channel = self.make_request(
@@ -975,7 +1097,7 @@ def test_deactivate_user_erase_false(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertEqual(True, channel.json_body["deactivated"])
         self.assertEqual(0, len(channel.json_body["threepids"]))
@@ -989,6 +1111,60 @@ def test_deactivate_user_erase_false(self):
 
         self._is_erased("@user:test", False)
 
+    def test_deactivate_user_erase_true_no_profile(self):
+        """
+        Test deactivating a user and set `erase` to `true`
+        if user has no profile information (stored in the database table `profiles`).
+        """
+
+        # Users normally have an entry in `profiles`, but occasionally they are created without one.
+        # To test deactivation for users without a profile, we delete the profile information for our user.
+        self.get_success(
+            self.store.db_pool.simple_delete_one(
+                table="profiles", keyvalues={"user_id": "user"}
+            )
+        )
+
+        # Get user
+        channel = self.make_request(
+            "GET",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertEqual(False, channel.json_body["deactivated"])
+        self.assertEqual("foo@bar.com", channel.json_body["threepids"][0]["address"])
+        self.assertIsNone(channel.json_body["avatar_url"])
+        self.assertIsNone(channel.json_body["displayname"])
+
+        # Deactivate and erase user
+        channel = self.make_request(
+            "POST",
+            self.url,
+            access_token=self.admin_user_tok,
+            content={"erase": True},
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+
+        # Get user
+        channel = self.make_request(
+            "GET",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertEqual(True, channel.json_body["deactivated"])
+        self.assertEqual(0, len(channel.json_body["threepids"]))
+        self.assertIsNone(channel.json_body["avatar_url"])
+        self.assertIsNone(channel.json_body["displayname"])
+
+        self._is_erased("@user:test", True)
+
     def _is_erased(self, user_id: str, expect: bool) -> None:
         """Assert that the user is erased or not"""
         d = self.store.is_user_erased(user_id)
@@ -1041,7 +1217,7 @@ def test_requester_is_no_admin(self):
             access_token=self.other_user_token,
         )
 
-        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(403, channel.code, msg=channel.json_body)
         self.assertEqual("You are not a server admin", channel.json_body["error"])
 
         channel = self.make_request(
@@ -1051,7 +1227,7 @@ def test_requester_is_no_admin(self):
             content=b"{}",
         )
 
-        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(403, channel.code, msg=channel.json_body)
         self.assertEqual("You are not a server admin", channel.json_body["error"])
 
     def test_user_does_not_exist(self):
@@ -1068,6 +1244,116 @@ def test_user_does_not_exist(self):
         self.assertEqual(404, channel.code, msg=channel.json_body)
         self.assertEqual("M_NOT_FOUND", channel.json_body["errcode"])
 
+    def test_invalid_parameter(self):
+        """
+        If parameters are invalid, an error is returned.
+        """
+
+        # admin not bool
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"admin": "not_bool"},
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.BAD_JSON, channel.json_body["errcode"])
+
+        # deactivated not bool
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"deactivated": "not_bool"},
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
+
+        # password not str
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"password": True},
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
+
+        # password not length
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"password": "x" * 513},
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
+
+        # user_type not valid
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"user_type": "new type"},
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
+
+        # external_ids not valid
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={
+                "external_ids": {"auth_provider": "prov", "wrong_external_id": "id"}
+            },
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"])
+
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"external_ids": {"external_id": "id"}},
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"])
+
+        # threepids not valid
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"threepids": {"medium": "email", "wrong_address": "id"}},
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"])
+
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"threepids": {"address": "value"}},
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"])
+
+    def test_get_user(self):
+        """
+        Test a simple get of a user.
+        """
+        channel = self.make_request(
+            "GET",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertEqual("User", channel.json_body["displayname"])
+        self._check_fields(channel.json_body)
+
     def test_create_server_admin(self):
         """
         Check that a new admin user is created successfully.
@@ -1075,30 +1361,29 @@ def test_create_server_admin(self):
         url = "/_synapse/admin/v2/users/@bob:test"
 
         # Create user (server admin)
-        body = json.dumps(
-            {
-                "password": "abc123",
-                "admin": True,
-                "displayname": "Bob's name",
-                "threepids": [{"medium": "email", "address": "bob@bob.bob"}],
-                "avatar_url": "mxc://fibble/wibble",
-            }
-        )
+        body = {
+            "password": "abc123",
+            "admin": True,
+            "displayname": "Bob's name",
+            "threepids": [{"medium": "email", "address": "bob@bob.bob"}],
+            "avatar_url": "mxc://fibble/wibble",
+        }
 
         channel = self.make_request(
             "PUT",
             url,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content=body,
         )
 
-        self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(201, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertEqual("Bob's name", channel.json_body["displayname"])
         self.assertEqual("email", channel.json_body["threepids"][0]["medium"])
         self.assertEqual("bob@bob.bob", channel.json_body["threepids"][0]["address"])
         self.assertTrue(channel.json_body["admin"])
         self.assertEqual("mxc://fibble/wibble", channel.json_body["avatar_url"])
+        self._check_fields(channel.json_body)
 
         # Get user
         channel = self.make_request(
@@ -1107,7 +1392,7 @@ def test_create_server_admin(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertEqual("Bob's name", channel.json_body["displayname"])
         self.assertEqual("email", channel.json_body["threepids"][0]["medium"])
@@ -1116,6 +1401,7 @@ def test_create_server_admin(self):
         self.assertFalse(channel.json_body["is_guest"])
         self.assertFalse(channel.json_body["deactivated"])
         self.assertEqual("mxc://fibble/wibble", channel.json_body["avatar_url"])
+        self._check_fields(channel.json_body)
 
     def test_create_user(self):
         """
@@ -1124,30 +1410,41 @@ def test_create_user(self):
         url = "/_synapse/admin/v2/users/@bob:test"
 
         # Create user
-        body = json.dumps(
-            {
-                "password": "abc123",
-                "admin": False,
-                "displayname": "Bob's name",
-                "threepids": [{"medium": "email", "address": "bob@bob.bob"}],
-                "avatar_url": "mxc://fibble/wibble",
-            }
-        )
+        body = {
+            "password": "abc123",
+            "admin": False,
+            "displayname": "Bob's name",
+            "threepids": [{"medium": "email", "address": "bob@bob.bob"}],
+            "external_ids": [
+                {
+                    "external_id": "external_id1",
+                    "auth_provider": "auth_provider1",
+                },
+            ],
+            "avatar_url": "mxc://fibble/wibble",
+        }
 
         channel = self.make_request(
             "PUT",
             url,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content=body,
         )
 
-        self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(201, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertEqual("Bob's name", channel.json_body["displayname"])
         self.assertEqual("email", channel.json_body["threepids"][0]["medium"])
         self.assertEqual("bob@bob.bob", channel.json_body["threepids"][0]["address"])
+        self.assertEqual(
+            "external_id1", channel.json_body["external_ids"][0]["external_id"]
+        )
+        self.assertEqual(
+            "auth_provider1", channel.json_body["external_ids"][0]["auth_provider"]
+        )
         self.assertFalse(channel.json_body["admin"])
         self.assertEqual("mxc://fibble/wibble", channel.json_body["avatar_url"])
+        self._check_fields(channel.json_body)
 
         # Get user
         channel = self.make_request(
@@ -1156,7 +1453,7 @@ def test_create_user(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertEqual("Bob's name", channel.json_body["displayname"])
         self.assertEqual("email", channel.json_body["threepids"][0]["medium"])
@@ -1166,6 +1463,7 @@ def test_create_user(self):
         self.assertFalse(channel.json_body["deactivated"])
         self.assertFalse(channel.json_body["shadow_banned"])
         self.assertEqual("mxc://fibble/wibble", channel.json_body["avatar_url"])
+        self._check_fields(channel.json_body)
 
     @override_config(
         {"limit_usage_by_mau": True, "max_mau_value": 2, "mau_trial_days": 0}
@@ -1202,16 +1500,14 @@ def test_create_user_mau_limit_reached_active_admin(self):
         url = "/_synapse/admin/v2/users/@bob:test"
 
         # Create user
-        body = json.dumps({"password": "abc123", "admin": False})
-
         channel = self.make_request(
             "PUT",
             url,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content={"password": "abc123", "admin": False},
         )
 
-        self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(201, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertFalse(channel.json_body["admin"])
 
@@ -1241,17 +1537,15 @@ def test_create_user_mau_limit_reached_passive_admin(self):
         url = "/_synapse/admin/v2/users/@bob:test"
 
         # Create user
-        body = json.dumps({"password": "abc123", "admin": False})
-
         channel = self.make_request(
             "PUT",
             url,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content={"password": "abc123", "admin": False},
         )
 
         # Admin user is not blocked by mau anymore
-        self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(201, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertFalse(channel.json_body["admin"])
 
@@ -1273,21 +1567,19 @@ def test_create_user_email_notif_for_new_users(self):
         url = "/_synapse/admin/v2/users/@bob:test"
 
         # Create user
-        body = json.dumps(
-            {
-                "password": "abc123",
-                "threepids": [{"medium": "email", "address": "bob@bob.bob"}],
-            }
-        )
+        body = {
+            "password": "abc123",
+            "threepids": [{"medium": "email", "address": "bob@bob.bob"}],
+        }
 
         channel = self.make_request(
             "PUT",
             url,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content=body,
         )
 
-        self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(201, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertEqual("email", channel.json_body["threepids"][0]["medium"])
         self.assertEqual("bob@bob.bob", channel.json_body["threepids"][0]["address"])
@@ -1317,21 +1609,19 @@ def test_create_user_email_no_notif_for_new_users(self):
         url = "/_synapse/admin/v2/users/@bob:test"
 
         # Create user
-        body = json.dumps(
-            {
-                "password": "abc123",
-                "threepids": [{"medium": "email", "address": "bob@bob.bob"}],
-            }
-        )
+        body = {
+            "password": "abc123",
+            "threepids": [{"medium": "email", "address": "bob@bob.bob"}],
+        }
 
         channel = self.make_request(
             "PUT",
             url,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content=body,
         )
 
-        self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(201, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertEqual("email", channel.json_body["threepids"][0]["medium"])
         self.assertEqual("bob@bob.bob", channel.json_body["threepids"][0]["address"])
@@ -1348,16 +1638,15 @@ def test_set_password(self):
         """
 
         # Change password
-        body = json.dumps({"password": "hahaha"})
-
         channel = self.make_request(
             "PUT",
             self.url_other_user,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content={"password": "hahaha"},
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self._check_fields(channel.json_body)
 
     def test_set_displayname(self):
         """
@@ -1365,16 +1654,14 @@ def test_set_displayname(self):
         """
 
         # Modify user
-        body = json.dumps({"displayname": "foobar"})
-
         channel = self.make_request(
             "PUT",
             self.url_other_user,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content={"displayname": "foobar"},
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertEqual("foobar", channel.json_body["displayname"])
 
@@ -1385,7 +1672,7 @@ def test_set_displayname(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertEqual("foobar", channel.json_body["displayname"])
 
@@ -1395,18 +1682,14 @@ def test_set_threepid(self):
         """
 
         # Delete old and add new threepid to user
-        body = json.dumps(
-            {"threepids": [{"medium": "email", "address": "bob3@bob.bob"}]}
-        )
-
         channel = self.make_request(
             "PUT",
             self.url_other_user,
             access_token=self.admin_user_tok,
-            content=body.encode(encoding="utf_8"),
+            content={"threepids": [{"medium": "email", "address": "bob3@bob.bob"}]},
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertEqual("email", channel.json_body["threepids"][0]["medium"])
         self.assertEqual("bob3@bob.bob", channel.json_body["threepids"][0]["address"])
@@ -1418,11 +1701,108 @@ def test_set_threepid(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertEqual("email", channel.json_body["threepids"][0]["medium"])
         self.assertEqual("bob3@bob.bob", channel.json_body["threepids"][0]["address"])
 
+    def test_set_external_id(self):
+        """
+        Test setting external id for an other user.
+        """
+
+        # Add two external_ids
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={
+                "external_ids": [
+                    {
+                        "external_id": "external_id1",
+                        "auth_provider": "auth_provider1",
+                    },
+                    {
+                        "external_id": "external_id2",
+                        "auth_provider": "auth_provider2",
+                    },
+                ]
+            },
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertEqual(2, len(channel.json_body["external_ids"]))
+        # result does not always have the same sort order, therefore it becomes sorted
+        self.assertEqual(
+            sorted(channel.json_body["external_ids"], key=lambda k: k["auth_provider"]),
+            [
+                {"auth_provider": "auth_provider1", "external_id": "external_id1"},
+                {"auth_provider": "auth_provider2", "external_id": "external_id2"},
+            ],
+        )
+        self._check_fields(channel.json_body)
+
+        # Set a new and remove an external_id
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={
+                "external_ids": [
+                    {
+                        "external_id": "external_id2",
+                        "auth_provider": "auth_provider2",
+                    },
+                    {
+                        "external_id": "external_id3",
+                        "auth_provider": "auth_provider3",
+                    },
+                ]
+            },
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertEqual(2, len(channel.json_body["external_ids"]))
+        self.assertEqual(
+            channel.json_body["external_ids"],
+            [
+                {"auth_provider": "auth_provider2", "external_id": "external_id2"},
+                {"auth_provider": "auth_provider3", "external_id": "external_id3"},
+            ],
+        )
+        self._check_fields(channel.json_body)
+
+        # Get user
+        channel = self.make_request(
+            "GET",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertEqual(
+            channel.json_body["external_ids"],
+            [
+                {"auth_provider": "auth_provider2", "external_id": "external_id2"},
+                {"auth_provider": "auth_provider3", "external_id": "external_id3"},
+            ],
+        )
+        self._check_fields(channel.json_body)
+
+        # Remove external_ids
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"external_ids": []},
+        )
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertEqual(0, len(channel.json_body["external_ids"]))
+
     def test_deactivate_user(self):
         """
         Test deactivating another user.
@@ -1443,7 +1823,7 @@ def test_deactivate_user(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertFalse(channel.json_body["deactivated"])
         self.assertEqual("foo@bar.com", channel.json_body["threepids"][0]["address"])
@@ -1463,7 +1843,7 @@ def test_deactivate_user(self):
             content={"deactivated": True},
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertTrue(channel.json_body["deactivated"])
         self.assertIsNone(channel.json_body["password_hash"])
@@ -1484,7 +1864,7 @@ def test_deactivate_user(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertTrue(channel.json_body["deactivated"])
         self.assertIsNone(channel.json_body["password_hash"])
@@ -1511,7 +1891,7 @@ def test_change_name_deactivate_user_user_directory(self):
             content={"deactivated": True},
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertTrue(channel.json_body["deactivated"])
 
@@ -1527,7 +1907,7 @@ def test_change_name_deactivate_user_user_directory(self):
             content={"displayname": "Foobar"},
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertTrue(channel.json_body["deactivated"])
         self.assertEqual("Foobar", channel.json_body["displayname"])
@@ -1551,7 +1931,7 @@ def test_reactivate_user(self):
             access_token=self.admin_user_tok,
             content={"deactivated": False},
         )
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
 
         # Reactivate the user.
         channel = self.make_request(
@@ -1560,7 +1940,7 @@ def test_reactivate_user(self):
             access_token=self.admin_user_tok,
             content={"deactivated": False, "password": "foo"},
         )
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertFalse(channel.json_body["deactivated"])
         self.assertIsNotNone(channel.json_body["password_hash"])
@@ -1582,7 +1962,7 @@ def test_reactivate_user_localdb_disabled(self):
             access_token=self.admin_user_tok,
             content={"deactivated": False, "password": "foo"},
         )
-        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(403, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
 
         # Reactivate the user without a password.
@@ -1592,7 +1972,7 @@ def test_reactivate_user_localdb_disabled(self):
             access_token=self.admin_user_tok,
             content={"deactivated": False},
         )
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertFalse(channel.json_body["deactivated"])
         self.assertIsNone(channel.json_body["password_hash"])
@@ -1614,7 +1994,7 @@ def test_reactivate_user_password_disabled(self):
             access_token=self.admin_user_tok,
             content={"deactivated": False, "password": "foo"},
         )
-        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(403, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
 
         # Reactivate the user without a password.
@@ -1624,7 +2004,7 @@ def test_reactivate_user_password_disabled(self):
             access_token=self.admin_user_tok,
             content={"deactivated": False},
         )
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertFalse(channel.json_body["deactivated"])
         self.assertIsNone(channel.json_body["password_hash"])
@@ -1643,7 +2023,7 @@ def test_set_user_as_admin(self):
             content={"admin": True},
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertTrue(channel.json_body["admin"])
 
@@ -1654,7 +2034,7 @@ def test_set_user_as_admin(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertTrue(channel.json_body["admin"])
 
@@ -1673,7 +2053,7 @@ def test_accidental_deactivation_prevention(self):
             content={"password": "abc123"},
         )
 
-        self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(201, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertEqual("bob", channel.json_body["displayname"])
 
@@ -1684,7 +2064,7 @@ def test_accidental_deactivation_prevention(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertEqual("bob", channel.json_body["displayname"])
         self.assertEqual(0, channel.json_body["deactivated"])
@@ -1697,7 +2077,7 @@ def test_accidental_deactivation_prevention(self):
             content={"password": "abc123", "deactivated": "false"},
         )
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
 
         # Check user is not deactivated
         channel = self.make_request(
@@ -1706,7 +2086,7 @@ def test_accidental_deactivation_prevention(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual("@bob:test", channel.json_body["name"])
         self.assertEqual("bob", channel.json_body["displayname"])
 
@@ -1731,7 +2111,7 @@ def _deactivate_user(self, user_id: str) -> None:
             access_token=self.admin_user_tok,
             content={"deactivated": True},
         )
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertTrue(channel.json_body["deactivated"])
         self.assertIsNone(channel.json_body["password_hash"])
         self._is_erased(user_id, False)
@@ -1739,6 +2119,25 @@ def _deactivate_user(self, user_id: str) -> None:
         self.assertIsNone(self.get_success(d))
         self._is_erased(user_id, True)
 
+    def _check_fields(self, content: JsonDict):
+        """Checks that the expected user attributes are present in content
+
+        Args:
+            content: Content dictionary to check
+        """
+        self.assertIn("displayname", content)
+        self.assertIn("threepids", content)
+        self.assertIn("avatar_url", content)
+        self.assertIn("admin", content)
+        self.assertIn("deactivated", content)
+        self.assertIn("shadow_banned", content)
+        self.assertIn("password_hash", content)
+        self.assertIn("creation_ts", content)
+        self.assertIn("appservice_id", content)
+        self.assertIn("consent_server_notice_sent", content)
+        self.assertIn("consent_version", content)
+        self.assertIn("external_ids", content)
+
 
 class UserMembershipRestTestCase(unittest.HomeserverTestCase):
 
@@ -1835,7 +2234,7 @@ def test_get_rooms(self):
         # Create rooms and join
         other_user_tok = self.login("user", "pass")
         number_rooms = 5
-        for n in range(number_rooms):
+        for _ in range(number_rooms):
             self.helper.create_room_as(self.other_user, tok=other_user_tok)
 
         # Get rooms
@@ -1966,7 +2365,7 @@ def test_user_is_not_local(self):
         )
 
         self.assertEqual(400, channel.code, msg=channel.json_body)
-        self.assertEqual("Can only lookup local users", channel.json_body["error"])
+        self.assertEqual("Can only look up local users", channel.json_body["error"])
 
     def test_get_pushers(self):
         """
@@ -2035,6 +2434,7 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase):
     def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
         self.media_repo = hs.get_media_repository_resource()
+        self.filepaths = MediaFilePaths(hs.config.media_store_path)
 
         self.admin_user = self.register_user("admin", "pass", admin=True)
         self.admin_user_tok = self.login("admin", "pass")
@@ -2044,37 +2444,34 @@ def prepare(self, reactor, clock, hs):
             self.other_user
         )
 
-    def test_no_auth(self):
-        """
-        Try to list media of an user without authentication.
-        """
-        channel = self.make_request("GET", self.url, b"{}")
+    @parameterized.expand(["GET", "DELETE"])
+    def test_no_auth(self, method: str):
+        """Try to list media of an user without authentication."""
+        channel = self.make_request(method, self.url, {})
 
-        self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(401, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
 
-    def test_requester_is_no_admin(self):
-        """
-        If the user is not a server admin, an error is returned.
-        """
+    @parameterized.expand(["GET", "DELETE"])
+    def test_requester_is_no_admin(self, method: str):
+        """If the user is not a server admin, an error is returned."""
         other_user_token = self.login("user", "pass")
 
         channel = self.make_request(
-            "GET",
+            method,
             self.url,
             access_token=other_user_token,
         )
 
-        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(403, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
 
-    def test_user_does_not_exist(self):
-        """
-        Tests that a lookup for a user that does not exist returns a 404
-        """
+    @parameterized.expand(["GET", "DELETE"])
+    def test_user_does_not_exist(self, method: str):
+        """Tests that a lookup for a user that does not exist returns a 404"""
         url = "/_synapse/admin/v1/users/@unknown_person:test/media"
         channel = self.make_request(
-            "GET",
+            method,
             url,
             access_token=self.admin_user_tok,
         )
@@ -2082,25 +2479,22 @@ def test_user_does_not_exist(self):
         self.assertEqual(404, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
 
-    def test_user_is_not_local(self):
-        """
-        Tests that a lookup for a user that is not a local returns a 400
-        """
+    @parameterized.expand(["GET", "DELETE"])
+    def test_user_is_not_local(self, method: str):
+        """Tests that a lookup for a user that is not a local returns a 400"""
         url = "/_synapse/admin/v1/users/@unknown_person:unknown_domain/media"
 
         channel = self.make_request(
-            "GET",
+            method,
             url,
             access_token=self.admin_user_tok,
         )
 
         self.assertEqual(400, channel.code, msg=channel.json_body)
-        self.assertEqual("Can only lookup local users", channel.json_body["error"])
+        self.assertEqual("Can only look up local users", channel.json_body["error"])
 
-    def test_limit(self):
-        """
-        Testing list of media with limit
-        """
+    def test_limit_GET(self):
+        """Testing list of media with limit"""
 
         number_media = 20
         other_user_tok = self.login("user", "pass")
@@ -2112,16 +2506,31 @@ def test_limit(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_media)
         self.assertEqual(len(channel.json_body["media"]), 5)
         self.assertEqual(channel.json_body["next_token"], 5)
         self._check_fields(channel.json_body["media"])
 
-    def test_from(self):
-        """
-        Testing list of media with a defined starting point (from)
-        """
+    def test_limit_DELETE(self):
+        """Testing delete of media with limit"""
+
+        number_media = 20
+        other_user_tok = self.login("user", "pass")
+        self._create_media_for_user(other_user_tok, number_media)
+
+        channel = self.make_request(
+            "DELETE",
+            self.url + "?limit=5",
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(channel.json_body["total"], 5)
+        self.assertEqual(len(channel.json_body["deleted_media"]), 5)
+
+    def test_from_GET(self):
+        """Testing list of media with a defined starting point (from)"""
 
         number_media = 20
         other_user_tok = self.login("user", "pass")
@@ -2133,16 +2542,31 @@ def test_from(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_media)
         self.assertEqual(len(channel.json_body["media"]), 15)
         self.assertNotIn("next_token", channel.json_body)
         self._check_fields(channel.json_body["media"])
 
-    def test_limit_and_from(self):
-        """
-        Testing list of media with a defined starting point and limit
-        """
+    def test_from_DELETE(self):
+        """Testing delete of media with a defined starting point (from)"""
+
+        number_media = 20
+        other_user_tok = self.login("user", "pass")
+        self._create_media_for_user(other_user_tok, number_media)
+
+        channel = self.make_request(
+            "DELETE",
+            self.url + "?from=5",
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(channel.json_body["total"], 15)
+        self.assertEqual(len(channel.json_body["deleted_media"]), 15)
+
+    def test_limit_and_from_GET(self):
+        """Testing list of media with a defined starting point and limit"""
 
         number_media = 20
         other_user_tok = self.login("user", "pass")
@@ -2154,59 +2578,78 @@ def test_limit_and_from(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_media)
         self.assertEqual(channel.json_body["next_token"], 15)
         self.assertEqual(len(channel.json_body["media"]), 10)
         self._check_fields(channel.json_body["media"])
 
-    def test_invalid_parameter(self):
-        """
-        If parameters are invalid, an error is returned.
-        """
+    def test_limit_and_from_DELETE(self):
+        """Testing delete of media with a defined starting point and limit"""
+
+        number_media = 20
+        other_user_tok = self.login("user", "pass")
+        self._create_media_for_user(other_user_tok, number_media)
+
+        channel = self.make_request(
+            "DELETE",
+            self.url + "?from=5&limit=10",
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(channel.json_body["total"], 10)
+        self.assertEqual(len(channel.json_body["deleted_media"]), 10)
+
+    @parameterized.expand(["GET", "DELETE"])
+    def test_invalid_parameter(self, method: str):
+        """If parameters are invalid, an error is returned."""
         # unkown order_by
         channel = self.make_request(
-            "GET",
+            method,
             self.url + "?order_by=bar",
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
 
         # invalid search order
         channel = self.make_request(
-            "GET",
+            method,
             self.url + "?dir=bar",
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
 
         # negative limit
         channel = self.make_request(
-            "GET",
+            method,
             self.url + "?limit=-5",
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
 
         # negative from
         channel = self.make_request(
-            "GET",
+            method,
             self.url + "?from=-5",
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(400, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
 
     def test_next_token(self):
         """
         Testing that `next_token` appears at the right place
+
+        For deleting media `next_token` is not useful, because
+        after deleting media the media has a new order.
         """
 
         number_media = 20
@@ -2221,7 +2664,7 @@ def test_next_token(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_media)
         self.assertEqual(len(channel.json_body["media"]), number_media)
         self.assertNotIn("next_token", channel.json_body)
@@ -2234,7 +2677,7 @@ def test_next_token(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_media)
         self.assertEqual(len(channel.json_body["media"]), number_media)
         self.assertNotIn("next_token", channel.json_body)
@@ -2247,7 +2690,7 @@ def test_next_token(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_media)
         self.assertEqual(len(channel.json_body["media"]), 19)
         self.assertEqual(channel.json_body["next_token"], 19)
@@ -2261,12 +2704,12 @@ def test_next_token(self):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         self.assertEqual(channel.json_body["total"], number_media)
         self.assertEqual(len(channel.json_body["media"]), 1)
         self.assertNotIn("next_token", channel.json_body)
 
-    def test_user_has_no_media(self):
+    def test_user_has_no_media_GET(self):
         """
         Tests that a normal lookup for media is successfully
         if user has no media created
@@ -2282,11 +2725,24 @@ def test_user_has_no_media(self):
         self.assertEqual(0, channel.json_body["total"])
         self.assertEqual(0, len(channel.json_body["media"]))
 
-    def test_get_media(self):
+    def test_user_has_no_media_DELETE(self):
         """
-        Tests that a normal lookup for media is successfully
+        Tests that a delete is successful if user has no media
         """
 
+        channel = self.make_request(
+            "DELETE",
+            self.url,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(0, channel.json_body["total"])
+        self.assertEqual(0, len(channel.json_body["deleted_media"]))
+
+    def test_get_media(self):
+        """Tests that a normal lookup for media is successful"""
+
         number_media = 5
         other_user_tok = self.login("user", "pass")
         self._create_media_for_user(other_user_tok, number_media)
@@ -2303,6 +2759,35 @@ def test_get_media(self):
         self.assertNotIn("next_token", channel.json_body)
         self._check_fields(channel.json_body["media"])
 
+    def test_delete_media(self):
+        """Tests that a normal delete of media is successful"""
+
+        number_media = 5
+        other_user_tok = self.login("user", "pass")
+        media_ids = self._create_media_for_user(other_user_tok, number_media)
+
+        # Test if the file exists
+        local_paths = []
+        for media_id in media_ids:
+            local_path = self.filepaths.local_media_filepath(media_id)
+            self.assertTrue(os.path.exists(local_path))
+            local_paths.append(local_path)
+
+        channel = self.make_request(
+            "DELETE",
+            self.url,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(number_media, channel.json_body["total"])
+        self.assertEqual(number_media, len(channel.json_body["deleted_media"]))
+        self.assertCountEqual(channel.json_body["deleted_media"], media_ids)
+
+        # Test if the file is deleted
+        for local_path in local_paths:
+            self.assertFalse(os.path.exists(local_path))
+
     def test_order_by(self):
         """
         Testing order list with parameter `order_by`
@@ -2408,14 +2893,17 @@ def test_order_by(self):
             [media2] + sorted([media1, media3]), "safe_from_quarantine", "b"
         )
 
-    def _create_media_for_user(self, user_token: str, number_media: int):
+    def _create_media_for_user(self, user_token: str, number_media: int) -> List[str]:
         """
         Create a number of media for a specific user
         Args:
             user_token: Access token of the user
             number_media: Number of media to be created for the user
+        Returns:
+            List of created media ID
         """
-        for i in range(number_media):
+        media_ids = []
+        for _ in range(number_media):
             # file size is 67 Byte
             image_data = unhexlify(
                 b"89504e470d0a1a0a0000000d4948445200000001000000010806"
@@ -2423,7 +2911,9 @@ def _create_media_for_user(self, user_token: str, number_media: int):
                 b"0a2db40000000049454e44ae426082"
             )
 
-            self._create_media_and_access(user_token, image_data)
+            media_ids.append(self._create_media_and_access(user_token, image_data))
+
+        return media_ids
 
     def _create_media_and_access(
         self,
@@ -2466,7 +2956,7 @@ def _create_media_and_access(
             200,
             channel.code,
             msg=(
-                "Expected to receive a 200 on accessing media: %s" % server_and_media_id
+                f"Expected to receive a 200 on accessing media: {server_and_media_id}"
             ),
         )
 
@@ -2504,12 +2994,12 @@ def _order_test(
 
         url = self.url + "?"
         if order_by is not None:
-            url += "order_by=%s&" % (order_by,)
+            url += f"order_by={order_by}&"
         if dir is not None and dir in ("b", "f"):
-            url += "dir=%s" % (dir,)
+            url += f"dir={dir}"
         channel = self.make_request(
             "GET",
-            url.encode("ascii"),
+            url,
             access_token=self.admin_user_tok,
         )
         self.assertEqual(200, channel.code, msg=channel.json_body)
@@ -2548,7 +3038,7 @@ def _get_token(self) -> str:
         channel = self.make_request(
             "POST", self.url, b"{}", access_token=self.admin_user_tok
         )
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
         return channel.json_body["access_token"]
 
     def test_no_auth(self):
@@ -2589,7 +3079,7 @@ def test_devices(self):
         channel = self.make_request(
             "GET", "devices", b"{}", access_token=self.other_user_tok
         )
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
         # We should only see the one device (from the login in `prepare`)
         self.assertEqual(len(channel.json_body["devices"]), 1)
@@ -2601,11 +3091,11 @@ def test_logout(self):
 
         # Test that we can successfully make a request
         channel = self.make_request("GET", "devices", b"{}", access_token=puppet_token)
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
         # Logout with the puppet token
         channel = self.make_request("POST", "logout", b"{}", access_token=puppet_token)
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
         # The puppet token should no longer work
         channel = self.make_request("GET", "devices", b"{}", access_token=puppet_token)
@@ -2615,7 +3105,7 @@ def test_logout(self):
         channel = self.make_request(
             "GET", "devices", b"{}", access_token=self.other_user_tok
         )
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
     def test_user_logout_all(self):
         """Tests that the target user calling `/logout/all` does *not* expire
@@ -2626,17 +3116,17 @@ def test_user_logout_all(self):
 
         # Test that we can successfully make a request
         channel = self.make_request("GET", "devices", b"{}", access_token=puppet_token)
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
         # Logout all with the real user token
         channel = self.make_request(
             "POST", "logout/all", b"{}", access_token=self.other_user_tok
         )
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
         # The puppet token should still work
         channel = self.make_request("GET", "devices", b"{}", access_token=puppet_token)
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
         # .. but the real user's tokens shouldn't
         channel = self.make_request(
@@ -2653,13 +3143,13 @@ def test_admin_logout_all(self):
 
         # Test that we can successfully make a request
         channel = self.make_request("GET", "devices", b"{}", access_token=puppet_token)
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
         # Logout all with the admin user token
         channel = self.make_request(
             "POST", "logout/all", b"{}", access_token=self.admin_user_tok
         )
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
         # The puppet token should no longer work
         channel = self.make_request("GET", "devices", b"{}", access_token=puppet_token)
@@ -2669,7 +3159,7 @@ def test_admin_logout_all(self):
         channel = self.make_request(
             "GET", "devices", b"{}", access_token=self.other_user_tok
         )
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(200, channel.code, msg=channel.json_body)
 
     @unittest.override_config(
         {
@@ -2908,3 +3398,287 @@ def test_success(self):
         # Ensure the user is shadow-banned (and the cache was cleared).
         result = self.get_success(self.store.get_user_by_access_token(other_user_token))
         self.assertTrue(result.shadow_banned)
+
+
+class RateLimitTestCase(unittest.HomeserverTestCase):
+
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        self.store = hs.get_datastore()
+
+        self.admin_user = self.register_user("admin", "pass", admin=True)
+        self.admin_user_tok = self.login("admin", "pass")
+
+        self.other_user = self.register_user("user", "pass")
+        self.url = (
+            "/_synapse/admin/v1/users/%s/override_ratelimit"
+            % urllib.parse.quote(self.other_user)
+        )
+
+    def test_no_auth(self):
+        """
+        Try to get information of a user without authentication.
+        """
+        channel = self.make_request("GET", self.url, b"{}")
+
+        self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
+
+        channel = self.make_request("POST", self.url, b"{}")
+
+        self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
+
+        channel = self.make_request("DELETE", self.url, b"{}")
+
+        self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
+
+    def test_requester_is_no_admin(self):
+        """
+        If the user is not a server admin, an error is returned.
+        """
+        other_user_token = self.login("user", "pass")
+
+        channel = self.make_request(
+            "GET",
+            self.url,
+            access_token=other_user_token,
+        )
+
+        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
+
+        channel = self.make_request(
+            "POST",
+            self.url,
+            access_token=other_user_token,
+        )
+
+        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
+
+        channel = self.make_request(
+            "DELETE",
+            self.url,
+            access_token=other_user_token,
+        )
+
+        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
+
+    def test_user_does_not_exist(self):
+        """
+        Tests that a lookup for a user that does not exist returns a 404
+        """
+        url = "/_synapse/admin/v1/users/@unknown_person:test/override_ratelimit"
+
+        channel = self.make_request(
+            "GET",
+            url,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(404, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
+
+        channel = self.make_request(
+            "POST",
+            url,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(404, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
+
+        channel = self.make_request(
+            "DELETE",
+            url,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(404, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
+
+    def test_user_is_not_local(self):
+        """
+        Tests that a lookup for a user that is not a local returns a 400
+        """
+        url = (
+            "/_synapse/admin/v1/users/@unknown_person:unknown_domain/override_ratelimit"
+        )
+
+        channel = self.make_request(
+            "GET",
+            url,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual("Can only look up local users", channel.json_body["error"])
+
+        channel = self.make_request(
+            "POST",
+            url,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(
+            "Only local users can be ratelimited", channel.json_body["error"]
+        )
+
+        channel = self.make_request(
+            "DELETE",
+            url,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(
+            "Only local users can be ratelimited", channel.json_body["error"]
+        )
+
+    def test_invalid_parameter(self):
+        """
+        If parameters are invalid, an error is returned.
+        """
+        # messages_per_second is a string
+        channel = self.make_request(
+            "POST",
+            self.url,
+            access_token=self.admin_user_tok,
+            content={"messages_per_second": "string"},
+        )
+
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
+
+        # messages_per_second is negative
+        channel = self.make_request(
+            "POST",
+            self.url,
+            access_token=self.admin_user_tok,
+            content={"messages_per_second": -1},
+        )
+
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
+
+        # burst_count is a string
+        channel = self.make_request(
+            "POST",
+            self.url,
+            access_token=self.admin_user_tok,
+            content={"burst_count": "string"},
+        )
+
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
+
+        # burst_count is negative
+        channel = self.make_request(
+            "POST",
+            self.url,
+            access_token=self.admin_user_tok,
+            content={"burst_count": -1},
+        )
+
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
+
+    def test_return_zero_when_null(self):
+        """
+        If values in database are `null` API should return an int `0`
+        """
+
+        self.get_success(
+            self.store.db_pool.simple_upsert(
+                table="ratelimit_override",
+                keyvalues={"user_id": self.other_user},
+                values={
+                    "messages_per_second": None,
+                    "burst_count": None,
+                },
+            )
+        )
+
+        # request status
+        channel = self.make_request(
+            "GET",
+            self.url,
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(0, channel.json_body["messages_per_second"])
+        self.assertEqual(0, channel.json_body["burst_count"])
+
+    def test_success(self):
+        """
+        Rate-limiting (set/update/delete) should succeed for an admin.
+        """
+        # request status
+        channel = self.make_request(
+            "GET",
+            self.url,
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertNotIn("messages_per_second", channel.json_body)
+        self.assertNotIn("burst_count", channel.json_body)
+
+        # set ratelimit
+        channel = self.make_request(
+            "POST",
+            self.url,
+            access_token=self.admin_user_tok,
+            content={"messages_per_second": 10, "burst_count": 11},
+        )
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(10, channel.json_body["messages_per_second"])
+        self.assertEqual(11, channel.json_body["burst_count"])
+
+        # update ratelimit
+        channel = self.make_request(
+            "POST",
+            self.url,
+            access_token=self.admin_user_tok,
+            content={"messages_per_second": 20, "burst_count": 21},
+        )
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(20, channel.json_body["messages_per_second"])
+        self.assertEqual(21, channel.json_body["burst_count"])
+
+        # request status
+        channel = self.make_request(
+            "GET",
+            self.url,
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual(20, channel.json_body["messages_per_second"])
+        self.assertEqual(21, channel.json_body["burst_count"])
+
+        # delete ratelimit
+        channel = self.make_request(
+            "DELETE",
+            self.url,
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertNotIn("messages_per_second", channel.json_body)
+        self.assertNotIn("burst_count", channel.json_body)
+
+        # request status
+        channel = self.make_request(
+            "GET",
+            self.url,
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertNotIn("messages_per_second", channel.json_body)
+        self.assertNotIn("burst_count", channel.json_body)
diff --git a/tests/rest/admin/test_username_available.py b/tests/rest/admin/test_username_available.py
new file mode 100644
index 0000000000..4e1c49c28b
--- /dev/null
+++ b/tests/rest/admin/test_username_available.py
@@ -0,0 +1,62 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import synapse.rest.admin
+from synapse.api.errors import Codes, SynapseError
+from synapse.rest.client import login
+
+from tests import unittest
+
+
+class UsernameAvailableTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        login.register_servlets,
+    ]
+    url = "/_synapse/admin/v1/username_available"
+
+    def prepare(self, reactor, clock, hs):
+        self.register_user("admin", "pass", admin=True)
+        self.admin_user_tok = self.login("admin", "pass")
+
+        async def check_username(username):
+            if username == "allowed":
+                return True
+            raise SynapseError(400, "User ID already taken.", errcode=Codes.USER_IN_USE)
+
+        handler = self.hs.get_registration_handler()
+        handler.check_username = check_username
+
+    def test_username_available(self):
+        """
+        The endpoint should return a 200 response if the username does not exist
+        """
+
+        url = "%s?username=%s" % (self.url, "allowed")
+        channel = self.make_request("GET", url, None, self.admin_user_tok)
+
+        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertTrue(channel.json_body["available"])
+
+    def test_username_unavailable(self):
+        """
+        The endpoint should return a 200 response if the username does not exist
+        """
+
+        url = "%s?username=%s" % (self.url, "disallowed")
+        channel = self.make_request("GET", url, None, self.admin_user_tok)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(channel.json_body["errcode"], "M_USER_IN_USE")
+        self.assertEqual(channel.json_body["error"], "User ID already taken.")
diff --git a/tests/rest/client/__init__.py b/tests/rest/client/__init__.py
index fe0ac3f8e9..629e2df74a 100644
--- a/tests/rest/client/__init__.py
+++ b/tests/rest/client/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/rest/client/test_consent.py b/tests/rest/client/test_consent.py
index c74693e9b2..65c58ce70a 100644
--- a/tests/rest/client/test_consent.py
+++ b/tests/rest/client/test_consent.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +16,7 @@
 
 import synapse.rest.admin
 from synapse.api.urls import ConsentURIBuilder
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.rest.consent import consent_resource
 
 from tests import unittest
diff --git a/tests/rest/client/test_ephemeral_message.py b/tests/rest/client/test_ephemeral_message.py
index 56937dcd2e..3d7aa8ec86 100644
--- a/tests/rest/client/test_ephemeral_message.py
+++ b/tests/rest/client/test_ephemeral_message.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +13,7 @@
 # limitations under the License.
 from synapse.api.constants import EventContentFields, EventTypes
 from synapse.rest import admin
-from synapse.rest.client.v1 import room
+from synapse.rest.client import room
 
 from tests import unittest
 
diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py
index 61bdae0879..013b2049e6 100644
--- a/tests/rest/client/test_identity.py
+++ b/tests/rest/client/test_identity.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
 import synapse.rest.admin
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import account
+from synapse.rest.client import account, login, room
 
 from tests import unittest
 
diff --git a/tests/rest/client/test_power_levels.py b/tests/rest/client/test_power_levels.py
index 5256c11fe6..91d0762cb0 100644
--- a/tests/rest/client/test_power_levels.py
+++ b/tests/rest/client/test_power_levels.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +13,7 @@
 # limitations under the License.
 
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import sync
+from synapse.rest.client import login, room, sync
 
 from tests.unittest import HomeserverTestCase
 
diff --git a/tests/rest/client/test_redactions.py b/tests/rest/client/test_redactions.py
index e0c74591b6..433d715f69 100644
--- a/tests/rest/client/test_redactions.py
+++ b/tests/rest/client/test_redactions.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +13,7 @@
 # limitations under the License.
 
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import sync
+from synapse.rest.client import login, room, sync
 
 from tests.unittest import HomeserverTestCase
 
diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py
index b8285f3240..5c3bef645f 100644
--- a/tests/rest/client/test_retention.py
+++ b/tests/rest/client/test_retention.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,11 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.api.constants import EventTypes
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.visibility import filter_events_for_client
 
 from tests import unittest
diff --git a/tests/rest/client/test_room_access_rules.py b/tests/rest/client/test_room_access_rules.py
index 64744b7176..3dea0a639a 100644
--- a/tests/rest/client/test_room_access_rules.py
+++ b/tests/rest/client/test_room_access_rules.py
@@ -15,15 +15,15 @@
 import json
 import random
 import string
-
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, JoinRules, Membership, RoomCreationPreset
 from synapse.api.errors import SynapseError
+from synapse.events.third_party_rules import load_legacy_third_party_event_rules
 from synapse.rest import admin
-from synapse.rest.client.v1 import directory, login, room
+from synapse.rest.client import directory, login, room
 from synapse.third_party_rules.access_rules import (
     ACCESS_RULES_TYPE,
     FROZEN_STATE_TYPE,
@@ -60,11 +60,13 @@ def make_homeserver(self, reactor, clock):
         def send_invite(destination, room_id, event_id, pdu):
             return defer.succeed(pdu)
 
-        def get_json(uri, args={}, headers=None):
+        def get_json(uri, args=None, headers=None):
+            if args is None:
+                args = {}
             address_domain = args["address"].split("@")[1]
             return defer.succeed({"hs": address_domain})
 
-        def post_json_get_json(uri, post_json, args={}, headers=None):
+        def post_json_get_json(uri, post_json, args=None, headers=None):
             token = "".join(random.choice(string.ascii_letters) for _ in range(10))
             return defer.succeed(
                 {
@@ -106,6 +108,8 @@ def post_json_get_json(uri, post_json, args={}, headers=None):
 
         self.third_party_event_rules = self.hs.get_third_party_event_rules()
 
+        load_legacy_third_party_event_rules(self.hs)
+
         return self.hs
 
     def prepare(self, reactor, clock, homeserver):
@@ -1022,7 +1026,7 @@ def create_room(
         preset=RoomCreationPreset.TRUSTED_PRIVATE_CHAT,
         initial_state=None,
         power_levels_content_override=None,
-        invite=[],
+        invite=None,
         expected_code=200,
     ):
         content = {"is_direct": direct, "preset": preset}
diff --git a/tests/rest/client/test_shadow_banned.py b/tests/rest/client/test_shadow_banned.py
index d2cce44032..6a0d9a82be 100644
--- a/tests/rest/client/test_shadow_banned.py
+++ b/tests/rest/client/test_shadow_banned.py
@@ -12,12 +12,17 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from mock import Mock, patch
+from unittest.mock import Mock, patch
 
 import synapse.rest.admin
 from synapse.api.constants import EventTypes
-from synapse.rest.client.v1 import directory, login, profile, room
-from synapse.rest.client.v2_alpha import room_upgrade_rest_servlet
+from synapse.rest.client import (
+    directory,
+    login,
+    profile,
+    room,
+    room_upgrade_rest_servlet,
+)
 from synapse.types import UserID
 
 from tests import unittest
diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py
index bf39014277..0ae4029640 100644
--- a/tests/rest/client/test_third_party_rules.py
+++ b/tests/rest/client/test_third_party_rules.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
@@ -14,21 +13,22 @@
 # limitations under the License.
 import threading
 from typing import Dict
-
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.events import EventBase
+from synapse.events.third_party_rules import load_legacy_third_party_event_rules
 from synapse.module_api import ModuleApi
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.types import Requester, StateMap
+from synapse.util.frozenutils import unfreeze
 
 from tests import unittest
 
 thread_local = threading.local()
 
 
-class ThirdPartyRulesTestModule:
+class LegacyThirdPartyRulesTestModule:
     def __init__(self, config: Dict, module_api: ModuleApi):
         # keep a record of the "current" rules module, so that the test can patch
         # it if desired.
@@ -48,8 +48,26 @@ def parse_config(config):
         return config
 
 
-def current_rules_module() -> ThirdPartyRulesTestModule:
-    return thread_local.rules_module
+class LegacyDenyNewRooms(LegacyThirdPartyRulesTestModule):
+    def __init__(self, config: Dict, module_api: ModuleApi):
+        super().__init__(config, module_api)
+
+    def on_create_room(
+        self, requester: Requester, config: dict, is_requester_admin: bool
+    ):
+        return False
+
+
+class LegacyChangeEvents(LegacyThirdPartyRulesTestModule):
+    def __init__(self, config: Dict, module_api: ModuleApi):
+        super().__init__(config, module_api)
+
+    async def check_event_allowed(self, event: EventBase, state: StateMap[EventBase]):
+        d = event.get_dict()
+        content = unfreeze(event.content)
+        content["foo"] = "bar"
+        d["content"] = content
+        return d
 
 
 class ThirdPartyRulesTestCase(unittest.HomeserverTestCase):
@@ -59,20 +77,23 @@ class ThirdPartyRulesTestCase(unittest.HomeserverTestCase):
         room.register_servlets,
     ]
 
-    def default_config(self):
-        config = super().default_config()
-        config["third_party_event_rules"] = {
-            "module": __name__ + ".ThirdPartyRulesTestModule",
-            "config": {},
-        }
-        return config
+    def make_homeserver(self, reactor, clock):
+        hs = self.setup_test_homeserver()
+
+        load_legacy_third_party_event_rules(hs)
+
+        return hs
 
     def prepare(self, reactor, clock, homeserver):
         # Create a user and room to play with during the tests
         self.user_id = self.register_user("kermit", "monkey")
         self.tok = self.login("kermit", "monkey")
 
-        self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok)
+        # Some tests might prevent room creation on purpose.
+        try:
+            self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok)
+        except Exception:
+            pass
 
     def test_third_party_rules(self):
         """Tests that a forbidden event is forbidden from being sent, but an allowed one
@@ -81,10 +102,12 @@ def test_third_party_rules(self):
         # patch the rules module with a Mock which will return False for some event
         # types
         async def check(ev, state):
-            return ev.type != "foo.bar.forbidden"
+            return ev.type != "foo.bar.forbidden", None
 
         callback = Mock(spec=[], side_effect=check)
-        current_rules_module().check_event_allowed = callback
+        self.hs.get_third_party_event_rules()._check_event_allowed_callbacks = [
+            callback
+        ]
 
         channel = self.make_request(
             "PUT",
@@ -118,9 +141,9 @@ def test_cannot_modify_event(self):
         # first patch the event checker so that it will try to modify the event
         async def check(ev: EventBase, state):
             ev.content = {"x": "y"}
-            return True
+            return True, None
 
-        current_rules_module().check_event_allowed = check
+        self.hs.get_third_party_event_rules()._check_event_allowed_callbacks = [check]
 
         # now send the event
         channel = self.make_request(
@@ -129,7 +152,19 @@ async def check(ev: EventBase, state):
             {"x": "x"},
             access_token=self.tok,
         )
-        self.assertEqual(channel.result["code"], b"500", channel.result)
+        # check_event_allowed has some error handling, so it shouldn't 500 just because a
+        # module did something bad.
+        self.assertEqual(channel.code, 200, channel.result)
+        event_id = channel.json_body["event_id"]
+
+        channel = self.make_request(
+            "GET",
+            "/_matrix/client/r0/rooms/%s/event/%s" % (self.room_id, event_id),
+            access_token=self.tok,
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+        ev = channel.json_body
+        self.assertEqual(ev["content"]["x"], "x")
 
     def test_modify_event(self):
         """The module can return a modified version of the event"""
@@ -137,9 +172,9 @@ def test_modify_event(self):
         async def check(ev: EventBase, state):
             d = ev.get_dict()
             d["content"] = {"x": "y"}
-            return d
+            return True, d
 
-        current_rules_module().check_event_allowed = check
+        self.hs.get_third_party_event_rules()._check_event_allowed_callbacks = [check]
 
         # now send the event
         channel = self.make_request(
@@ -170,9 +205,9 @@ async def check(ev: EventBase, state):
                 "msgtype": "m.text",
                 "body": d["content"]["body"].upper(),
             }
-            return d
+            return True, d
 
-        current_rules_module().check_event_allowed = check
+        self.hs.get_third_party_event_rules()._check_event_allowed_callbacks = [check]
 
         # Send an event, then edit it.
         channel = self.make_request(
@@ -224,7 +259,7 @@ async def check(ev: EventBase, state):
         self.assertEqual(ev["content"]["body"], "EDITED BODY")
 
     def test_send_event(self):
-        """Tests that the module can send an event into a room via the module api"""
+        """Tests that a module can send an event into a room via the module api"""
         content = {
             "msgtype": "m.text",
             "body": "Hello!",
@@ -235,13 +270,60 @@ def test_send_event(self):
             "content": content,
             "sender": self.user_id,
         }
-        event = self.get_success(
-            current_rules_module().module_api.create_and_send_event_into_room(
-                event_dict
-            )
-        )  # type: EventBase
+        event: EventBase = self.get_success(
+            self.hs.get_module_api().create_and_send_event_into_room(event_dict)
+        )
 
         self.assertEquals(event.sender, self.user_id)
         self.assertEquals(event.room_id, self.room_id)
         self.assertEquals(event.type, "m.room.message")
         self.assertEquals(event.content, content)
+
+    @unittest.override_config(
+        {
+            "third_party_event_rules": {
+                "module": __name__ + ".LegacyChangeEvents",
+                "config": {},
+            }
+        }
+    )
+    def test_legacy_check_event_allowed(self):
+        """Tests that the wrapper for legacy check_event_allowed callbacks works
+        correctly.
+        """
+        channel = self.make_request(
+            "PUT",
+            "/_matrix/client/r0/rooms/%s/send/m.room.message/1" % self.room_id,
+            {
+                "msgtype": "m.text",
+                "body": "Original body",
+            },
+            access_token=self.tok,
+        )
+        self.assertEqual(channel.result["code"], b"200", channel.result)
+
+        event_id = channel.json_body["event_id"]
+
+        channel = self.make_request(
+            "GET",
+            "/_matrix/client/r0/rooms/%s/event/%s" % (self.room_id, event_id),
+            access_token=self.tok,
+        )
+        self.assertEqual(channel.result["code"], b"200", channel.result)
+
+        self.assertIn("foo", channel.json_body["content"].keys())
+        self.assertEqual(channel.json_body["content"]["foo"], "bar")
+
+    @unittest.override_config(
+        {
+            "third_party_event_rules": {
+                "module": __name__ + ".LegacyDenyNewRooms",
+                "config": {},
+            }
+        }
+    )
+    def test_legacy_on_create_room(self):
+        """Tests that the wrapper for legacy on_create_room callbacks works
+        correctly.
+        """
+        self.helper.create_room_as(self.user_id, tok=self.tok, expect_code=403)
diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py
index 171632e195..3b5747cb12 100644
--- a/tests/rest/client/test_transactions.py
+++ b/tests/rest/client/test_transactions.py
@@ -1,4 +1,4 @@
-from mock import Mock, call
+from unittest.mock import Mock, call
 
 from twisted.internet import defer, reactor
 
diff --git a/tests/rest/client/v1/__init__.py b/tests/rest/client/v1/__init__.py
index bfebb0f644..5e83dba2ed 100644
--- a/tests/rest/client/v1/__init__.py
+++ b/tests/rest/client/v1/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/rest/client/v1/test_directory.py b/tests/rest/client/v1/test_directory.py
index edd1d184f8..d2181ea907 100644
--- a/tests/rest/client/v1/test_directory.py
+++ b/tests/rest/client/v1/test_directory.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,7 +15,7 @@
 import json
 
 from synapse.rest import admin
-from synapse.rest.client.v1 import directory, login, room
+from synapse.rest.client import directory, login, room
 from synapse.types import RoomAlias
 from synapse.util.stringutils import random_string
 
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index 2ae896db1e..a90294003e 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,16 +14,16 @@
 
 """ Tests REST events for /events paths."""
 
-from mock import Mock
+from unittest.mock import Mock
 
 import synapse.rest.admin
-from synapse.rest.client.v1 import events, login, room
+from synapse.rest.client import events, login, room
 
 from tests import unittest
 
 
 class EventStreamPermissionsTestCase(unittest.HomeserverTestCase):
-    """ Tests event streaming (GET /events). """
+    """Tests event streaming (GET /events)."""
 
     servlets = [
         events.register_servlets,
diff --git a/tests/rest/client/v1/test_login.py b/tests/rest/client/v1/test_login.py
index 988821b16f..eba3552b19 100644
--- a/tests/rest/client/v1/test_login.py
+++ b/tests/rest/client/v1/test_login.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,19 +15,17 @@
 import time
 import urllib.parse
 from typing import Any, Dict, List, Optional, Union
+from unittest.mock import Mock
 from urllib.parse import urlencode
 
-from mock import Mock
-
 import pymacaroons
 
 from twisted.web.resource import Resource
 
 import synapse.rest.admin
 from synapse.appservice import ApplicationService
-from synapse.rest.client.v1 import login, logout
-from synapse.rest.client.v2_alpha import devices, register
-from synapse.rest.client.v2_alpha.account import WhoamiRestServlet
+from synapse.rest.client import devices, login, logout, register
+from synapse.rest.client.account import WhoamiRestServlet
 from synapse.rest.synapse.client import build_synapse_client_resource_tree
 from synapse.types import create_requester
 
@@ -455,7 +452,7 @@ def test_get_msc2858_login_flows(self):
         self.assertEqual(channel.code, 200, channel.result)
 
         # stick the flows results in a dict by type
-        flow_results = {}  # type: Dict[str, Any]
+        flow_results: Dict[str, Any] = {}
         for f in channel.json_body["flows"]:
             flow_type = f["type"]
             self.assertNotIn(
@@ -503,7 +500,7 @@ def test_multi_sso_redirect(self):
         p.close()
 
         # there should be a link for each href
-        returned_idps = []  # type: List[str]
+        returned_idps: List[str] = []
         for link in p.links:
             path, query = link.split("?", 1)
             self.assertEqual(path, "pick_idp")
@@ -584,7 +581,7 @@ def test_login_via_oidc(self):
         # ... and should have set a cookie including the redirect url
         cookie_headers = channel.headers.getRawHeaders("Set-Cookie")
         assert cookie_headers
-        cookies = {}  # type: Dict[str, str]
+        cookies: Dict[str, str] = {}
         for h in cookie_headers:
             key, value = h.split(";")[0].split("=", maxsplit=1)
             cookies[key] = value
@@ -876,9 +873,7 @@ def make_homeserver(self, reactor, clock):
 
     def jwt_encode(self, payload: Dict[str, Any], secret: str = jwt_secret) -> str:
         # PyJWT 2.0.0 changed the return type of jwt.encode from bytes to str.
-        result = jwt.encode(
-            payload, secret, self.jwt_algorithm
-        )  # type: Union[str, bytes]
+        result: Union[str, bytes] = jwt.encode(payload, secret, self.jwt_algorithm)
         if isinstance(result, bytes):
             return result.decode("ascii")
         return result
@@ -1086,7 +1081,7 @@ def make_homeserver(self, reactor, clock):
 
     def jwt_encode(self, payload: Dict[str, Any], secret: str = jwt_privatekey) -> str:
         # PyJWT 2.0.0 changed the return type of jwt.encode from bytes to str.
-        result = jwt.encode(payload, secret, "RS256")  # type: Union[bytes,str]
+        result: Union[bytes, str] = jwt.encode(payload, secret, "RS256")
         if isinstance(result, bytes):
             return result.decode("ascii")
         return result
@@ -1274,7 +1269,7 @@ def test_username_picker(self):
         self.assertEqual(picker_url, "/_synapse/client/pick_username/account_details")
 
         # ... with a username_mapping_session cookie
-        cookies = {}  # type: Dict[str,str]
+        cookies: Dict[str, str] = {}
         channel.extract_cookies(cookies)
         self.assertIn("username_mapping_session", cookies)
         session_id = cookies["username_mapping_session"]
diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py
index 94a5154834..1d152352d1 100644
--- a/tests/rest/client/v1/test_presence.py
+++ b/tests/rest/client/v1/test_presence.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,18 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
-from synapse.rest.client.v1 import presence
+from synapse.handlers.presence import PresenceHandler
+from synapse.rest.client import presence
 from synapse.types import UserID
 
 from tests import unittest
 
 
 class PresenceTestCase(unittest.HomeserverTestCase):
-    """ Tests presence REST API. """
+    """Tests presence REST API."""
 
     user_id = "@sid:red"
 
@@ -33,7 +33,7 @@ class PresenceTestCase(unittest.HomeserverTestCase):
 
     def make_homeserver(self, reactor, clock):
 
-        presence_handler = Mock()
+        presence_handler = Mock(spec=PresenceHandler)
         presence_handler.set_state.return_value = defer.succeed(None)
 
         hs = self.setup_test_homeserver(
@@ -60,12 +60,12 @@ def test_put_presence(self):
         self.assertEqual(channel.code, 200)
         self.assertEqual(self.hs.get_presence_handler().set_state.call_count, 1)
 
+    @unittest.override_config({"use_presence": False})
     def test_put_presence_disabled(self):
         """
         PUT to the status endpoint with use_presence disabled will NOT call
         set_state on the presence handler.
         """
-        self.hs.config.use_presence = False
 
         body = {"presence": "here", "status_msg": "beep boop"}
         channel = self.make_request(
diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py
index f3448c94dd..2860579c2e 100644
--- a/tests/rest/client/v1/test_profile.py
+++ b/tests/rest/client/v1/test_profile.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +14,7 @@
 
 """Tests REST events for /profile paths."""
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, profile, room
+from synapse.rest.client import login, profile, room
 
 from tests import unittest
 
diff --git a/tests/rest/client/v1/test_push_rule_attrs.py b/tests/rest/client/v1/test_push_rule_attrs.py
index 2bc512d75e..d0ce91ccd9 100644
--- a/tests/rest/client/v1/test_push_rule_attrs.py
+++ b/tests/rest/client/v1/test_push_rule_attrs.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +13,7 @@
 # limitations under the License.
 import synapse
 from synapse.api.errors import Codes
-from synapse.rest.client.v1 import login, push_rule, room
+from synapse.rest.client import login, push_rule, room
 
 from tests.unittest import HomeserverTestCase
 
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index ed65f645fc..50100a5ae4 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018-2019 New Vector Ltd
@@ -19,16 +18,18 @@
 """Tests REST events for /rooms paths."""
 
 import json
+from typing import Iterable
+from unittest.mock import Mock, call
 from urllib import parse as urlparse
 
-from mock import Mock
+from twisted.internet import defer
 
 import synapse.rest.admin
 from synapse.api.constants import EventContentFields, EventTypes, Membership
+from synapse.api.errors import HttpResponseException
 from synapse.handlers.pagination import PurgeStatus
 from synapse.rest import admin
-from synapse.rest.client.v1 import directory, login, profile, room
-from synapse.rest.client.v2_alpha import account
+from synapse.rest.client import account, directory, login, profile, room, sync
 from synapse.types import JsonDict, RoomAlias, UserID, create_requester
 from synapse.util.stringutils import random_string
 
@@ -65,7 +66,7 @@ async def _insert_client_ip(*args, **kwargs):
 
 
 class RoomPermissionsTestCase(RoomBase):
-    """ Tests room permissions. """
+    """Tests room permissions."""
 
     user_id = "@sid1:red"
     rmcreator_id = "@notme:red"
@@ -207,7 +208,9 @@ def test_topic_perms(self):
         )
         self.assertEquals(403, channel.code, msg=channel.result["body"])
 
-    def _test_get_membership(self, room=None, members=[], expect_code=None):
+    def _test_get_membership(
+        self, room=None, members: Iterable = frozenset(), expect_code=None
+    ):
         for member in members:
             path = "/rooms/%s/state/m.room.member/%s" % (room, member)
             channel = self.make_request("GET", path)
@@ -376,7 +379,9 @@ def test_leave_permissions(self):
 
 
 class RoomsMemberListTestCase(RoomBase):
-    """ Tests /rooms/$room_id/members/list REST events."""
+    """Tests /rooms/$room_id/members/list REST events."""
+
+    servlets = RoomBase.servlets + [sync.register_servlets]
 
     user_id = "@sid1:red"
 
@@ -394,6 +399,86 @@ def test_get_member_list_no_permission(self):
         channel = self.make_request("GET", "/rooms/%s/members" % room_id)
         self.assertEquals(403, channel.code, msg=channel.result["body"])
 
+    def test_get_member_list_no_permission_with_at_token(self):
+        """
+        Tests that a stranger to the room cannot get the member list
+        (in the case that they use an at token).
+        """
+        room_id = self.helper.create_room_as("@someone.else:red")
+
+        # first sync to get an at token
+        channel = self.make_request("GET", "/sync")
+        self.assertEquals(200, channel.code)
+        sync_token = channel.json_body["next_batch"]
+
+        # check that permission is denied for @sid1:red to get the
+        # memberships of @someone.else:red's room.
+        channel = self.make_request(
+            "GET",
+            f"/rooms/{room_id}/members?at={sync_token}",
+        )
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
+
+    def test_get_member_list_no_permission_former_member(self):
+        """
+        Tests that a former member of the room can not get the member list.
+        """
+        # create a room, invite the user and the user joins
+        room_id = self.helper.create_room_as("@alice:red")
+        self.helper.invite(room_id, "@alice:red", self.user_id)
+        self.helper.join(room_id, self.user_id)
+
+        # check that the user can see the member list to start with
+        channel = self.make_request("GET", "/rooms/%s/members" % room_id)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
+
+        # ban the user
+        self.helper.change_membership(room_id, "@alice:red", self.user_id, "ban")
+
+        # check the user can no longer see the member list
+        channel = self.make_request("GET", "/rooms/%s/members" % room_id)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
+
+    def test_get_member_list_no_permission_former_member_with_at_token(self):
+        """
+        Tests that a former member of the room can not get the member list
+        (in the case that they use an at token).
+        """
+        # create a room, invite the user and the user joins
+        room_id = self.helper.create_room_as("@alice:red")
+        self.helper.invite(room_id, "@alice:red", self.user_id)
+        self.helper.join(room_id, self.user_id)
+
+        # sync to get an at token
+        channel = self.make_request("GET", "/sync")
+        self.assertEquals(200, channel.code)
+        sync_token = channel.json_body["next_batch"]
+
+        # check that the user can see the member list to start with
+        channel = self.make_request(
+            "GET", "/rooms/%s/members?at=%s" % (room_id, sync_token)
+        )
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
+
+        # ban the user (Note: the user is actually allowed to see this event and
+        # state so that they know they're banned!)
+        self.helper.change_membership(room_id, "@alice:red", self.user_id, "ban")
+
+        # invite a third user and let them join
+        self.helper.invite(room_id, "@alice:red", "@bob:red")
+        self.helper.join(room_id, "@bob:red")
+
+        # now, with the original user, sync again to get a new at token
+        channel = self.make_request("GET", "/sync")
+        self.assertEquals(200, channel.code)
+        sync_token = channel.json_body["next_batch"]
+
+        # check the user can no longer see the updated member list
+        channel = self.make_request(
+            "GET", "/rooms/%s/members?at=%s" % (room_id, sync_token)
+        )
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
+
     def test_get_member_list_mixed_memberships(self):
         room_creator = "@some_other_guy:red"
         room_id = self.helper.create_room_as(room_creator)
@@ -415,7 +500,7 @@ def test_get_member_list_mixed_memberships(self):
 
 
 class RoomsCreateTestCase(RoomBase):
-    """ Tests /rooms and /rooms/$room_id REST events. """
+    """Tests /rooms and /rooms/$room_id REST events."""
 
     user_id = "@sid1:red"
 
@@ -462,9 +547,46 @@ def test_post_room_invitees_invalid_mxid(self):
         )
         self.assertEquals(400, channel.code)
 
+    @unittest.override_config({"rc_invites": {"per_room": {"burst_count": 3}}})
+    def test_post_room_invitees_ratelimit(self):
+        """Test that invites sent when creating a room are ratelimited by a RateLimiter,
+        which ratelimits them correctly, including by not limiting when the requester is
+        exempt from ratelimiting.
+        """
+
+        # Build the request's content. We use local MXIDs because invites over federation
+        # are more difficult to mock.
+        content = json.dumps(
+            {
+                "invite": [
+                    "@alice1:red",
+                    "@alice2:red",
+                    "@alice3:red",
+                    "@alice4:red",
+                ]
+            }
+        ).encode("utf8")
+
+        # Test that the invites are correctly ratelimited.
+        channel = self.make_request("POST", "/createRoom", content)
+        self.assertEqual(400, channel.code)
+        self.assertEqual(
+            "Cannot invite so many users at once",
+            channel.json_body["error"],
+        )
+
+        # Add the current user to the ratelimit overrides, allowing them no ratelimiting.
+        self.get_success(
+            self.hs.get_datastore().set_ratelimit_for_user(self.user_id, 0, 0)
+        )
+
+        # Test that the invites aren't ratelimited anymore.
+        channel = self.make_request("POST", "/createRoom", content)
+        self.assertEqual(200, channel.code)
+
 
 class RoomTopicTestCase(RoomBase):
-    """ Tests /rooms/$room_id/topic REST events. """
+    """Tests /rooms/$room_id/topic REST events."""
 
     user_id = "@sid1:red"
 
@@ -528,7 +650,7 @@ def test_rooms_topic_with_extra_keys(self):
 
 
 class RoomMemberStateTestCase(RoomBase):
-    """ Tests /rooms/$room_id/members/$user_id/state REST events. """
+    """Tests /rooms/$room_id/members/$user_id/state REST events."""
 
     user_id = "@sid1:red"
 
@@ -645,7 +767,7 @@ def test_invites_by_rooms_ratelimit(self):
     def test_invites_by_users_ratelimit(self):
         """Tests that invites to a specific user are actually rate-limited."""
 
-        for i in range(3):
+        for _ in range(3):
             room_id = self.helper.create_room_as(self.user_id)
             self.helper.invite(room_id, self.user_id, "@other-users:red")
 
@@ -667,7 +789,7 @@ class RoomJoinRatelimitTestCase(RoomBase):
     )
     def test_join_local_ratelimit(self):
         """Tests that local joins are actually rate-limited."""
-        for i in range(3):
+        for _ in range(3):
             self.helper.create_room_as(self.user_id)
 
         self.helper.create_room_as(self.user_id, expect_code=429)
@@ -732,7 +854,7 @@ def test_join_local_ratelimit_idempotent(self):
         for path in paths_to_test:
             # Make sure we send more requests than the rate-limiting config would allow
             # if all of these requests ended up joining the user to a room.
-            for i in range(4):
+            for _ in range(4):
                 channel = self.make_request("POST", path % room_id, {})
                 self.assertEquals(channel.code, 200)
 
@@ -752,7 +874,7 @@ def test_autojoin_rooms(self):
 
 
 class RoomMessagesTestCase(RoomBase):
-    """ Tests /rooms/$room_id/messages/$user_id/$msg_id REST events. """
+    """Tests /rooms/$room_id/messages/$user_id/$msg_id REST events."""
 
     user_id = "@sid1:red"
 
@@ -800,7 +922,7 @@ def test_rooms_messages_sent(self):
 
 
 class RoomInitialSyncTestCase(RoomBase):
-    """ Tests /rooms/$room_id/initialSync. """
+    """Tests /rooms/$room_id/initialSync."""
 
     user_id = "@sid1:red"
 
@@ -841,7 +963,7 @@ def test_initial_sync(self):
 
 
 class RoomMessageListTestCase(RoomBase):
-    """ Tests /rooms/$room_id/messages REST events. """
+    """Tests /rooms/$room_id/messages REST events."""
 
     user_id = "@sid1:red"
 
@@ -1086,6 +1208,93 @@ def test_restricted_auth(self):
         self.assertEqual(channel.code, 200, channel.result)
 
 
+class PublicRoomsTestRemoteSearchFallbackTestCase(unittest.HomeserverTestCase):
+    """Test that we correctly fallback to local filtering if a remote server
+    doesn't support search.
+    """
+
+    servlets = [
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def make_homeserver(self, reactor, clock):
+        return self.setup_test_homeserver(federation_client=Mock())
+
+    def prepare(self, reactor, clock, hs):
+        self.register_user("user", "pass")
+        self.token = self.login("user", "pass")
+
+        self.federation_client = hs.get_federation_client()
+
+    def test_simple(self):
+        "Simple test for searching rooms over federation"
+        self.federation_client.get_public_rooms.side_effect = (
+            lambda *a, **k: defer.succeed({})
+        )
+
+        search_filter = {"generic_search_term": "foobar"}
+
+        channel = self.make_request(
+            "POST",
+            b"/_matrix/client/r0/publicRooms?server=testserv",
+            content={"filter": search_filter},
+            access_token=self.token,
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+
+        self.federation_client.get_public_rooms.assert_called_once_with(
+            "testserv",
+            limit=100,
+            since_token=None,
+            search_filter=search_filter,
+            include_all_networks=False,
+            third_party_instance_id=None,
+        )
+
+    def test_fallback(self):
+        "Test that searching public rooms over federation falls back if it gets a 404"
+
+        # The `get_public_rooms` should be called again if the first call fails
+        # with a 404, when using search filters.
+        self.federation_client.get_public_rooms.side_effect = (
+            HttpResponseException(404, "Not Found", b""),
+            defer.succeed({}),
+        )
+
+        search_filter = {"generic_search_term": "foobar"}
+
+        channel = self.make_request(
+            "POST",
+            b"/_matrix/client/r0/publicRooms?server=testserv",
+            content={"filter": search_filter},
+            access_token=self.token,
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+
+        self.federation_client.get_public_rooms.assert_has_calls(
+            [
+                call(
+                    "testserv",
+                    limit=100,
+                    since_token=None,
+                    search_filter=search_filter,
+                    include_all_networks=False,
+                    third_party_instance_id=None,
+                ),
+                call(
+                    "testserv",
+                    limit=None,
+                    since_token=None,
+                    search_filter=None,
+                    include_all_networks=False,
+                    third_party_instance_id=None,
+                ),
+            ]
+        )
+
+
 class PerRoomProfilesForbiddenTestCase(unittest.HomeserverTestCase):
 
     servlets = [
@@ -1168,7 +1377,7 @@ def test_join_reason(self):
         reason = "hello"
         channel = self.make_request(
             "POST",
-            "/_matrix/client/r0/rooms/{}/join".format(self.room_id),
+            f"/_matrix/client/r0/rooms/{self.room_id}/join",
             content={"reason": reason},
             access_token=self.second_tok,
         )
@@ -1182,7 +1391,7 @@ def test_leave_reason(self):
         reason = "hello"
         channel = self.make_request(
             "POST",
-            "/_matrix/client/r0/rooms/{}/leave".format(self.room_id),
+            f"/_matrix/client/r0/rooms/{self.room_id}/leave",
             content={"reason": reason},
             access_token=self.second_tok,
         )
@@ -1196,7 +1405,7 @@ def test_kick_reason(self):
         reason = "hello"
         channel = self.make_request(
             "POST",
-            "/_matrix/client/r0/rooms/{}/kick".format(self.room_id),
+            f"/_matrix/client/r0/rooms/{self.room_id}/kick",
             content={"reason": reason, "user_id": self.second_user_id},
             access_token=self.second_tok,
         )
@@ -1210,7 +1419,7 @@ def test_ban_reason(self):
         reason = "hello"
         channel = self.make_request(
             "POST",
-            "/_matrix/client/r0/rooms/{}/ban".format(self.room_id),
+            f"/_matrix/client/r0/rooms/{self.room_id}/ban",
             content={"reason": reason, "user_id": self.second_user_id},
             access_token=self.creator_tok,
         )
@@ -1222,7 +1431,7 @@ def test_unban_reason(self):
         reason = "hello"
         channel = self.make_request(
             "POST",
-            "/_matrix/client/r0/rooms/{}/unban".format(self.room_id),
+            f"/_matrix/client/r0/rooms/{self.room_id}/unban",
             content={"reason": reason, "user_id": self.second_user_id},
             access_token=self.creator_tok,
         )
@@ -1234,7 +1443,7 @@ def test_invite_reason(self):
         reason = "hello"
         channel = self.make_request(
             "POST",
-            "/_matrix/client/r0/rooms/{}/invite".format(self.room_id),
+            f"/_matrix/client/r0/rooms/{self.room_id}/invite",
             content={"reason": reason, "user_id": self.second_user_id},
             access_token=self.creator_tok,
         )
@@ -1253,7 +1462,7 @@ def test_reject_invite_reason(self):
         reason = "hello"
         channel = self.make_request(
             "POST",
-            "/_matrix/client/r0/rooms/{}/leave".format(self.room_id),
+            f"/_matrix/client/r0/rooms/{self.room_id}/leave",
             content={"reason": reason},
             access_token=self.second_tok,
         )
@@ -1842,8 +2051,7 @@ def _get_aliases(self, access_token: str, expected_code: int = 200) -> JsonDict:
         """Calls the endpoint under test. returns the json response object."""
         channel = self.make_request(
             "GET",
-            "/_matrix/client/unstable/org.matrix.msc2432/rooms/%s/aliases"
-            % (self.room_id,),
+            "/_matrix/client/r0/rooms/%s/aliases" % (self.room_id,),
             access_token=access_token,
         )
         self.assertEqual(channel.code, expected_code, channel.result)
diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py
index 329dbd06de..b54b004733 100644
--- a/tests/rest/client/v1/test_typing.py
+++ b/tests/rest/client/v1/test_typing.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector
 #
@@ -16,9 +15,9 @@
 
 """Tests REST events for /rooms paths."""
 
-from mock import Mock
+from unittest.mock import Mock
 
-from synapse.rest.client.v1 import room
+from synapse.rest.client import room
 from synapse.types import UserID
 
 from tests import unittest
@@ -27,7 +26,7 @@
 
 
 class RoomTypingTestCase(unittest.HomeserverTestCase):
-    """ Tests /rooms/$room_id/typing/$user_id REST API. """
+    """Tests /rooms/$room_id/typing/$user_id REST API."""
 
     user_id = "@sid:red"
 
diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py
index 946740aa5d..954ad1a1fd 100644
--- a/tests/rest/client/v1/utils.py
+++ b/tests/rest/client/v1/utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018-2019 New Vector Ltd
@@ -20,9 +19,8 @@
 import re
 import time
 import urllib.parse
-from typing import Any, Dict, Mapping, MutableMapping, Optional
-
-from mock import patch
+from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Tuple, Union
+from unittest.mock import patch
 
 import attr
 
@@ -49,11 +47,15 @@ class RestHelper:
 
     def create_room_as(
         self,
-        room_creator: str = None,
+        room_creator: Optional[str] = None,
         is_public: bool = True,
-        room_version: str = None,
-        tok: str = None,
+        room_version: Optional[str] = None,
+        tok: Optional[str] = None,
         expect_code: int = 200,
+        extra_content: Optional[Dict] = None,
+        custom_headers: Optional[
+            Iterable[Tuple[Union[bytes, str], Union[bytes, str]]]
+        ] = None,
     ) -> str:
         """
         Create a room.
@@ -74,7 +76,7 @@ def create_room_as(
         temp_id = self.auth_user_id
         self.auth_user_id = room_creator
         path = "/_matrix/client/r0/createRoom"
-        content = {}
+        content = extra_content or {}
         if not is_public:
             content["visibility"] = "private"
         if room_version:
@@ -88,6 +90,7 @@ def create_room_as(
             "POST",
             path,
             json.dumps(content).encode("utf8"),
+            custom_headers=custom_headers,
         )
 
         assert channel.result["code"] == b"%d" % expect_code, channel.result
@@ -132,7 +135,7 @@ def change_membership(
         src: str,
         targ: str,
         membership: str,
-        extra_data: dict = {},
+        extra_data: Optional[dict] = None,
         tok: Optional[str] = None,
         expect_code: int = 200,
     ) -> None:
@@ -156,7 +159,7 @@ def change_membership(
             path = path + "?access_token=%s" % tok
 
         data = {"membership": membership}
-        data.update(extra_data)
+        data.update(extra_data or {})
 
         channel = make_request(
             self.hs.get_reactor(),
@@ -176,18 +179,43 @@ def change_membership(
 
         self.auth_user_id = temp_id
 
-    def send(self, room_id, body=None, txn_id=None, tok=None, expect_code=200):
+    def send(
+        self,
+        room_id,
+        body=None,
+        txn_id=None,
+        tok=None,
+        expect_code=200,
+        custom_headers: Optional[
+            Iterable[Tuple[Union[bytes, str], Union[bytes, str]]]
+        ] = None,
+    ):
         if body is None:
             body = "body_text_here"
 
         content = {"msgtype": "m.text", "body": body}
 
         return self.send_event(
-            room_id, "m.room.message", content, txn_id, tok, expect_code
+            room_id,
+            "m.room.message",
+            content,
+            txn_id,
+            tok,
+            expect_code,
+            custom_headers=custom_headers,
         )
 
     def send_event(
-        self, room_id, type, content={}, txn_id=None, tok=None, expect_code=200
+        self,
+        room_id,
+        type,
+        content: Optional[dict] = None,
+        txn_id=None,
+        tok=None,
+        expect_code=200,
+        custom_headers: Optional[
+            Iterable[Tuple[Union[bytes, str], Union[bytes, str]]]
+        ] = None,
     ):
         if txn_id is None:
             txn_id = "m%s" % (str(time.time()))
@@ -201,7 +229,8 @@ def send_event(
             self.site,
             "PUT",
             path,
-            json.dumps(content).encode("utf8"),
+            json.dumps(content or {}).encode("utf8"),
+            custom_headers=custom_headers,
         )
 
         assert (
diff --git a/tests/rest/client/v2_alpha/test_account.py b/tests/rest/client/v2_alpha/test_account.py
index e72b61963d..b946fca8b3 100644
--- a/tests/rest/client/v2_alpha/test_account.py
+++ b/tests/rest/client/v2_alpha/test_account.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -25,8 +24,8 @@
 import synapse.rest.admin
 from synapse.api.constants import LoginType, Membership
 from synapse.api.errors import Codes, HttpResponseException
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import account, register
+from synapse.appservice import ApplicationService
+from synapse.rest.client import account, login, register, room
 from synapse.rest.synapse.client.password_reset import PasswordResetSubmitTokenResource
 
 from tests import unittest
@@ -47,12 +46,6 @@ def make_homeserver(self, reactor, clock):
         config = self.default_config()
 
         # Email config.
-        self.email_attempts = []
-
-        async def sendmail(smtphost, from_addr, to_addrs, msg, **kwargs):
-            self.email_attempts.append(msg)
-            return
-
         config["email"] = {
             "enable_notifs": False,
             "template_dir": os.path.abspath(
@@ -67,7 +60,16 @@ async def sendmail(smtphost, from_addr, to_addrs, msg, **kwargs):
         }
         config["public_baseurl"] = "https://example.com"
 
-        hs = self.setup_test_homeserver(config=config, sendmail=sendmail)
+        hs = self.setup_test_homeserver(config=config)
+
+        async def sendmail(
+            reactor, smtphost, smtpport, from_addr, to_addrs, msg, **kwargs
+        ):
+            self.email_attempts.append(msg)
+
+        self.email_attempts = []
+        hs.get_send_email_handler()._sendmail = sendmail
+
         return hs
 
     def prepare(self, reactor, clock, hs):
@@ -398,7 +400,7 @@ def test_deactivate_account(self):
         self.assertTrue(self.get_success(store.get_user_deactivated_status(user_id)))
 
         # Check that this access token has been invalidated.
-        channel = self.make_request("GET", "account/whoami")
+        channel = self.make_request("GET", "account/whoami", access_token=tok)
         self.assertEqual(channel.code, 401)
 
     def test_pending_invites(self):
@@ -459,6 +461,46 @@ def deactivate(self, user_id, tok):
         self.assertEqual(channel.code, 200)
 
 
+class WhoamiTestCase(unittest.HomeserverTestCase):
+
+    servlets = [
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        login.register_servlets,
+        account.register_servlets,
+        register.register_servlets,
+    ]
+
+    def test_GET_whoami(self):
+        device_id = "wouldgohere"
+        user_id = self.register_user("kermit", "test")
+        tok = self.login("kermit", "test", device_id=device_id)
+
+        whoami = self.whoami(tok)
+        self.assertEqual(whoami, {"user_id": user_id, "device_id": device_id})
+
+    def test_GET_whoami_appservices(self):
+        user_id = "@as:test"
+        as_token = "i_am_an_app_service"
+
+        appservice = ApplicationService(
+            as_token,
+            self.hs.config.server_name,
+            id="1234",
+            namespaces={"users": [{"regex": user_id, "exclusive": True}]},
+            sender=user_id,
+        )
+        self.hs.get_datastore().services_cache.append(appservice)
+
+        whoami = self.whoami(as_token)
+        self.assertEqual(whoami, {"user_id": user_id})
+        self.assertFalse(hasattr(whoami, "device_id"))
+
+    def whoami(self, tok):
+        channel = self.make_request("GET", "account/whoami", {}, access_token=tok)
+        self.assertEqual(channel.code, 200)
+        return channel.json_body
+
+
 class ThreepidEmailRestTestCase(unittest.HomeserverTestCase):
 
     servlets = [
@@ -471,11 +513,6 @@ def make_homeserver(self, reactor, clock):
         config = self.default_config()
 
         # Email config.
-        self.email_attempts = []
-
-        async def sendmail(smtphost, from_addr, to_addrs, msg, **kwargs):
-            self.email_attempts.append(msg)
-
         config["email"] = {
             "enable_notifs": False,
             "template_dir": os.path.abspath(
@@ -490,7 +527,16 @@ async def sendmail(smtphost, from_addr, to_addrs, msg, **kwargs):
         }
         config["public_baseurl"] = "https://example.com"
 
-        self.hs = self.setup_test_homeserver(config=config, sendmail=sendmail)
+        self.hs = self.setup_test_homeserver(config=config)
+
+        async def sendmail(
+            reactor, smtphost, smtpport, from_addr, to_addrs, msg, **kwargs
+        ):
+            self.email_attempts.append(msg)
+
+        self.email_attempts = []
+        self.hs.get_send_email_handler()._sendmail = sendmail
+
         return self.hs
 
     def prepare(self, reactor, clock, hs):
diff --git a/tests/rest/client/v2_alpha/test_auth.py b/tests/rest/client/v2_alpha/test_auth.py
index 9734a2159a..cf5cfb910c 100644
--- a/tests/rest/client/v2_alpha/test_auth.py
+++ b/tests/rest/client/v2_alpha/test_auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector
 # Copyright 2020-2021 The Matrix.org Foundation C.I.C
 #
@@ -13,15 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Union
+from typing import Optional, Union
 
 from twisted.internet.defer import succeed
 
 import synapse.rest.admin
 from synapse.api.constants import LoginType
 from synapse.handlers.ui_auth.checkers import UserInteractiveAuthChecker
-from synapse.rest.client.v1 import login
-from synapse.rest.client.v2_alpha import auth, devices, register
+from synapse.rest.client import account, auth, devices, login, register
 from synapse.rest.synapse.client import build_synapse_client_resource_tree
 from synapse.types import JsonDict, UserID
 
@@ -74,7 +72,10 @@ def register(self, expected_response: int, body: JsonDict) -> FakeChannel:
         return channel
 
     def recaptcha(
-        self, session: str, expected_post_response: int, post_session: str = None
+        self,
+        session: str,
+        expected_post_response: int,
+        post_session: Optional[str] = None,
     ) -> None:
         """Get and respond to a fallback recaptcha. Returns the second request."""
         if post_session is None:
@@ -496,3 +497,221 @@ def test_ui_auth_fails_for_incorrect_sso_user(self):
         self.delete_device(
             self.user_tok, self.device_id, 403, body={"auth": {"session": session_id}}
         )
+
+
+class RefreshAuthTests(unittest.HomeserverTestCase):
+    servlets = [
+        auth.register_servlets,
+        account.register_servlets,
+        login.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        register.register_servlets,
+    ]
+    hijack_auth = False
+
+    def prepare(self, reactor, clock, hs):
+        self.user_pass = "pass"
+        self.user = self.register_user("test", self.user_pass)
+
+    def test_login_issue_refresh_token(self):
+        """
+        A login response should include a refresh_token only if asked.
+        """
+        # Test login
+        body = {"type": "m.login.password", "user": "test", "password": self.user_pass}
+
+        login_without_refresh = self.make_request(
+            "POST", "/_matrix/client/r0/login", body
+        )
+        self.assertEqual(login_without_refresh.code, 200, login_without_refresh.result)
+        self.assertNotIn("refresh_token", login_without_refresh.json_body)
+
+        login_with_refresh = self.make_request(
+            "POST",
+            "/_matrix/client/r0/login?org.matrix.msc2918.refresh_token=true",
+            body,
+        )
+        self.assertEqual(login_with_refresh.code, 200, login_with_refresh.result)
+        self.assertIn("refresh_token", login_with_refresh.json_body)
+        self.assertIn("expires_in_ms", login_with_refresh.json_body)
+
+    def test_register_issue_refresh_token(self):
+        """
+        A register response should include a refresh_token only if asked.
+        """
+        register_without_refresh = self.make_request(
+            "POST",
+            "/_matrix/client/r0/register",
+            {
+                "username": "test2",
+                "password": self.user_pass,
+                "auth": {"type": LoginType.DUMMY},
+            },
+        )
+        self.assertEqual(
+            register_without_refresh.code, 200, register_without_refresh.result
+        )
+        self.assertNotIn("refresh_token", register_without_refresh.json_body)
+
+        register_with_refresh = self.make_request(
+            "POST",
+            "/_matrix/client/r0/register?org.matrix.msc2918.refresh_token=true",
+            {
+                "username": "test3",
+                "password": self.user_pass,
+                "auth": {"type": LoginType.DUMMY},
+            },
+        )
+        self.assertEqual(register_with_refresh.code, 200, register_with_refresh.result)
+        self.assertIn("refresh_token", register_with_refresh.json_body)
+        self.assertIn("expires_in_ms", register_with_refresh.json_body)
+
+    def test_token_refresh(self):
+        """
+        A refresh token can be used to issue a new access token.
+        """
+        body = {"type": "m.login.password", "user": "test", "password": self.user_pass}
+        login_response = self.make_request(
+            "POST",
+            "/_matrix/client/r0/login?org.matrix.msc2918.refresh_token=true",
+            body,
+        )
+        self.assertEqual(login_response.code, 200, login_response.result)
+
+        refresh_response = self.make_request(
+            "POST",
+            "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh",
+            {"refresh_token": login_response.json_body["refresh_token"]},
+        )
+        self.assertEqual(refresh_response.code, 200, refresh_response.result)
+        self.assertIn("access_token", refresh_response.json_body)
+        self.assertIn("refresh_token", refresh_response.json_body)
+        self.assertIn("expires_in_ms", refresh_response.json_body)
+
+        # The access and refresh tokens should be different from the original ones after refresh
+        self.assertNotEqual(
+            login_response.json_body["access_token"],
+            refresh_response.json_body["access_token"],
+        )
+        self.assertNotEqual(
+            login_response.json_body["refresh_token"],
+            refresh_response.json_body["refresh_token"],
+        )
+
+    @override_config({"access_token_lifetime": "1m"})
+    def test_refresh_token_expiration(self):
+        """
+        The access token should have some time as specified in the config.
+        """
+        body = {"type": "m.login.password", "user": "test", "password": self.user_pass}
+        login_response = self.make_request(
+            "POST",
+            "/_matrix/client/r0/login?org.matrix.msc2918.refresh_token=true",
+            body,
+        )
+        self.assertEqual(login_response.code, 200, login_response.result)
+        self.assertApproximates(
+            login_response.json_body["expires_in_ms"], 60 * 1000, 100
+        )
+
+        refresh_response = self.make_request(
+            "POST",
+            "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh",
+            {"refresh_token": login_response.json_body["refresh_token"]},
+        )
+        self.assertEqual(refresh_response.code, 200, refresh_response.result)
+        self.assertApproximates(
+            refresh_response.json_body["expires_in_ms"], 60 * 1000, 100
+        )
+
+    def test_refresh_token_invalidation(self):
+        """Refresh tokens are invalidated after first use of the next token.
+
+        A refresh token is considered invalid if:
+            - it was already used at least once
+            - and either
+                - the next access token was used
+                - the next refresh token was used
+
+        The chain of tokens goes like this:
+
+            login -|-> first_refresh -> third_refresh (fails)
+                   |-> second_refresh -> fifth_refresh
+                   |-> fourth_refresh (fails)
+        """
+
+        body = {"type": "m.login.password", "user": "test", "password": self.user_pass}
+        login_response = self.make_request(
+            "POST",
+            "/_matrix/client/r0/login?org.matrix.msc2918.refresh_token=true",
+            body,
+        )
+        self.assertEqual(login_response.code, 200, login_response.result)
+
+        # This first refresh should work properly
+        first_refresh_response = self.make_request(
+            "POST",
+            "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh",
+            {"refresh_token": login_response.json_body["refresh_token"]},
+        )
+        self.assertEqual(
+            first_refresh_response.code, 200, first_refresh_response.result
+        )
+
+        # This one as well, since the token in the first one was never used
+        second_refresh_response = self.make_request(
+            "POST",
+            "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh",
+            {"refresh_token": login_response.json_body["refresh_token"]},
+        )
+        self.assertEqual(
+            second_refresh_response.code, 200, second_refresh_response.result
+        )
+
+        # This one should not, since the token from the first refresh is not valid anymore
+        third_refresh_response = self.make_request(
+            "POST",
+            "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh",
+            {"refresh_token": first_refresh_response.json_body["refresh_token"]},
+        )
+        self.assertEqual(
+            third_refresh_response.code, 401, third_refresh_response.result
+        )
+
+        # The associated access token should also be invalid
+        whoami_response = self.make_request(
+            "GET",
+            "/_matrix/client/r0/account/whoami",
+            access_token=first_refresh_response.json_body["access_token"],
+        )
+        self.assertEqual(whoami_response.code, 401, whoami_response.result)
+
+        # But all other tokens should work (they will expire after some time)
+        for access_token in [
+            second_refresh_response.json_body["access_token"],
+            login_response.json_body["access_token"],
+        ]:
+            whoami_response = self.make_request(
+                "GET", "/_matrix/client/r0/account/whoami", access_token=access_token
+            )
+            self.assertEqual(whoami_response.code, 200, whoami_response.result)
+
+        # Now that the access token from the last valid refresh was used once, refreshing with the N-1 token should fail
+        fourth_refresh_response = self.make_request(
+            "POST",
+            "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh",
+            {"refresh_token": login_response.json_body["refresh_token"]},
+        )
+        self.assertEqual(
+            fourth_refresh_response.code, 403, fourth_refresh_response.result
+        )
+
+        # But refreshing from the last valid refresh token still works
+        fifth_refresh_response = self.make_request(
+            "POST",
+            "/_matrix/client/unstable/org.matrix.msc2918.refresh_token/refresh",
+            {"refresh_token": second_refresh_response.json_body["refresh_token"]},
+        )
+        self.assertEqual(
+            fifth_refresh_response.code, 200, fifth_refresh_response.result
+        )
diff --git a/tests/rest/client/v2_alpha/test_capabilities.py b/tests/rest/client/v2_alpha/test_capabilities.py
index 287a1a485c..13b3c5f499 100644
--- a/tests/rest/client/v2_alpha/test_capabilities.py
+++ b/tests/rest/client/v2_alpha/test_capabilities.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +13,7 @@
 # limitations under the License.
 import synapse.rest.admin
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
-from synapse.rest.client.v1 import login
-from synapse.rest.client.v2_alpha import capabilities
+from synapse.rest.client import capabilities, login
 
 from tests import unittest
 from tests.unittest import override_config
@@ -103,3 +101,49 @@ def test_get_change_password_capabilities_password_disabled(self):
 
         self.assertEqual(channel.code, 200)
         self.assertFalse(capabilities["m.change_password"]["enabled"])
+
+    @override_config({"experimental_features": {"msc3244_enabled": False}})
+    def test_get_does_not_include_msc3244_fields_when_disabled(self):
+        localpart = "user"
+        password = "pass"
+        user = self.register_user(localpart, password)
+        access_token = self.get_success(
+            self.auth_handler.get_access_token_for_user_id(
+                user, device_id=None, valid_until_ms=None
+            )
+        )
+
+        channel = self.make_request("GET", self.url, access_token=access_token)
+        capabilities = channel.json_body["capabilities"]
+
+        self.assertEqual(channel.code, 200)
+        self.assertNotIn(
+            "org.matrix.msc3244.room_capabilities", capabilities["m.room_versions"]
+        )
+
+    def test_get_does_include_msc3244_fields_when_enabled(self):
+        localpart = "user"
+        password = "pass"
+        user = self.register_user(localpart, password)
+        access_token = self.get_success(
+            self.auth_handler.get_access_token_for_user_id(
+                user, device_id=None, valid_until_ms=None
+            )
+        )
+
+        channel = self.make_request("GET", self.url, access_token=access_token)
+        capabilities = channel.json_body["capabilities"]
+
+        self.assertEqual(channel.code, 200)
+        for details in capabilities["m.room_versions"][
+            "org.matrix.msc3244.room_capabilities"
+        ].values():
+            if details["preferred"] is not None:
+                self.assertTrue(
+                    details["preferred"] in KNOWN_ROOM_VERSIONS,
+                    str(details["preferred"]),
+                )
+
+            self.assertGreater(len(details["support"]), 0)
+            for room_version in details["support"]:
+                self.assertTrue(room_version in KNOWN_ROOM_VERSIONS, str(room_version))
diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py
index f761c44936..475c6bed3d 100644
--- a/tests/rest/client/v2_alpha/test_filter.py
+++ b/tests/rest/client/v2_alpha/test_filter.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,7 +15,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import Codes
-from synapse.rest.client.v2_alpha import filter
+from synapse.rest.client import filter
 
 from tests import unittest
 
diff --git a/tests/rest/client/v2_alpha/test_groups.py b/tests/rest/client/v2_alpha/test_groups.py
new file mode 100644
index 0000000000..ad0425ae65
--- /dev/null
+++ b/tests/rest/client/v2_alpha/test_groups.py
@@ -0,0 +1,56 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.rest.client import groups, room
+
+from tests import unittest
+from tests.unittest import override_config
+
+
+class GroupsTestCase(unittest.HomeserverTestCase):
+    user_id = "@alice:test"
+    room_creator_user_id = "@bob:test"
+
+    servlets = [room.register_servlets, groups.register_servlets]
+
+    @override_config({"enable_group_creation": True})
+    def test_rooms_limited_by_visibility(self):
+        group_id = "+spqr:test"
+
+        # Alice creates a group
+        channel = self.make_request("POST", "/create_group", {"localpart": "spqr"})
+        self.assertEquals(channel.code, 200, msg=channel.text_body)
+        self.assertEquals(channel.json_body, {"group_id": group_id})
+
+        # Bob creates a private room
+        room_id = self.helper.create_room_as(self.room_creator_user_id, is_public=False)
+        self.helper.auth_user_id = self.room_creator_user_id
+        self.helper.send_state(
+            room_id, "m.room.name", {"name": "bob's secret room"}, tok=None
+        )
+        self.helper.auth_user_id = self.user_id
+
+        # Alice adds the room to her group.
+        channel = self.make_request(
+            "PUT", f"/groups/{group_id}/admin/rooms/{room_id}", {}
+        )
+        self.assertEquals(channel.code, 200, msg=channel.text_body)
+        self.assertEquals(channel.json_body, {})
+
+        # Alice now tries to retrieve the room list of the space.
+        channel = self.make_request("GET", f"/groups/{group_id}/rooms")
+        self.assertEquals(channel.code, 200, msg=channel.text_body)
+        self.assertEquals(
+            channel.json_body, {"chunk": [], "total_room_count_estimate": 0}
+        )
diff --git a/tests/rest/client/v2_alpha/test_password_policy.py b/tests/rest/client/v2_alpha/test_password_policy.py
index 5ebc5707a5..3cf5871899 100644
--- a/tests/rest/client/v2_alpha/test_password_policy.py
+++ b/tests/rest/client/v2_alpha/test_password_policy.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,8 +17,7 @@
 from synapse.api.constants import LoginType
 from synapse.api.errors import Codes
 from synapse.rest import admin
-from synapse.rest.client.v1 import login
-from synapse.rest.client.v2_alpha import account, password_policy, register
+from synapse.rest.client import account, login, password_policy, register
 
 from tests import unittest
 
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index 2d4ce871eb..ec3193955a 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -14,25 +13,32 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import datetime
 import json
 import os
 import os.path
 import tempfile
-
-from mock import Mock
+from unittest.mock import Mock
 
 import pkg_resources
 
 from twisted.internet import defer
 
 import synapse.rest.admin
-from synapse.api.constants import LoginType
+from synapse.api.constants import APP_SERVICE_REGISTRATION_TYPE, LoginType
 from synapse.api.errors import Codes
 from synapse.appservice import ApplicationService
-from synapse.rest.client.v1 import login, logout
-from synapse.rest.client.v2_alpha import account, account_validity, register, sync
+from synapse.rest.client import (
+    account,
+    account_validity,
+    login,
+    logout,
+    profile,
+    register,
+    room,
+    sync,
+    user_directory,
+)
 
 from tests import unittest
 from tests.unittest import override_config
@@ -65,7 +71,9 @@ def test_POST_appservice_registration_valid(self):
         )
 
         self.hs.get_datastore().services_cache.append(appservice)
-        request_data = json.dumps({"username": "as_user_kermit"})
+        request_data = json.dumps(
+            {"username": "as_user_kermit", "type": APP_SERVICE_REGISTRATION_TYPE}
+        )
 
         channel = self.make_request(
             b"POST", self.url + b"?access_token=i_am_an_app_service", request_data
@@ -75,9 +83,31 @@ def test_POST_appservice_registration_valid(self):
         det_data = {"user_id": user_id, "home_server": self.hs.hostname}
         self.assertDictContainsSubset(det_data, channel.json_body)
 
+    def test_POST_appservice_registration_no_type(self):
+        as_token = "i_am_an_app_service"
+
+        appservice = ApplicationService(
+            as_token,
+            self.hs.config.server_name,
+            id="1234",
+            namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]},
+            sender="@as:test",
+        )
+
+        self.hs.get_datastore().services_cache.append(appservice)
+        request_data = json.dumps({"username": "as_user_kermit"})
+
+        channel = self.make_request(
+            b"POST", self.url + b"?access_token=i_am_an_app_service", request_data
+        )
+
+        self.assertEquals(channel.result["code"], b"400", channel.result)
+
     def test_POST_appservice_registration_invalid(self):
         self.appservice = None  # no application service exists
-        request_data = json.dumps({"username": "kermit"})
+        request_data = json.dumps(
+            {"username": "kermit", "type": APP_SERVICE_REGISTRATION_TYPE}
+        )
         channel = self.make_request(
             b"POST", self.url + b"?access_token=i_am_an_app_service", request_data
         )
@@ -287,6 +317,57 @@ def test_request_token_existing_email_inhibit_error(self):
 
         self.assertIsNotNone(channel.json_body.get("sid"))
 
+    @unittest.override_config(
+        {
+            "public_baseurl": "https://test_server",
+            "email": {
+                "smtp_host": "mail_server",
+                "smtp_port": 2525,
+                "notif_from": "sender@host",
+            },
+        }
+    )
+    def test_reject_invalid_email(self):
+        """Check that bad emails are rejected"""
+
+        # Test for email with multiple @
+        channel = self.make_request(
+            "POST",
+            b"register/email/requestToken",
+            {"client_secret": "foobar", "email": "email@@email", "send_attempt": 1},
+        )
+        self.assertEquals(400, channel.code, channel.result)
+        # Check error to ensure that we're not erroring due to a bug in the test.
+        self.assertEquals(
+            channel.json_body,
+            {"errcode": "M_UNKNOWN", "error": "Unable to parse email address"},
+        )
+
+        # Test for email with no @
+        channel = self.make_request(
+            "POST",
+            b"register/email/requestToken",
+            {"client_secret": "foobar", "email": "email", "send_attempt": 1},
+        )
+        self.assertEquals(400, channel.code, channel.result)
+        self.assertEquals(
+            channel.json_body,
+            {"errcode": "M_UNKNOWN", "error": "Unable to parse email address"},
+        )
+
+        # Test for super long email
+        email = "a@" + "a" * 1000
+        channel = self.make_request(
+            "POST",
+            b"register/email/requestToken",
+            {"client_secret": "foobar", "email": email, "send_attempt": 1},
+        )
+        self.assertEquals(400, channel.code, channel.result)
+        self.assertEquals(
+            channel.json_body,
+            {"errcode": "M_UNKNOWN", "error": "Unable to parse email address"},
+        )
+
 
 class RegisterHideProfileTestCase(unittest.HomeserverTestCase):
 
@@ -501,9 +582,9 @@ def test_logging_out_expired_user(self):
 class AccountValidityUserDirectoryTestCase(unittest.HomeserverTestCase):
 
     servlets = [
-        synapse.rest.client.v1.profile.register_servlets,
-        synapse.rest.client.v1.room.register_servlets,
-        synapse.rest.client.v2_alpha.user_directory.register_servlets,
+        profile.register_servlets,
+        room.register_servlets,
+        user_directory.register_servlets,
         login.register_servlets,
         register.register_servlets,
         synapse.rest.admin.register_servlets_for_client_rest_resource,
@@ -666,10 +747,6 @@ def make_homeserver(self, reactor, clock):
         }
 
         # Email config.
-        self.email_attempts = []
-
-        async def sendmail(*args, **kwargs):
-            self.email_attempts.append((args, kwargs))
 
         config["email"] = {
             "enable_notifs": True,
@@ -689,7 +766,13 @@ async def sendmail(*args, **kwargs):
         }
         config["public_baseurl"] = "aaa"
 
-        self.hs = self.setup_test_homeserver(config=config, sendmail=sendmail)
+        self.hs = self.setup_test_homeserver(config=config)
+
+        async def sendmail(*args, **kwargs):
+            self.email_attempts.append((args, kwargs))
+
+        self.email_attempts = []
+        self.hs.get_send_email_handler()._sendmail = sendmail
 
         self.store = self.hs.get_datastore()
 
@@ -717,7 +800,7 @@ def test_renewal_email(self):
 
         # Check that the HTML we're getting is the one we expect on a successful renewal.
         expiration_ts = self.get_success(self.store.get_expiration_ts_for_user(user_id))
-        expected_html = self.hs.config.account_validity_account_renewed_template.render(
+        expected_html = self.hs.config.account_validity.account_validity_account_renewed_template.render(
             expiration_ts=expiration_ts
         )
         self.assertEqual(
@@ -735,10 +818,8 @@ def test_renewal_email(self):
 
         # Check that the HTML we're getting is the one we expect when reusing a
         # token. The account expiration date should not have changed.
-        expected_html = (
-            self.hs.config.account_validity_account_previously_renewed_template.render(
-                expiration_ts=expiration_ts
-            )
+        expected_html = self.hs.config.account_validity.account_validity_account_previously_renewed_template.render(
+            expiration_ts=expiration_ts
         )
         self.assertEqual(
             channel.result["body"], expected_html.encode("utf8"), channel.result
@@ -764,7 +845,9 @@ def test_renewal_invalid_token(self):
 
         # Check that the HTML we're getting is the one we expect when using an
         # invalid/unknown token.
-        expected_html = self.hs.config.account_validity_invalid_token_template.render()
+        expected_html = (
+            self.hs.config.account_validity.account_validity_invalid_token_template.render()
+        )
         self.assertEqual(
             channel.result["body"], expected_html.encode("utf8"), channel.result
         )
diff --git a/tests/rest/client/v2_alpha/test_relations.py b/tests/rest/client/v2_alpha/test_relations.py
index e7bb5583fc..02b5e9a8d0 100644
--- a/tests/rest/client/v2_alpha/test_relations.py
+++ b/tests/rest/client/v2_alpha/test_relations.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,11 +15,11 @@
 import itertools
 import json
 import urllib
+from typing import Optional
 
 from synapse.api.constants import EventTypes, RelationTypes
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import register, relations
+from synapse.rest.client import login, register, relations, room
 
 from tests import unittest
 
@@ -273,7 +272,7 @@ def test_aggregation_pagination_within_group(self):
 
         prev_token = None
         found_event_ids = []
-        encoded_key = urllib.parse.quote_plus("👍".encode("utf-8"))
+        encoded_key = urllib.parse.quote_plus("👍".encode())
         for _ in range(20):
             from_token = ""
             if prev_token:
@@ -681,7 +680,7 @@ def _send_relation(
         relation_type,
         event_type,
         key=None,
-        content={},
+        content: Optional[dict] = None,
         access_token=None,
         parent_id=None,
     ):
@@ -713,7 +712,7 @@ def _send_relation(
             "POST",
             "/_matrix/client/unstable/rooms/%s/send_relation/%s/%s/%s%s"
             % (self.room, original_id, relation_type, event_type, query),
-            json.dumps(content).encode("utf-8"),
+            json.dumps(content or {}).encode("utf-8"),
             access_token=access_token,
         )
         return channel
diff --git a/tests/rest/client/v2_alpha/test_report_event.py b/tests/rest/client/v2_alpha/test_report_event.py
new file mode 100644
index 0000000000..ee6b0b9ebf
--- /dev/null
+++ b/tests/rest/client/v2_alpha/test_report_event.py
@@ -0,0 +1,82 @@
+# Copyright 2021 Callum Brown
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import synapse.rest.admin
+from synapse.rest.client import login, report_event, room
+
+from tests import unittest
+
+
+class ReportEventTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        login.register_servlets,
+        room.register_servlets,
+        report_event.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        self.admin_user = self.register_user("admin", "pass", admin=True)
+        self.admin_user_tok = self.login("admin", "pass")
+        self.other_user = self.register_user("user", "pass")
+        self.other_user_tok = self.login("user", "pass")
+
+        self.room_id = self.helper.create_room_as(
+            self.other_user, tok=self.other_user_tok, is_public=True
+        )
+        self.helper.join(self.room_id, user=self.admin_user, tok=self.admin_user_tok)
+        resp = self.helper.send(self.room_id, tok=self.admin_user_tok)
+        self.event_id = resp["event_id"]
+        self.report_path = f"rooms/{self.room_id}/report/{self.event_id}"
+
+    def test_reason_str_and_score_int(self):
+        data = {"reason": "this makes me sad", "score": -100}
+        self._assert_status(200, data)
+
+    def test_no_reason(self):
+        data = {"score": 0}
+        self._assert_status(200, data)
+
+    def test_no_score(self):
+        data = {"reason": "this makes me sad"}
+        self._assert_status(200, data)
+
+    def test_no_reason_and_no_score(self):
+        data = {}
+        self._assert_status(200, data)
+
+    def test_reason_int_and_score_str(self):
+        data = {"reason": 10, "score": "string"}
+        self._assert_status(400, data)
+
+    def test_reason_zero_and_score_blank(self):
+        data = {"reason": 0, "score": ""}
+        self._assert_status(400, data)
+
+    def test_reason_and_score_null(self):
+        data = {"reason": None, "score": None}
+        self._assert_status(400, data)
+
+    def _assert_status(self, response_status, data):
+        channel = self.make_request(
+            "POST",
+            self.report_path,
+            json.dumps(data),
+            access_token=self.other_user_tok,
+        )
+        self.assertEqual(
+            response_status, int(channel.result["code"]), msg=channel.result["body"]
+        )
diff --git a/tests/rest/client/v2_alpha/test_sendtodevice.py b/tests/rest/client/v2_alpha/test_sendtodevice.py
new file mode 100644
index 0000000000..6db7062a8e
--- /dev/null
+++ b/tests/rest/client/v2_alpha/test_sendtodevice.py
@@ -0,0 +1,200 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.rest import admin
+from synapse.rest.client import login, sendtodevice, sync
+
+from tests.unittest import HomeserverTestCase, override_config
+
+
+class SendToDeviceTestCase(HomeserverTestCase):
+    servlets = [
+        admin.register_servlets,
+        login.register_servlets,
+        sendtodevice.register_servlets,
+        sync.register_servlets,
+    ]
+
+    def test_user_to_user(self):
+        """A to-device message from one user to another should get delivered"""
+
+        user1 = self.register_user("u1", "pass")
+        user1_tok = self.login("u1", "pass", "d1")
+
+        user2 = self.register_user("u2", "pass")
+        user2_tok = self.login("u2", "pass", "d2")
+
+        # send the message
+        test_msg = {"foo": "bar"}
+        chan = self.make_request(
+            "PUT",
+            "/_matrix/client/r0/sendToDevice/m.test/1234",
+            content={"messages": {user2: {"d2": test_msg}}},
+            access_token=user1_tok,
+        )
+        self.assertEqual(chan.code, 200, chan.result)
+
+        # check it appears
+        channel = self.make_request("GET", "/sync", access_token=user2_tok)
+        self.assertEqual(channel.code, 200, channel.result)
+        expected_result = {
+            "events": [
+                {
+                    "sender": user1,
+                    "type": "m.test",
+                    "content": test_msg,
+                }
+            ]
+        }
+        self.assertEqual(channel.json_body["to_device"], expected_result)
+
+        # it should re-appear if we do another sync
+        channel = self.make_request("GET", "/sync", access_token=user2_tok)
+        self.assertEqual(channel.code, 200, channel.result)
+        self.assertEqual(channel.json_body["to_device"], expected_result)
+
+        # it should *not* appear if we do an incremental sync
+        sync_token = channel.json_body["next_batch"]
+        channel = self.make_request(
+            "GET", f"/sync?since={sync_token}", access_token=user2_tok
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+        self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), [])
+
+    @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}})
+    def test_local_room_key_request(self):
+        """m.room_key_request has special-casing; test from local user"""
+        user1 = self.register_user("u1", "pass")
+        user1_tok = self.login("u1", "pass", "d1")
+
+        user2 = self.register_user("u2", "pass")
+        user2_tok = self.login("u2", "pass", "d2")
+
+        # send three messages
+        for i in range(3):
+            chan = self.make_request(
+                "PUT",
+                f"/_matrix/client/r0/sendToDevice/m.room_key_request/{i}",
+                content={"messages": {user2: {"d2": {"idx": i}}}},
+                access_token=user1_tok,
+            )
+            self.assertEqual(chan.code, 200, chan.result)
+
+        # now sync: we should get two of the three
+        channel = self.make_request("GET", "/sync", access_token=user2_tok)
+        self.assertEqual(channel.code, 200, channel.result)
+        msgs = channel.json_body["to_device"]["events"]
+        self.assertEqual(len(msgs), 2)
+        for i in range(2):
+            self.assertEqual(
+                msgs[i],
+                {"sender": user1, "type": "m.room_key_request", "content": {"idx": i}},
+            )
+        sync_token = channel.json_body["next_batch"]
+
+        # ... time passes
+        self.reactor.advance(1)
+
+        # and we can send more messages
+        chan = self.make_request(
+            "PUT",
+            "/_matrix/client/r0/sendToDevice/m.room_key_request/3",
+            content={"messages": {user2: {"d2": {"idx": 3}}}},
+            access_token=user1_tok,
+        )
+        self.assertEqual(chan.code, 200, chan.result)
+
+        # ... which should arrive
+        channel = self.make_request(
+            "GET", f"/sync?since={sync_token}", access_token=user2_tok
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+        msgs = channel.json_body["to_device"]["events"]
+        self.assertEqual(len(msgs), 1)
+        self.assertEqual(
+            msgs[0],
+            {"sender": user1, "type": "m.room_key_request", "content": {"idx": 3}},
+        )
+
+    @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}})
+    def test_remote_room_key_request(self):
+        """m.room_key_request has special-casing; test from remote user"""
+        user2 = self.register_user("u2", "pass")
+        user2_tok = self.login("u2", "pass", "d2")
+
+        federation_registry = self.hs.get_federation_registry()
+
+        # send three messages
+        for i in range(3):
+            self.get_success(
+                federation_registry.on_edu(
+                    "m.direct_to_device",
+                    "remote_server",
+                    {
+                        "sender": "@user:remote_server",
+                        "type": "m.room_key_request",
+                        "messages": {user2: {"d2": {"idx": i}}},
+                        "message_id": f"{i}",
+                    },
+                )
+            )
+
+        # now sync: we should get two of the three
+        channel = self.make_request("GET", "/sync", access_token=user2_tok)
+        self.assertEqual(channel.code, 200, channel.result)
+        msgs = channel.json_body["to_device"]["events"]
+        self.assertEqual(len(msgs), 2)
+        for i in range(2):
+            self.assertEqual(
+                msgs[i],
+                {
+                    "sender": "@user:remote_server",
+                    "type": "m.room_key_request",
+                    "content": {"idx": i},
+                },
+            )
+        sync_token = channel.json_body["next_batch"]
+
+        # ... time passes
+        self.reactor.advance(1)
+
+        # and we can send more messages
+        self.get_success(
+            federation_registry.on_edu(
+                "m.direct_to_device",
+                "remote_server",
+                {
+                    "sender": "@user:remote_server",
+                    "type": "m.room_key_request",
+                    "messages": {user2: {"d2": {"idx": 3}}},
+                    "message_id": "3",
+                },
+            )
+        )
+
+        # ... which should arrive
+        channel = self.make_request(
+            "GET", f"/sync?since={sync_token}", access_token=user2_tok
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+        msgs = channel.json_body["to_device"]["events"]
+        self.assertEqual(len(msgs), 1)
+        self.assertEqual(
+            msgs[0],
+            {
+                "sender": "@user:remote_server",
+                "type": "m.room_key_request",
+                "content": {"idx": 3},
+            },
+        )
diff --git a/tests/rest/client/v2_alpha/test_shared_rooms.py b/tests/rest/client/v2_alpha/test_shared_rooms.py
index dd83a1f8ff..283eccd53f 100644
--- a/tests/rest/client/v2_alpha/test_shared_rooms.py
+++ b/tests/rest/client/v2_alpha/test_shared_rooms.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Half-Shot
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import synapse.rest.admin
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import shared_rooms
+from synapse.rest.client import login, room, shared_rooms
 
 from tests import unittest
 from tests.server import FakeChannel
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index 8755bfb38a..95be369d4b 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018-2019 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
@@ -19,12 +18,10 @@
 from synapse.api.constants import (
     EventContentFields,
     EventTypes,
-    Membership,
+    ReadReceiptEventFields,
     RelationTypes,
 )
-from synapse.api.room_versions import RoomVersions
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import knock, read_marker, sync
+from synapse.rest.client import knock, login, read_marker, receipts, room, sync
 
 from tests import unittest
 from tests.federation.transport.test_knocking import (
@@ -48,35 +45,7 @@ def test_sync_argless(self):
         channel = self.make_request("GET", "/sync")
 
         self.assertEqual(channel.code, 200)
-        self.assertTrue(
-            {
-                "next_batch",
-                "rooms",
-                "presence",
-                "account_data",
-                "to_device",
-                "device_lists",
-            }.issubset(set(channel.json_body.keys()))
-        )
-
-    def test_sync_presence_disabled(self):
-        """
-        When presence is disabled, the key does not appear in /sync.
-        """
-        self.hs.config.use_presence = False
-
-        channel = self.make_request("GET", "/sync")
-
-        self.assertEqual(channel.code, 200)
-        self.assertTrue(
-            {
-                "next_batch",
-                "rooms",
-                "account_data",
-                "to_device",
-                "device_lists",
-            }.issubset(set(channel.json_body.keys()))
-        )
+        self.assertIn("next_batch", channel.json_body)
 
 
 class SyncFilterTestCase(unittest.HomeserverTestCase):
@@ -340,7 +309,7 @@ def prepare(self, reactor, clock, hs):
         self.room_id = self.helper.create_room_as(
             self.user_id,
             is_public=False,
-            room_version=RoomVersions.V7.identifier,
+            room_version="7",
             tok=self.tok,
         )
 
@@ -370,7 +339,7 @@ def test_knock_room_state(self):
         # Knock on a room
         channel = self.make_request(
             "POST",
-            "/_matrix/client/unstable/xyz.amorgan.knock/%s" % (self.room_id,),
+            "/_matrix/client/r0/knock/%s" % (self.room_id,),
             b"{}",
             self.knocker_tok,
         )
@@ -378,7 +347,7 @@ def test_knock_room_state(self):
 
         # We expect to see the knock event in the stripped room state later
         self.expected_room_state[EventTypes.Member] = {
-            "content": {"membership": Membership.KNOCK, "displayname": "knocker"},
+            "content": {"membership": "knock", "displayname": "knocker"},
             "state_key": "@knocker:test",
         }
 
@@ -391,7 +360,7 @@ def test_knock_room_state(self):
         self.assertEqual(channel.code, 200, channel.json_body)
 
         # Extract the stripped room state events from /sync
-        knock_entry = channel.json_body["rooms"][Membership.KNOCK]
+        knock_entry = channel.json_body["rooms"]["knock"]
         room_state_events = knock_entry[self.room_id]["knock_state"]["events"]
 
         # Validate that the knock membership event came last
@@ -403,6 +372,88 @@ def test_knock_room_state(self):
         )
 
 
+class ReadReceiptsTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        login.register_servlets,
+        receipts.register_servlets,
+        room.register_servlets,
+        sync.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        self.url = "/sync?since=%s"
+        self.next_batch = "s0"
+
+        # Register the first user
+        self.user_id = self.register_user("kermit", "monkey")
+        self.tok = self.login("kermit", "monkey")
+
+        # Create the room
+        self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok)
+
+        # Register the second user
+        self.user2 = self.register_user("kermit2", "monkey")
+        self.tok2 = self.login("kermit2", "monkey")
+
+        # Join the second user
+        self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2)
+
+    @override_config({"experimental_features": {"msc2285_enabled": True}})
+    def test_hidden_read_receipts(self):
+        # Send a message as the first user
+        res = self.helper.send(self.room_id, body="hello", tok=self.tok)
+
+        # Send a read receipt to tell the server the first user's message was read
+        body = json.dumps({ReadReceiptEventFields.MSC2285_HIDDEN: True}).encode("utf8")
+        channel = self.make_request(
+            "POST",
+            "/rooms/%s/receipt/m.read/%s" % (self.room_id, res["event_id"]),
+            body,
+            access_token=self.tok2,
+        )
+        self.assertEqual(channel.code, 200)
+
+        # Test that the first user can't see the other user's hidden read receipt
+        self.assertEqual(self._get_read_receipt(), None)
+
+    def test_read_receipt_with_empty_body(self):
+        # Send a message as the first user
+        res = self.helper.send(self.room_id, body="hello", tok=self.tok)
+
+        # Send a read receipt for this message with an empty body
+        channel = self.make_request(
+            "POST",
+            "/rooms/%s/receipt/m.read/%s" % (self.room_id, res["event_id"]),
+            access_token=self.tok2,
+        )
+        self.assertEqual(channel.code, 200)
+
+    def _get_read_receipt(self):
+        """Syncs and returns the read receipt."""
+
+        # Checks if event is a read receipt
+        def is_read_receipt(event):
+            return event["type"] == "m.receipt"
+
+        # Sync
+        channel = self.make_request(
+            "GET",
+            self.url % self.next_batch,
+            access_token=self.tok,
+        )
+        self.assertEqual(channel.code, 200)
+
+        # Store the next batch for the next request.
+        self.next_batch = channel.json_body["next_batch"]
+
+        # Return the read receipt
+        ephemeral_events = channel.json_body["rooms"]["join"][self.room_id][
+            "ephemeral"
+        ]["events"]
+        return next(filter(is_read_receipt, ephemeral_events), None)
+
+
 class UnreadMessagesTestCase(unittest.HomeserverTestCase):
     servlets = [
         synapse.rest.admin.register_servlets,
@@ -410,6 +461,7 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase):
         read_marker.register_servlets,
         room.register_servlets,
         sync.register_servlets,
+        receipts.register_servlets,
     ]
 
     def prepare(self, reactor, clock, hs):
@@ -483,6 +535,23 @@ def test_unread_counts(self):
         # Check that the unread counter is back to 0.
         self._check_unread_count(0)
 
+        # Check that hidden read receipts don't break unread counts
+        res = self.helper.send(self.room_id, "hello", tok=self.tok2)
+        self._check_unread_count(1)
+
+        # Send a read receipt to tell the server we've read the latest event.
+        body = json.dumps({ReadReceiptEventFields.MSC2285_HIDDEN: True}).encode("utf8")
+        channel = self.make_request(
+            "POST",
+            "/rooms/%s/receipt/m.read/%s" % (self.room_id, res["event_id"]),
+            body,
+            access_token=self.tok,
+        )
+        self.assertEqual(channel.code, 200, channel.json_body)
+
+        # Check that the unread counter is back to 0.
+        self._check_unread_count(0)
+
         # Check that room name changes increase the unread counter.
         self.helper.send_state(
             self.room_id,
@@ -565,3 +634,53 @@ def _check_unread_count(self, expected_count: int):
 
         # Store the next batch for the next request.
         self.next_batch = channel.json_body["next_batch"]
+
+
+class SyncCacheTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        login.register_servlets,
+        sync.register_servlets,
+    ]
+
+    def test_noop_sync_does_not_tightloop(self):
+        """If the sync times out, we shouldn't cache the result
+
+        Essentially a regression test for #8518.
+        """
+        self.user_id = self.register_user("kermit", "monkey")
+        self.tok = self.login("kermit", "monkey")
+
+        # we should immediately get an initial sync response
+        channel = self.make_request("GET", "/sync", access_token=self.tok)
+        self.assertEqual(channel.code, 200, channel.json_body)
+
+        # now, make an incremental sync request, with a timeout
+        next_batch = channel.json_body["next_batch"]
+        channel = self.make_request(
+            "GET",
+            f"/sync?since={next_batch}&timeout=10000",
+            access_token=self.tok,
+            await_result=False,
+        )
+        # that should block for 10 seconds
+        with self.assertRaises(TimedOutException):
+            channel.await_result(timeout_ms=9900)
+        channel.await_result(timeout_ms=200)
+        self.assertEqual(channel.code, 200, channel.json_body)
+
+        # we expect the next_batch in the result to be the same as before
+        self.assertEqual(channel.json_body["next_batch"], next_batch)
+
+        # another incremental sync should also block.
+        channel = self.make_request(
+            "GET",
+            f"/sync?since={next_batch}&timeout=10000",
+            access_token=self.tok,
+            await_result=False,
+        )
+        # that should block for 10 seconds
+        with self.assertRaises(TimedOutException):
+            channel.await_result(timeout_ms=9900)
+        channel.await_result(timeout_ms=200)
+        self.assertEqual(channel.code, 200, channel.json_body)
diff --git a/tests/rest/client/v2_alpha/test_upgrade_room.py b/tests/rest/client/v2_alpha/test_upgrade_room.py
index d890d11863..72f976d8e2 100644
--- a/tests/rest/client/v2_alpha/test_upgrade_room.py
+++ b/tests/rest/client/v2_alpha/test_upgrade_room.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,8 +15,7 @@
 
 from synapse.config.server import DEFAULT_ROOM_VERSION
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import room_upgrade_rest_servlet
+from synapse.rest.client import login, room, room_upgrade_rest_servlet
 
 from tests import unittest
 from tests.server import FakeChannel
diff --git a/tests/rest/key/v2/test_remote_key_resource.py b/tests/rest/key/v2/test_remote_key_resource.py
index 9d0d0ef414..a75c0ea3f0 100644
--- a/tests/rest/key/v2/test_remote_key_resource.py
+++ b/tests/rest/key/v2/test_remote_key_resource.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +13,7 @@
 # limitations under the License.
 import urllib.parse
 from io import BytesIO, StringIO
-
-from mock import Mock
+from unittest.mock import Mock
 
 import signedjson.key
 from canonicaljson import encode_canonical_json
@@ -210,10 +208,10 @@ def test_get_key(self):
         keyid = "ed25519:%s" % (testkey.version,)
 
         fetcher = PerspectivesKeyFetcher(self.hs2)
-        d = fetcher.get_keys({"targetserver": {keyid: 1000}})
+        d = fetcher.get_keys("targetserver", [keyid], 1000)
         res = self.get_success(d)
-        self.assertIn("targetserver", res)
-        keyres = res["targetserver"][keyid]
+        self.assertIn(keyid, res)
+        keyres = res[keyid]
         assert isinstance(keyres, FetchKeyResult)
         self.assertEqual(
             signedjson.key.encode_verify_key_base64(keyres.verify_key),
@@ -232,10 +230,10 @@ def test_get_notary_key(self):
         keyid = "ed25519:%s" % (testkey.version,)
 
         fetcher = PerspectivesKeyFetcher(self.hs2)
-        d = fetcher.get_keys({self.hs.hostname: {keyid: 1000}})
+        d = fetcher.get_keys(self.hs.hostname, [keyid], 1000)
         res = self.get_success(d)
-        self.assertIn(self.hs.hostname, res)
-        keyres = res[self.hs.hostname][keyid]
+        self.assertIn(keyid, res)
+        keyres = res[keyid]
         assert isinstance(keyres, FetchKeyResult)
         self.assertEqual(
             signedjson.key.encode_verify_key_base64(keyres.verify_key),
@@ -249,10 +247,10 @@ def test_get_notary_keyserver_key(self):
         keyid = "ed25519:%s" % (self.hs_signing_key.version,)
 
         fetcher = PerspectivesKeyFetcher(self.hs2)
-        d = fetcher.get_keys({self.hs.hostname: {keyid: 1000}})
+        d = fetcher.get_keys(self.hs.hostname, [keyid], 1000)
         res = self.get_success(d)
-        self.assertIn(self.hs.hostname, res)
-        keyres = res[self.hs.hostname][keyid]
+        self.assertIn(keyid, res)
+        keyres = res[keyid]
         assert isinstance(keyres, FetchKeyResult)
         self.assertEqual(
             signedjson.key.encode_verify_key_base64(keyres.verify_key),
diff --git a/tests/rest/media/__init__.py b/tests/rest/media/__init__.py
index a354d38ca8..b1ee10cfcc 100644
--- a/tests/rest/media/__init__.py
+++ b/tests/rest/media/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/rest/media/v1/__init__.py b/tests/rest/media/v1/__init__.py
index a354d38ca8..b1ee10cfcc 100644
--- a/tests/rest/media/v1/__init__.py
+++ b/tests/rest/media/v1/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/rest/media/v1/test_base.py b/tests/rest/media/v1/test_base.py
index ebd7869208..f761e23f1b 100644
--- a/tests/rest/media/v1/test_base.py
+++ b/tests/rest/media/v1/test_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/rest/media/v1/test_media_storage.py b/tests/rest/media/v1/test_media_storage.py
index 9f77125fd4..6085444b9d 100644
--- a/tests/rest/media/v1/test_media_storage.py
+++ b/tests/rest/media/v1/test_media_storage.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,10 +17,9 @@
 from binascii import unhexlify
 from io import BytesIO
 from typing import Optional
+from unittest.mock import Mock
 from urllib import parse
 
-from mock import Mock
-
 import attr
 from parameterized import parameterized_class
 from PIL import Image as Image
@@ -29,9 +27,10 @@
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 
+from synapse.events.spamcheck import load_legacy_spam_checkers
 from synapse.logging.context import make_deferred_yieldable
 from synapse.rest import admin
-from synapse.rest.client.v1 import login
+from synapse.rest.client import login
 from synapse.rest.media.v1._base import FileInfo
 from synapse.rest.media.v1.filepath import MediaFilePaths
 from synapse.rest.media.v1.media_storage import MediaStorage
@@ -311,7 +310,7 @@ def test_disposition_filenamestar_utf8escaped(self):
         correctly decode it as the UTF-8 string, and use filename* in the
         response.
         """
-        filename = parse.quote("\u2603".encode("utf8")).encode("ascii")
+        filename = parse.quote("\u2603".encode()).encode("ascii")
         channel = self._req(
             b"inline; filename*=utf-8''" + filename + self.test_image.extension
         )
@@ -537,6 +536,8 @@ def prepare(self, reactor, clock, hs):
         self.download_resource = self.media_repo.children[b"download"]
         self.upload_resource = self.media_repo.children[b"upload"]
 
+        load_legacy_spam_checkers(hs)
+
     def default_config(self):
         config = default_config("test")
 
diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py
index 6968502433..d3ef7bb4c6 100644
--- a/tests/rest/media/v1/test_url_preview.py
+++ b/tests/rest/media/v1/test_url_preview.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,8 +14,7 @@
 import json
 import os
 import re
-
-from mock import patch
+from unittest.mock import patch
 
 from twisted.internet._resolver import HostResolution
 from twisted.internet.address import IPv4Address, IPv6Address
diff --git a/tests/rest/test_health.py b/tests/rest/test_health.py
index 32acd93dc1..01d48c3860 100644
--- a/tests/rest/test_health.py
+++ b/tests/rest/test_health.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/rest/test_well_known.py b/tests/rest/test_well_known.py
index 14de0921be..ac0e427752 100644
--- a/tests/rest/test_well_known.py
+++ b/tests/rest/test_well_known.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/rulecheck/test_domainrulecheck.py b/tests/rulecheck/test_domainrulecheck.py
index 3c70a9c8c0..eee980c9a6 100644
--- a/tests/rulecheck/test_domainrulecheck.py
+++ b/tests/rulecheck/test_domainrulecheck.py
@@ -18,7 +18,8 @@
 
 import synapse.rest.admin
 from synapse.config._base import ConfigError
-from synapse.rest.client.v1 import login, room
+from synapse.events.spamcheck import load_legacy_spam_checkers
+from synapse.rest.client import login, room
 from synapse.rulecheck.domain_rule_checker import DomainRuleChecker
 
 from tests import unittest
@@ -179,6 +180,7 @@ def make_homeserver(self, reactor, clock):
         }
 
         hs = self.setup_test_homeserver(config=config)
+        load_legacy_spam_checkers(hs)
         return hs
 
     def prepare(self, reactor, clock, hs):
@@ -317,13 +319,13 @@ def test_cannot_3pid_invite(self):
         )
         self.assertEqual(channel.code, 403, channel.result["body"])
 
-    def _create_room(self, token, content={}):
+    def _create_room(self, token, content=None):
         path = "/_matrix/client/r0/createRoom?access_token=%s" % (token,)
 
         channel = self.make_request(
             "POST",
             path,
-            content=json.dumps(content).encode("utf8"),
+            content=json.dumps(content or {}).encode("utf8"),
         )
 
         return channel
diff --git a/tests/scripts/test_new_matrix_user.py b/tests/scripts/test_new_matrix_user.py
index 6f56893f5e..6f3c365c9a 100644
--- a/tests/scripts/test_new_matrix_user.py
+++ b/tests/scripts/test_new_matrix_user.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse._scripts.register_new_matrix_user import request_registration
 
diff --git a/tests/server.py b/tests/server.py
index b535a5d886..b861c7b866 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -10,9 +10,10 @@
 
 from twisted.internet import address, threads, udp
 from twisted.internet._resolver import SimpleResolverComplexifier
-from twisted.internet.defer import Deferred, fail, succeed
+from twisted.internet.defer import Deferred, fail, maybeDeferred, succeed
 from twisted.internet.error import DNSLookupError
 from twisted.internet.interfaces import (
+    IAddress,
     IHostnameResolver,
     IProtocol,
     IPullProducer,
@@ -52,7 +53,7 @@ class FakeChannel:
     _reactor = attr.ib()
     result = attr.ib(type=dict, default=attr.Factory(dict))
     _ip = attr.ib(type=str, default="127.0.0.1")
-    _producer = None  # type: Optional[Union[IPullProducer, IPushProducer]]
+    _producer: Optional[Union[IPullProducer, IPushProducer]] = None
 
     @property
     def json_body(self):
@@ -138,21 +139,19 @@ def isSecure(self):
     def transport(self):
         return self
 
-    def await_result(self, timeout: int = 100) -> None:
+    def await_result(self, timeout_ms: int = 1000) -> None:
         """
         Wait until the request is finished.
         """
+        end_time = self._reactor.seconds() + timeout_ms / 1000.0
         self._reactor.run()
-        x = 0
 
         while not self.is_finished():
             # If there's a producer, tell it to resume producing so we get content
             if self._producer:
                 self._producer.resumeProducing()
 
-            x += 1
-
-            if x > timeout:
+            if self._reactor.seconds() > end_time:
                 raise TimedOutException("Timed out waiting for request to finish.")
 
             self._reactor.advance(0.1)
@@ -318,8 +317,10 @@ def __init__(self):
 
         self._tcp_callbacks = {}
         self._udp = []
-        lookups = self.lookups = {}  # type: Dict[str, str]
-        self._thread_callbacks = deque()  # type: Deque[Callable[[], None]]
+        self.lookups: Dict[str, str] = {}
+        self._thread_callbacks: Deque[Callable[[], None]] = deque()
+
+        lookups = self.lookups
 
         @implementer(IResolverSimple)
         class FakeResolver:
@@ -511,6 +512,9 @@ class FakeTransport:
     will get called back for connectionLost() notifications etc.
     """
 
+    _peer_address: Optional[IAddress] = attr.ib(default=None)
+    """The value to be returend by getPeer"""
+
     disconnecting = False
     disconnected = False
     connected = True
@@ -519,7 +523,7 @@ class FakeTransport:
     autoflush = attr.ib(default=True)
 
     def getPeer(self):
-        return None
+        return self._peer_address
 
     def getHost(self):
         return None
@@ -572,7 +576,12 @@ def registerProducer(self, producer, streaming):
         self.producerStreaming = streaming
 
         def _produce():
-            d = self.producer.resumeProducing()
+            if not self.producer:
+                # we've been unregistered
+                return
+            # some implementations of IProducer (for example, FileSender)
+            # don't return a deferred.
+            d = maybeDeferred(self.producer.resumeProducing)
             d.addCallback(lambda x: self._reactor.callLater(0.1, _produce))
 
         if not streaming:
@@ -603,12 +612,6 @@ def flush(self, maxbytes=None):
         if self.disconnected:
             return
 
-        if not hasattr(self.other, "transport"):
-            # the other has no transport yet; reschedule
-            if self.autoflush:
-                self._reactor.callLater(0.0, self.flush)
-            return
-
         if maxbytes is not None:
             to_write = self.buffer[:maxbytes]
         else:
diff --git a/tests/server_notices/test_consent.py b/tests/server_notices/test_consent.py
index 4dd5a36178..58b399a043 100644
--- a/tests/server_notices/test_consent.py
+++ b/tests/server_notices/test_consent.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,8 +15,7 @@
 import os
 
 import synapse.rest.admin
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import sync
+from synapse.rest.client import login, room, sync
 
 from tests import unittest
 
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index d40d65b06a..8701b5f7e3 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018, 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,15 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, LimitBlockingTypes, ServerNoticeMsgType
 from synapse.api.errors import ResourceLimitError
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
-from synapse.rest.client.v2_alpha import sync
+from synapse.rest.client import login, room, sync
 from synapse.server_notices.resource_limits_server_notices import (
     ResourceLimitsServerNotices,
 )
@@ -307,8 +305,9 @@ def test_no_invite_without_notice(self):
 
         channel = self.make_request("GET", "/sync?timeout=0", access_token=tok)
 
-        invites = channel.json_body["rooms"]["invite"]
-        self.assertEqual(len(invites), 0, invites)
+        self.assertNotIn(
+            "rooms", channel.json_body, "Got invites without server notice"
+        )
 
     def test_invite_with_notice(self):
         """Tests that, if the MAU limit is hit, the server notices user invites each user
@@ -365,7 +364,8 @@ def _trigger_notice_and_join(self):
             # We could also pick another user and sync with it, which would return an
             # invite to a system notices room, but it doesn't matter which user we're
             # using so we use the last one because it saves us an extra sync.
-            invites = channel.json_body["rooms"]["invite"]
+            if "rooms" in channel.json_body:
+                invites = channel.json_body["rooms"]["invite"]
 
         # Make sure we have an invite to process.
         self.assertEqual(len(invites), 1, invites)
diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py
index 66e3cafe8e..8370a27195 100644
--- a/tests/state/test_v2.py
+++ b/tests/state/test_v2.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -485,7 +484,7 @@ def do_check(self, events, edges, expected_state_ids):
                 state_d = resolve_events_with_store(
                     FakeClock(),
                     ROOM_ID,
-                    RoomVersions.V2.identifier,
+                    RoomVersions.V2,
                     [state_at_event[n] for n in prev_events],
                     event_map=event_map,
                     state_res_store=TestStateResolutionStore(event_map),
@@ -497,7 +496,7 @@ def do_check(self, events, edges, expected_state_ids):
             if fake_event.state_key is not None:
                 state_after[(fake_event.type, fake_event.state_key)] = event_id
 
-            auth_types = set(auth_types_for_event(fake_event))
+            auth_types = set(auth_types_for_event(RoomVersions.V6, fake_event))
 
             auth_events = []
             for key in auth_types:
@@ -634,7 +633,7 @@ def test_event_map_none(self):
         state_d = resolve_events_with_store(
             FakeClock(),
             ROOM_ID,
-            RoomVersions.V2.identifier,
+            RoomVersions.V2,
             [self.state_at_bob, self.state_at_charlie],
             event_map=None,
             state_res_store=TestStateResolutionStore(self.event_map),
diff --git a/synapse/rest/client/v1/__init__.py b/tests/storage/databases/__init__.py
similarity index 89%
rename from synapse/rest/client/v1/__init__.py
rename to tests/storage/databases/__init__.py
index bfebb0f644..c24c7ecd92 100644
--- a/synapse/rest/client/v1/__init__.py
+++ b/tests/storage/databases/__init__.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/synapse/rest/client/v2_alpha/__init__.py b/tests/storage/databases/main/__init__.py
similarity index 89%
rename from synapse/rest/client/v2_alpha/__init__.py
rename to tests/storage/databases/main/__init__.py
index bfebb0f644..c24c7ecd92 100644
--- a/synapse/rest/client/v2_alpha/__init__.py
+++ b/tests/storage/databases/main/__init__.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py
new file mode 100644
index 0000000000..a649e8c618
--- /dev/null
+++ b/tests/storage/databases/main/test_events_worker.py
@@ -0,0 +1,146 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+
+from synapse.logging.context import LoggingContext
+from synapse.rest import admin
+from synapse.rest.client import login, room
+from synapse.storage.databases.main.events_worker import EventsWorkerStore
+from synapse.util.async_helpers import yieldable_gather_results
+
+from tests import unittest
+
+
+class HaveSeenEventsTestCase(unittest.HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
+        self.store: EventsWorkerStore = hs.get_datastore()
+
+        # insert some test data
+        for rid in ("room1", "room2"):
+            self.get_success(
+                self.store.db_pool.simple_insert(
+                    "rooms",
+                    {"room_id": rid, "room_version": 4},
+                )
+            )
+
+        for idx, (rid, eid) in enumerate(
+            (
+                ("room1", "event10"),
+                ("room1", "event11"),
+                ("room1", "event12"),
+                ("room2", "event20"),
+            )
+        ):
+            self.get_success(
+                self.store.db_pool.simple_insert(
+                    "events",
+                    {
+                        "event_id": eid,
+                        "room_id": rid,
+                        "topological_ordering": idx,
+                        "stream_ordering": idx,
+                        "type": "test",
+                        "processed": True,
+                        "outlier": False,
+                    },
+                )
+            )
+            self.get_success(
+                self.store.db_pool.simple_insert(
+                    "event_json",
+                    {
+                        "event_id": eid,
+                        "room_id": rid,
+                        "json": json.dumps({"type": "test", "room_id": rid}),
+                        "internal_metadata": "{}",
+                        "format_version": 3,
+                    },
+                )
+            )
+
+    def test_simple(self):
+        with LoggingContext(name="test") as ctx:
+            res = self.get_success(
+                self.store.have_seen_events("room1", ["event10", "event19"])
+            )
+            self.assertEquals(res, {"event10"})
+
+            # that should result in a single db query
+            self.assertEquals(ctx.get_resource_usage().db_txn_count, 1)
+
+        # a second lookup of the same events should cause no queries
+        with LoggingContext(name="test") as ctx:
+            res = self.get_success(
+                self.store.have_seen_events("room1", ["event10", "event19"])
+            )
+            self.assertEquals(res, {"event10"})
+            self.assertEquals(ctx.get_resource_usage().db_txn_count, 0)
+
+    def test_query_via_event_cache(self):
+        # fetch an event into the event cache
+        self.get_success(self.store.get_event("event10"))
+
+        # looking it up should now cause no db hits
+        with LoggingContext(name="test") as ctx:
+            res = self.get_success(self.store.have_seen_events("room1", ["event10"]))
+            self.assertEquals(res, {"event10"})
+            self.assertEquals(ctx.get_resource_usage().db_txn_count, 0)
+
+
+class EventCacheTestCase(unittest.HomeserverTestCase):
+    """Test that the various layers of event cache works."""
+
+    servlets = [
+        admin.register_servlets,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        self.store: EventsWorkerStore = hs.get_datastore()
+
+        self.user = self.register_user("user", "pass")
+        self.token = self.login(self.user, "pass")
+
+        self.room = self.helper.create_room_as(self.user, tok=self.token)
+
+        res = self.helper.send(self.room, tok=self.token)
+        self.event_id = res["event_id"]
+
+        # Reset the event cache so the tests start with it empty
+        self.store._get_event_cache.clear()
+
+    def test_simple(self):
+        """Test that we cache events that we pull from the DB."""
+
+        with LoggingContext("test") as ctx:
+            self.get_success(self.store.get_event(self.event_id))
+
+            # We should have fetched the event from the DB
+            self.assertEqual(ctx.get_resource_usage().evt_db_fetch_count, 1)
+
+    def test_dedupe(self):
+        """Test that if we request the same event multiple times we only pull it
+        out once.
+        """
+
+        with LoggingContext("test") as ctx:
+            d = yieldable_gather_results(
+                self.store.get_event, [self.event_id, self.event_id]
+            )
+            self.get_success(d)
+
+            # We should have fetched the event from the DB
+            self.assertEqual(ctx.get_resource_usage().evt_db_fetch_count, 1)
diff --git a/tests/storage/databases/main/test_lock.py b/tests/storage/databases/main/test_lock.py
new file mode 100644
index 0000000000..d326a1d6a6
--- /dev/null
+++ b/tests/storage/databases/main/test_lock.py
@@ -0,0 +1,113 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.server import HomeServer
+from synapse.storage.databases.main.lock import _LOCK_TIMEOUT_MS
+
+from tests import unittest
+
+
+class LockTestCase(unittest.HomeserverTestCase):
+    def prepare(self, reactor, clock, hs: HomeServer):
+        self.store = hs.get_datastore()
+
+    def test_simple_lock(self):
+        """Test that we can take out a lock and that while we hold it nobody
+        else can take it out.
+        """
+        # First to acquire this lock, so it should complete
+        lock = self.get_success(self.store.try_acquire_lock("name", "key"))
+        self.assertIsNotNone(lock)
+
+        # Enter the context manager
+        self.get_success(lock.__aenter__())
+
+        # Attempting to acquire the lock again fails.
+        lock2 = self.get_success(self.store.try_acquire_lock("name", "key"))
+        self.assertIsNone(lock2)
+
+        # Calling `is_still_valid` reports true.
+        self.assertTrue(self.get_success(lock.is_still_valid()))
+
+        # Drop the lock
+        self.get_success(lock.__aexit__(None, None, None))
+
+        # We can now acquire the lock again.
+        lock3 = self.get_success(self.store.try_acquire_lock("name", "key"))
+        self.assertIsNotNone(lock3)
+        self.get_success(lock3.__aenter__())
+        self.get_success(lock3.__aexit__(None, None, None))
+
+    def test_maintain_lock(self):
+        """Test that we don't time out locks while they're still active"""
+
+        lock = self.get_success(self.store.try_acquire_lock("name", "key"))
+        self.assertIsNotNone(lock)
+
+        self.get_success(lock.__aenter__())
+
+        # Wait for ages with the lock, we should not be able to get the lock.
+        self.reactor.advance(5 * _LOCK_TIMEOUT_MS / 1000)
+
+        lock2 = self.get_success(self.store.try_acquire_lock("name", "key"))
+        self.assertIsNone(lock2)
+
+        self.get_success(lock.__aexit__(None, None, None))
+
+    def test_timeout_lock(self):
+        """Test that we time out locks if they're not updated for ages"""
+
+        lock = self.get_success(self.store.try_acquire_lock("name", "key"))
+        self.assertIsNotNone(lock)
+
+        self.get_success(lock.__aenter__())
+
+        # We simulate the process getting stuck by cancelling the looping call
+        # that keeps the lock active.
+        lock._looping_call.stop()
+
+        # Wait for the lock to timeout.
+        self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000)
+
+        lock2 = self.get_success(self.store.try_acquire_lock("name", "key"))
+        self.assertIsNotNone(lock2)
+
+        self.assertFalse(self.get_success(lock.is_still_valid()))
+
+    def test_drop(self):
+        """Test that dropping the context manager means we stop renewing the lock"""
+
+        lock = self.get_success(self.store.try_acquire_lock("name", "key"))
+        self.assertIsNotNone(lock)
+
+        del lock
+
+        # Wait for the lock to timeout.
+        self.reactor.advance(2 * _LOCK_TIMEOUT_MS / 1000)
+
+        lock2 = self.get_success(self.store.try_acquire_lock("name", "key"))
+        self.assertIsNotNone(lock2)
+
+    def test_shutdown(self):
+        """Test that shutting down Synapse releases the locks"""
+        # Acquire two locks
+        lock = self.get_success(self.store.try_acquire_lock("name", "key1"))
+        self.assertIsNotNone(lock)
+        lock2 = self.get_success(self.store.try_acquire_lock("name", "key2"))
+        self.assertIsNotNone(lock2)
+
+        # Now call the shutdown code
+        self.get_success(self.store._on_shutdown())
+
+        self.assertEqual(self.store._live_tokens, {})
diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py
index 1ac4ebc61d..200b9198f9 100644
--- a/tests/storage/test__base.py
+++ b/tests/storage/test__base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 #
@@ -14,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import secrets
 
 from tests import unittest
 
@@ -22,7 +22,7 @@ class UpsertManyTests(unittest.HomeserverTestCase):
     def prepare(self, reactor, clock, hs):
         self.storage = hs.get_datastore()
 
-        self.table_name = "table_" + hs.get_secrets().token_hex(6)
+        self.table_name = "table_" + secrets.token_hex(6)
         self.get_success(
             self.storage.db_pool.runInteraction(
                 "create",
diff --git a/tests/storage/test_account_data.py b/tests/storage/test_account_data.py
index 38444e48e2..01af49a16b 100644
--- a/tests/storage/test_account_data.py
+++ b/tests/storage/test_account_data.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index 1ce29af5fd..666bffe257 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,8 +14,7 @@
 import json
 import os
 import tempfile
-
-from mock import Mock
+from unittest.mock import Mock
 
 import yaml
 
diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py
index 1b4fae0bb5..0da42b5ac5 100644
--- a/tests/storage/test_background_update.py
+++ b/tests/storage/test_background_update.py
@@ -1,4 +1,4 @@
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.storage.background_updates import BackgroundUpdater
 
@@ -7,9 +7,7 @@
 
 class BackgroundUpdateTestCase(unittest.HomeserverTestCase):
     def prepare(self, reactor, clock, homeserver):
-        self.updates = (
-            self.hs.get_datastore().db_pool.updates
-        )  # type: BackgroundUpdater
+        self.updates: BackgroundUpdater = self.hs.get_datastore().db_pool.updates
         # the base test class should have run the real bg updates for us
         self.assertTrue(
             self.get_success(self.updates.has_completed_background_updates())
diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py
index eac7e4dcd2..ddad44bd6c 100644
--- a/tests/storage/test_base.py
+++ b/tests/storage/test_base.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,8 +14,7 @@
 
 
 from collections import OrderedDict
-
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
@@ -29,7 +27,7 @@
 
 
 class SQLBaseStoreTestCase(unittest.TestCase):
-    """ Test the "simple" SQL generating methods in SQLBaseStore. """
+    """Test the "simple" SQL generating methods in SQLBaseStore."""
 
     def setUp(self):
         self.db_pool = Mock(spec=["runInteraction"])
diff --git a/tests/storage/test_cleanup_extrems.py b/tests/storage/test_cleanup_extrems.py
index 7791138688..da98733ce8 100644
--- a/tests/storage/test_cleanup_extrems.py
+++ b/tests/storage/test_cleanup_extrems.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,13 +13,11 @@
 # limitations under the License.
 
 import os.path
-from unittest.mock import patch
-
-from mock import Mock
+from unittest.mock import Mock, patch
 
 import synapse.rest.admin
 from synapse.api.constants import EventTypes
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.storage import prepare_database
 from synapse.types import UserID, create_requester
 
@@ -50,10 +47,8 @@ def run_background_update(self):
         )
 
         schema_path = os.path.join(
-            prepare_database.dir_path,
-            "databases",
+            prepare_database.schema_path,
             "main",
-            "schema",
             "delta",
             "54",
             "delete_forward_extremities.sql",
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index 34e6526097..1c2df54ecc 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -14,11 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 import synapse.rest.admin
 from synapse.http.site import XForwardedForRequest
-from synapse.rest.client.v1 import login
+from synapse.rest.client import login
 
 from tests import unittest
 from tests.server import make_request
@@ -390,7 +389,7 @@ def test_old_user_ips_pruned(self):
 class ClientIpAuthTestCase(unittest.HomeserverTestCase):
 
     servlets = [
-        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        synapse.rest.admin.register_servlets,
         login.register_servlets,
     ]
 
@@ -434,7 +433,7 @@ def _runtest(self, headers, expected_ip, make_request_args):
             self.reactor,
             self.site,
             "GET",
-            "/_synapse/admin/v1/users/" + self.user_id,
+            "/_synapse/admin/v2/users/" + self.user_id,
             access_token=access_token,
             custom_headers=headers1.items(),
             **make_request_args,
diff --git a/tests/storage/test_database.py b/tests/storage/test_database.py
index 5a77c84962..6fbac0ab14 100644
--- a/tests/storage/test_database.py
+++ b/tests/storage/test_database.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -36,17 +35,6 @@ def _stub_db_engine(**kwargs) -> BaseDatabaseEngine:
 
 class TupleComparisonClauseTestCase(unittest.TestCase):
     def test_native_tuple_comparison(self):
-        db_engine = _stub_db_engine(supports_tuple_comparison=True)
-        clause, args = make_tuple_comparison_clause(db_engine, [("a", 1), ("b", 2)])
+        clause, args = make_tuple_comparison_clause([("a", 1), ("b", 2)])
         self.assertEqual(clause, "(a,b) > (?,?)")
         self.assertEqual(args, [1, 2])
-
-    def test_emulated_tuple_comparison(self):
-        db_engine = _stub_db_engine(supports_tuple_comparison=False)
-        clause, args = make_tuple_comparison_clause(
-            db_engine, [("a", 1), ("b", 2), ("c", 3)]
-        )
-        self.assertEqual(
-            clause, "(a >= ? AND (a > ? OR (b >= ? AND (b > ? OR c > ?))))"
-        )
-        self.assertEqual(args, [1, 1, 2, 2, 3])
diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py
index dabc1c5f09..6790aa5242 100644
--- a/tests/storage/test_devices.py
+++ b/tests/storage/test_devices.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2016 OpenMarket Ltd
+# Copyright 2016-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,32 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer
-
 import synapse.api.errors
 
-import tests.unittest
-import tests.utils
-
-
-class DeviceStoreTestCase(tests.unittest.TestCase):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.store = None  # type: synapse.storage.DataStore
+from tests.unittest import HomeserverTestCase
 
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
 
+class DeviceStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
 
-    @defer.inlineCallbacks
     def test_store_new_device(self):
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_device("user_id", "device_id", "display_name")
         )
 
-        res = yield defer.ensureDeferred(self.store.get_device("user_id", "device_id"))
+        res = self.get_success(self.store.get_device("user_id", "device_id"))
         self.assertDictContainsSubset(
             {
                 "user_id": "user_id",
@@ -48,19 +36,18 @@ def test_store_new_device(self):
             res,
         )
 
-    @defer.inlineCallbacks
     def test_get_devices_by_user(self):
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_device("user_id", "device1", "display_name 1")
         )
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_device("user_id", "device2", "display_name 2")
         )
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_device("user_id2", "device3", "display_name 3")
         )
 
-        res = yield defer.ensureDeferred(self.store.get_devices_by_user("user_id"))
+        res = self.get_success(self.store.get_devices_by_user("user_id"))
         self.assertEqual(2, len(res.keys()))
         self.assertDictContainsSubset(
             {
@@ -79,43 +66,41 @@ def test_get_devices_by_user(self):
             res["device2"],
         )
 
-    @defer.inlineCallbacks
     def test_count_devices_by_users(self):
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_device("user_id", "device1", "display_name 1")
         )
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_device("user_id", "device2", "display_name 2")
         )
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_device("user_id2", "device3", "display_name 3")
         )
 
-        res = yield defer.ensureDeferred(self.store.count_devices_by_users())
+        res = self.get_success(self.store.count_devices_by_users())
         self.assertEqual(0, res)
 
-        res = yield defer.ensureDeferred(self.store.count_devices_by_users(["unknown"]))
+        res = self.get_success(self.store.count_devices_by_users(["unknown"]))
         self.assertEqual(0, res)
 
-        res = yield defer.ensureDeferred(self.store.count_devices_by_users(["user_id"]))
+        res = self.get_success(self.store.count_devices_by_users(["user_id"]))
         self.assertEqual(2, res)
 
-        res = yield defer.ensureDeferred(
+        res = self.get_success(
             self.store.count_devices_by_users(["user_id", "user_id2"])
         )
         self.assertEqual(3, res)
 
-    @defer.inlineCallbacks
     def test_get_device_updates_by_remote(self):
         device_ids = ["device_id1", "device_id2"]
 
         # Add two device updates with a single stream_id
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.add_device_change_to_streams("user_id", device_ids, ["somehost"])
         )
 
         # Get all device updates ever meant for this remote
-        now_stream_id, device_updates = yield defer.ensureDeferred(
+        now_stream_id, device_updates = self.get_success(
             self.store.get_device_updates_by_remote("somehost", -1, limit=100)
         )
 
@@ -131,37 +116,35 @@ def _check_devices_in_updates(self, expected_device_ids, device_updates):
         }
         self.assertEqual(received_device_ids, set(expected_device_ids))
 
-    @defer.inlineCallbacks
     def test_update_device(self):
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_device("user_id", "device_id", "display_name 1")
         )
 
-        res = yield defer.ensureDeferred(self.store.get_device("user_id", "device_id"))
+        res = self.get_success(self.store.get_device("user_id", "device_id"))
         self.assertEqual("display_name 1", res["display_name"])
 
         # do a no-op first
-        yield defer.ensureDeferred(self.store.update_device("user_id", "device_id"))
-        res = yield defer.ensureDeferred(self.store.get_device("user_id", "device_id"))
+        self.get_success(self.store.update_device("user_id", "device_id"))
+        res = self.get_success(self.store.get_device("user_id", "device_id"))
         self.assertEqual("display_name 1", res["display_name"])
 
         # do the update
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.update_device(
                 "user_id", "device_id", new_display_name="display_name 2"
             )
         )
 
         # check it worked
-        res = yield defer.ensureDeferred(self.store.get_device("user_id", "device_id"))
+        res = self.get_success(self.store.get_device("user_id", "device_id"))
         self.assertEqual("display_name 2", res["display_name"])
 
-    @defer.inlineCallbacks
     def test_update_unknown_device(self):
-        with self.assertRaises(synapse.api.errors.StoreError) as cm:
-            yield defer.ensureDeferred(
-                self.store.update_device(
-                    "user_id", "unknown_device_id", new_display_name="display_name 2"
-                )
-            )
-        self.assertEqual(404, cm.exception.code)
+        exc = self.get_failure(
+            self.store.update_device(
+                "user_id", "unknown_device_id", new_display_name="display_name 2"
+            ),
+            synapse.api.errors.StoreError,
+        )
+        self.assertEqual(404, exc.value.code)
diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py
index da93ca3980..43628ce44f 100644
--- a/tests/storage/test_directory.py
+++ b/tests/storage/test_directory.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,28 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from twisted.internet import defer
-
 from synapse.types import RoomAlias, RoomID
 
-from tests import unittest
-from tests.utils import setup_test_homeserver
+from tests.unittest import HomeserverTestCase
 
 
-class DirectoryStoreTestCase(unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield setup_test_homeserver(self.addCleanup)
-
+class DirectoryStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
 
         self.room = RoomID.from_string("!abcde:test")
         self.alias = RoomAlias.from_string("#my-room:test")
 
-    @defer.inlineCallbacks
     def test_room_to_alias(self):
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.create_room_alias_association(
                 room_alias=self.alias, room_id=self.room.to_string(), servers=["test"]
             )
@@ -42,16 +33,11 @@ def test_room_to_alias(self):
 
         self.assertEquals(
             ["#my-room:test"],
-            (
-                yield defer.ensureDeferred(
-                    self.store.get_aliases_for_room(self.room.to_string())
-                )
-            ),
+            (self.get_success(self.store.get_aliases_for_room(self.room.to_string()))),
         )
 
-    @defer.inlineCallbacks
     def test_alias_to_room(self):
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.create_room_alias_association(
                 room_alias=self.alias, room_id=self.room.to_string(), servers=["test"]
             )
@@ -59,28 +45,19 @@ def test_alias_to_room(self):
 
         self.assertObjectHasAttributes(
             {"room_id": self.room.to_string(), "servers": ["test"]},
-            (
-                yield defer.ensureDeferred(
-                    self.store.get_association_from_room_alias(self.alias)
-                )
-            ),
+            (self.get_success(self.store.get_association_from_room_alias(self.alias))),
         )
 
-    @defer.inlineCallbacks
     def test_delete_alias(self):
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.create_room_alias_association(
                 room_alias=self.alias, room_id=self.room.to_string(), servers=["test"]
             )
         )
 
-        room_id = yield defer.ensureDeferred(self.store.delete_room_alias(self.alias))
+        room_id = self.get_success(self.store.delete_room_alias(self.alias))
         self.assertEqual(self.room.to_string(), room_id)
 
         self.assertIsNone(
-            (
-                yield defer.ensureDeferred(
-                    self.store.get_association_from_room_alias(self.alias)
-                )
-            )
+            self.get_success(self.store.get_association_from_room_alias(self.alias))
         )
diff --git a/tests/storage/test_e2e_room_keys.py b/tests/storage/test_e2e_room_keys.py
index 3d7760d5d9..9b6b425425 100644
--- a/tests/storage/test_e2e_room_keys.py
+++ b/tests/storage/test_e2e_room_keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/storage/test_end_to_end_keys.py b/tests/storage/test_end_to_end_keys.py
index 3fc4bb13b6..3bf6e337f4 100644
--- a/tests/storage/test_end_to_end_keys.py
+++ b/tests/storage/test_end_to_end_keys.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2016 OpenMarket Ltd
+# Copyright 2016-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,30 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer
+from tests.unittest import HomeserverTestCase
 
-import tests.unittest
-import tests.utils
 
-
-class EndToEndKeyStoreTestCase(tests.unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
+class EndToEndKeyStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
 
-    @defer.inlineCallbacks
     def test_key_without_device_name(self):
         now = 1470174257070
         json = {"key": "value"}
 
-        yield defer.ensureDeferred(self.store.store_device("user", "device", None))
+        self.get_success(self.store.store_device("user", "device", None))
 
-        yield defer.ensureDeferred(
-            self.store.set_e2e_device_keys("user", "device", now, json)
-        )
+        self.get_success(self.store.set_e2e_device_keys("user", "device", now, json))
 
-        res = yield defer.ensureDeferred(
+        res = self.get_success(
             self.store.get_e2e_device_keys_for_cs_api((("user", "device"),))
         )
         self.assertIn("user", res)
@@ -44,38 +35,32 @@ def test_key_without_device_name(self):
         dev = res["user"]["device"]
         self.assertDictContainsSubset(json, dev)
 
-    @defer.inlineCallbacks
     def test_reupload_key(self):
         now = 1470174257070
         json = {"key": "value"}
 
-        yield defer.ensureDeferred(self.store.store_device("user", "device", None))
+        self.get_success(self.store.store_device("user", "device", None))
 
-        changed = yield defer.ensureDeferred(
+        changed = self.get_success(
             self.store.set_e2e_device_keys("user", "device", now, json)
         )
         self.assertTrue(changed)
 
         # If we try to upload the same key then we should be told nothing
         # changed
-        changed = yield defer.ensureDeferred(
+        changed = self.get_success(
             self.store.set_e2e_device_keys("user", "device", now, json)
         )
         self.assertFalse(changed)
 
-    @defer.inlineCallbacks
     def test_get_key_with_device_name(self):
         now = 1470174257070
         json = {"key": "value"}
 
-        yield defer.ensureDeferred(
-            self.store.set_e2e_device_keys("user", "device", now, json)
-        )
-        yield defer.ensureDeferred(
-            self.store.store_device("user", "device", "display_name")
-        )
+        self.get_success(self.store.set_e2e_device_keys("user", "device", now, json))
+        self.get_success(self.store.store_device("user", "device", "display_name"))
 
-        res = yield defer.ensureDeferred(
+        res = self.get_success(
             self.store.get_e2e_device_keys_for_cs_api((("user", "device"),))
         )
         self.assertIn("user", res)
@@ -85,29 +70,28 @@ def test_get_key_with_device_name(self):
             {"key": "value", "unsigned": {"device_display_name": "display_name"}}, dev
         )
 
-    @defer.inlineCallbacks
     def test_multiple_devices(self):
         now = 1470174257070
 
-        yield defer.ensureDeferred(self.store.store_device("user1", "device1", None))
-        yield defer.ensureDeferred(self.store.store_device("user1", "device2", None))
-        yield defer.ensureDeferred(self.store.store_device("user2", "device1", None))
-        yield defer.ensureDeferred(self.store.store_device("user2", "device2", None))
+        self.get_success(self.store.store_device("user1", "device1", None))
+        self.get_success(self.store.store_device("user1", "device2", None))
+        self.get_success(self.store.store_device("user2", "device1", None))
+        self.get_success(self.store.store_device("user2", "device2", None))
 
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.set_e2e_device_keys("user1", "device1", now, {"key": "json11"})
         )
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.set_e2e_device_keys("user1", "device2", now, {"key": "json12"})
         )
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.set_e2e_device_keys("user2", "device1", now, {"key": "json21"})
         )
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.set_e2e_device_keys("user2", "device2", now, {"key": "json22"})
         )
 
-        res = yield defer.ensureDeferred(
+        res = self.get_success(
             self.store.get_e2e_device_keys_for_cs_api(
                 (("user1", "device1"), ("user2", "device2"))
             )
diff --git a/tests/storage/test_event_chain.py b/tests/storage/test_event_chain.py
index 16daa66cc9..93136f0717 100644
--- a/tests/storage/test_event_chain.py
+++ b/tests/storage/test_event_chain.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
@@ -21,7 +20,7 @@
 from synapse.api.room_versions import RoomVersions
 from synapse.events import EventBase
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.storage.databases.main.events import _LinkMap
 from synapse.types import create_requester
 
diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py
index d597d712d6..c3fcf7e7b4 100644
--- a/tests/storage/test_event_federation.py
+++ b/tests/storage/test_event_federation.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
@@ -16,7 +15,9 @@
 import attr
 from parameterized import parameterized
 
+from synapse.api.room_versions import RoomVersions
 from synapse.events import _EventInternalMetadata
+from synapse.util import json_encoder
 
 import tests.unittest
 import tests.utils
@@ -505,6 +506,61 @@ def insert_event(txn):
         )
         self.assertSetEqual(difference, set())
 
+    def test_prune_inbound_federation_queue(self):
+        "Test that pruning of inbound federation queues work"
+
+        room_id = "some_room_id"
+
+        # Insert a bunch of events that all reference the previous one.
+        self.get_success(
+            self.store.db_pool.simple_insert_many(
+                table="federation_inbound_events_staging",
+                values=[
+                    {
+                        "origin": "some_origin",
+                        "room_id": room_id,
+                        "received_ts": 0,
+                        "event_id": f"$fake_event_id_{i + 1}",
+                        "event_json": json_encoder.encode(
+                            {"prev_events": [f"$fake_event_id_{i}"]}
+                        ),
+                        "internal_metadata": "{}",
+                    }
+                    for i in range(500)
+                ],
+                desc="test_prune_inbound_federation_queue",
+            )
+        )
+
+        # Calling prune once should return True, i.e. a prune happen. The second
+        # time it shouldn't.
+        pruned = self.get_success(
+            self.store.prune_staged_events_in_room(room_id, RoomVersions.V6)
+        )
+        self.assertTrue(pruned)
+
+        pruned = self.get_success(
+            self.store.prune_staged_events_in_room(room_id, RoomVersions.V6)
+        )
+        self.assertFalse(pruned)
+
+        # Assert that we only have a single event left in the queue, and that it
+        # is the last one.
+        count = self.get_success(
+            self.store.db_pool.simple_select_one_onecol(
+                table="federation_inbound_events_staging",
+                keyvalues={"room_id": room_id},
+                retcol="COALESCE(COUNT(*), 0)",
+                desc="test_prune_inbound_federation_queue",
+            )
+        )
+        self.assertEqual(count, 1)
+
+        _, event_id = self.get_success(
+            self.store.get_next_staged_event_id_for_room(room_id)
+        )
+        self.assertEqual(event_id, "$fake_event_id_500")
+
 
 @attr.s
 class FakeEvent:
diff --git a/tests/storage/test_event_metrics.py b/tests/storage/test_event_metrics.py
index 7691f2d790..088fbb247b 100644
--- a/tests/storage/test_event_metrics.py
+++ b/tests/storage/test_event_metrics.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
@@ -39,12 +38,12 @@ def test_exposed_to_prometheus(self):
             last_event = None
 
             # Make a real event chain
-            for i in range(event_count):
+            for _ in range(event_count):
                 ev = self.create_and_send_event(room_id, user, False, last_event)
                 last_event = [ev]
 
             # Sprinkle in some extremities
-            for i in range(extrems):
+            for _ in range(extrems):
                 ev = self.create_and_send_event(room_id, user, False, last_event)
 
         # Let it run for a while, then pull out the statistics from the
diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py
index 485f1ee033..1930b37eda 100644
--- a/tests/storage/test_event_push_actions.py
+++ b/tests/storage/test_event_push_actions.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2016 OpenMarket Ltd
+# Copyright 2016-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,12 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
-from twisted.internet import defer
-
-import tests.unittest
-import tests.utils
+from tests.unittest import HomeserverTestCase
 
 USER_ID = "@user:example.com"
 
@@ -30,37 +26,31 @@
 ]
 
 
-class EventPushActionsStoreTestCase(tests.unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
+class EventPushActionsStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
         self.persist_events_store = hs.get_datastores().persist_events
 
-    @defer.inlineCallbacks
     def test_get_unread_push_actions_for_user_in_range_for_http(self):
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.get_unread_push_actions_for_user_in_range_for_http(
                 USER_ID, 0, 1000, 20
             )
         )
 
-    @defer.inlineCallbacks
     def test_get_unread_push_actions_for_user_in_range_for_email(self):
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.get_unread_push_actions_for_user_in_range_for_email(
                 USER_ID, 0, 1000, 20
             )
         )
 
-    @defer.inlineCallbacks
     def test_count_aggregation(self):
         room_id = "!foo:example.com"
         user_id = "@user1235:example.com"
 
-        @defer.inlineCallbacks
         def _assert_counts(noitf_count, highlight_count):
-            counts = yield defer.ensureDeferred(
+            counts = self.get_success(
                 self.store.db_pool.runInteraction(
                     "", self.store._get_unread_counts_by_pos_txn, room_id, user_id, 0
                 )
@@ -74,7 +64,6 @@ def _assert_counts(noitf_count, highlight_count):
                 },
             )
 
-        @defer.inlineCallbacks
         def _inject_actions(stream, action):
             event = Mock()
             event.room_id = room_id
@@ -82,14 +71,14 @@ def _inject_actions(stream, action):
             event.internal_metadata.stream_ordering = stream
             event.depth = stream
 
-            yield defer.ensureDeferred(
+            self.get_success(
                 self.store.add_push_actions_to_staging(
                     event.event_id,
                     {user_id: action},
                     False,
                 )
             )
-            yield defer.ensureDeferred(
+            self.get_success(
                 self.store.db_pool.runInteraction(
                     "",
                     self.persist_events_store._set_push_actions_for_event_and_users_txn,
@@ -99,14 +88,14 @@ def _inject_actions(stream, action):
             )
 
         def _rotate(stream):
-            return defer.ensureDeferred(
+            self.get_success(
                 self.store.db_pool.runInteraction(
                     "", self.store._rotate_notifs_before_txn, stream
                 )
             )
 
         def _mark_read(stream, depth):
-            return defer.ensureDeferred(
+            self.get_success(
                 self.store.db_pool.runInteraction(
                     "",
                     self.store._remove_old_push_actions_before_txn,
@@ -116,49 +105,48 @@ def _mark_read(stream, depth):
                 )
             )
 
-        yield _assert_counts(0, 0)
-        yield _inject_actions(1, PlAIN_NOTIF)
-        yield _assert_counts(1, 0)
-        yield _rotate(2)
-        yield _assert_counts(1, 0)
+        _assert_counts(0, 0)
+        _inject_actions(1, PlAIN_NOTIF)
+        _assert_counts(1, 0)
+        _rotate(2)
+        _assert_counts(1, 0)
 
-        yield _inject_actions(3, PlAIN_NOTIF)
-        yield _assert_counts(2, 0)
-        yield _rotate(4)
-        yield _assert_counts(2, 0)
+        _inject_actions(3, PlAIN_NOTIF)
+        _assert_counts(2, 0)
+        _rotate(4)
+        _assert_counts(2, 0)
 
-        yield _inject_actions(5, PlAIN_NOTIF)
-        yield _mark_read(3, 3)
-        yield _assert_counts(1, 0)
+        _inject_actions(5, PlAIN_NOTIF)
+        _mark_read(3, 3)
+        _assert_counts(1, 0)
 
-        yield _mark_read(5, 5)
-        yield _assert_counts(0, 0)
+        _mark_read(5, 5)
+        _assert_counts(0, 0)
 
-        yield _inject_actions(6, PlAIN_NOTIF)
-        yield _rotate(7)
+        _inject_actions(6, PlAIN_NOTIF)
+        _rotate(7)
 
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.db_pool.simple_delete(
                 table="event_push_actions", keyvalues={"1": 1}, desc=""
             )
         )
 
-        yield _assert_counts(1, 0)
+        _assert_counts(1, 0)
 
-        yield _mark_read(7, 7)
-        yield _assert_counts(0, 0)
+        _mark_read(7, 7)
+        _assert_counts(0, 0)
 
-        yield _inject_actions(8, HIGHLIGHT)
-        yield _assert_counts(1, 1)
-        yield _rotate(9)
-        yield _assert_counts(1, 1)
-        yield _rotate(10)
-        yield _assert_counts(1, 1)
+        _inject_actions(8, HIGHLIGHT)
+        _assert_counts(1, 1)
+        _rotate(9)
+        _assert_counts(1, 1)
+        _rotate(10)
+        _assert_counts(1, 1)
 
-    @defer.inlineCallbacks
     def test_find_first_stream_ordering_after_ts(self):
         def add_event(so, ts):
-            return defer.ensureDeferred(
+            self.get_success(
                 self.store.db_pool.simple_insert(
                     "events",
                     {
@@ -177,24 +165,16 @@ def add_event(so, ts):
             )
 
         # start with the base case where there are no events in the table
-        r = yield defer.ensureDeferred(
-            self.store.find_first_stream_ordering_after_ts(11)
-        )
+        r = self.get_success(self.store.find_first_stream_ordering_after_ts(11))
         self.assertEqual(r, 0)
 
         # now with one event
-        yield add_event(2, 10)
-        r = yield defer.ensureDeferred(
-            self.store.find_first_stream_ordering_after_ts(9)
-        )
+        add_event(2, 10)
+        r = self.get_success(self.store.find_first_stream_ordering_after_ts(9))
         self.assertEqual(r, 2)
-        r = yield defer.ensureDeferred(
-            self.store.find_first_stream_ordering_after_ts(10)
-        )
+        r = self.get_success(self.store.find_first_stream_ordering_after_ts(10))
         self.assertEqual(r, 2)
-        r = yield defer.ensureDeferred(
-            self.store.find_first_stream_ordering_after_ts(11)
-        )
+        r = self.get_success(self.store.find_first_stream_ordering_after_ts(11))
         self.assertEqual(r, 3)
 
         # add a bunch of dummy events to the events table
@@ -205,39 +185,27 @@ def add_event(so, ts):
             (10, 130),
             (20, 140),
         ):
-            yield add_event(stream_ordering, ts)
+            add_event(stream_ordering, ts)
 
-        r = yield defer.ensureDeferred(
-            self.store.find_first_stream_ordering_after_ts(110)
-        )
+        r = self.get_success(self.store.find_first_stream_ordering_after_ts(110))
         self.assertEqual(r, 3, "First event after 110ms should be 3, was %i" % r)
 
         # 4 and 5 are both after 120: we want 4 rather than 5
-        r = yield defer.ensureDeferred(
-            self.store.find_first_stream_ordering_after_ts(120)
-        )
+        r = self.get_success(self.store.find_first_stream_ordering_after_ts(120))
         self.assertEqual(r, 4, "First event after 120ms should be 4, was %i" % r)
 
-        r = yield defer.ensureDeferred(
-            self.store.find_first_stream_ordering_after_ts(129)
-        )
+        r = self.get_success(self.store.find_first_stream_ordering_after_ts(129))
         self.assertEqual(r, 10, "First event after 129ms should be 10, was %i" % r)
 
         # check we can get the last event
-        r = yield defer.ensureDeferred(
-            self.store.find_first_stream_ordering_after_ts(140)
-        )
+        r = self.get_success(self.store.find_first_stream_ordering_after_ts(140))
         self.assertEqual(r, 20, "First event after 14ms should be 20, was %i" % r)
 
         # off the end
-        r = yield defer.ensureDeferred(
-            self.store.find_first_stream_ordering_after_ts(160)
-        )
+        r = self.get_success(self.store.find_first_stream_ordering_after_ts(160))
         self.assertEqual(r, 21)
 
         # check we can find an event at ordering zero
-        yield add_event(0, 5)
-        r = yield defer.ensureDeferred(
-            self.store.find_first_stream_ordering_after_ts(1)
-        )
+        add_event(0, 5)
+        r = self.get_success(self.store.find_first_stream_ordering_after_ts(1))
         self.assertEqual(r, 0)
diff --git a/tests/storage/test_events.py b/tests/storage/test_events.py
index ed898b8dbb..f462a8b1c7 100644
--- a/tests/storage/test_events.py
+++ b/tests/storage/test_events.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +17,7 @@
 from synapse.api.room_versions import RoomVersions
 from synapse.federation.federation_base import event_from_pdu_json
 from synapse.rest import admin
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 
 from tests.unittest import HomeserverTestCase
 
diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py
index aad6bc907e..7486078284 100644
--- a/tests/storage/test_id_generators.py
+++ b/tests/storage/test_id_generators.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import List, Optional
+
 from synapse.storage.database import DatabasePool
 from synapse.storage.engines import IncorrectDatabaseSetup
 from synapse.storage.util.id_generators import MultiWriterIdGenerator
@@ -26,7 +27,7 @@ class MultiWriterIdGeneratorTestCase(HomeserverTestCase):
 
     def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
-        self.db_pool = self.store.db_pool  # type: DatabasePool
+        self.db_pool: DatabasePool = self.store.db_pool
 
         self.get_success(self.db_pool.runInteraction("_setup_db", self._setup_db))
 
@@ -43,7 +44,7 @@ def _setup_db(self, txn):
         )
 
     def _create_id_generator(
-        self, instance_name="master", writers=["master"]
+        self, instance_name="master", writers: Optional[List[str]] = None
     ) -> MultiWriterIdGenerator:
         def _create(conn):
             return MultiWriterIdGenerator(
@@ -53,7 +54,7 @@ def _create(conn):
                 instance_name=instance_name,
                 tables=[("foobar", "instance_name", "stream_id")],
                 sequence_name="foobar_seq",
-                writers=writers,
+                writers=writers or ["master"],
             )
 
         return self.get_success_or_raise(self.db_pool.runWithConnection(_create))
@@ -459,7 +460,7 @@ class BackwardsMultiWriterIdGeneratorTestCase(HomeserverTestCase):
 
     def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
-        self.db_pool = self.store.db_pool  # type: DatabasePool
+        self.db_pool: DatabasePool = self.store.db_pool
 
         self.get_success(self.db_pool.runInteraction("_setup_db", self._setup_db))
 
@@ -476,7 +477,7 @@ def _setup_db(self, txn):
         )
 
     def _create_id_generator(
-        self, instance_name="master", writers=["master"]
+        self, instance_name="master", writers: Optional[List[str]] = None
     ) -> MultiWriterIdGenerator:
         def _create(conn):
             return MultiWriterIdGenerator(
@@ -486,7 +487,7 @@ def _create(conn):
                 instance_name=instance_name,
                 tables=[("foobar", "instance_name", "stream_id")],
                 sequence_name="foobar_seq",
-                writers=writers,
+                writers=writers or ["master"],
                 positive=False,
             )
 
@@ -585,7 +586,7 @@ class MultiTableMultiWriterIdGeneratorTestCase(HomeserverTestCase):
 
     def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
-        self.db_pool = self.store.db_pool  # type: DatabasePool
+        self.db_pool: DatabasePool = self.store.db_pool
 
         self.get_success(self.db_pool.runInteraction("_setup_db", self._setup_db))
 
@@ -612,7 +613,7 @@ def _setup_db(self, txn):
         )
 
     def _create_id_generator(
-        self, instance_name="master", writers=["master"]
+        self, instance_name="master", writers: Optional[List[str]] = None
     ) -> MultiWriterIdGenerator:
         def _create(conn):
             return MultiWriterIdGenerator(
@@ -625,7 +626,7 @@ def _create(conn):
                     ("foobar2", "instance_name", "stream_id"),
                 ],
                 sequence_name="foobar_seq",
-                writers=writers,
+                writers=writers or ["master"],
             )
 
         return self.get_success_or_raise(self.db_pool.runWithConnection(_create))
diff --git a/tests/storage/test_keys.py b/tests/storage/test_keys.py
index 95f309fbbc..a94b5fd721 100644
--- a/tests/storage/test_keys.py
+++ b/tests/storage/test_keys.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/storage/test_main.py b/tests/storage/test_main.py
index 30e46c650d..3dcf6a22e5 100644
--- a/tests/storage/test_main.py
+++ b/tests/storage/test_main.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Awesome Technologies Innovationslabor GmbH
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 5858c7fcc4..944dbc34a2 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py
index b7dde51224..acbb9a57c4 100644
--- a/tests/storage/test_profile.py
+++ b/tests/storage/test_profile.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,59 +12,46 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from twisted.internet import defer
-
 from synapse.types import UserID
 
 from tests import unittest
-from tests.utils import setup_test_homeserver
-
 
-class ProfileStoreTestCase(unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield setup_test_homeserver(self.addCleanup)
 
+class ProfileStoreTestCase(unittest.HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
 
         self.u_frank = UserID.from_string("@frank:test")
 
-    @defer.inlineCallbacks
     def test_displayname(self):
-        yield defer.ensureDeferred(self.store.create_profile(self.u_frank.localpart))
+        self.get_success(self.store.create_profile(self.u_frank.localpart))
 
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.set_profile_displayname(self.u_frank.localpart, "Frank", 1)
         )
 
         self.assertEquals(
             "Frank",
             (
-                yield defer.ensureDeferred(
+                self.get_success(
                     self.store.get_profile_displayname(self.u_frank.localpart)
                 )
             ),
         )
 
         # test set to None
-        yield defer.ensureDeferred(
-            self.store.set_profile_displayname(self.u_frank.localpart, None, 2)
+        self.get_success(
+            self.store.set_profile_displayname(self.u_frank.localpart, None, 1)
         )
 
         self.assertIsNone(
-            (
-                yield defer.ensureDeferred(
-                    self.store.get_profile_displayname(self.u_frank.localpart)
-                )
-            )
+            self.get_success(self.store.get_profile_displayname(self.u_frank.localpart))
         )
 
-    @defer.inlineCallbacks
     def test_avatar_url(self):
-        yield defer.ensureDeferred(self.store.create_profile(self.u_frank.localpart))
+        self.get_success(self.store.create_profile(self.u_frank.localpart))
 
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.set_profile_avatar_url(
                 self.u_frank.localpart, "http://my.site/here", 1
             )
@@ -74,21 +60,17 @@ def test_avatar_url(self):
         self.assertEquals(
             "http://my.site/here",
             (
-                yield defer.ensureDeferred(
+                self.get_success(
                     self.store.get_profile_avatar_url(self.u_frank.localpart)
                 )
             ),
         )
 
         # test set to None
-        yield defer.ensureDeferred(
-            self.store.set_profile_avatar_url(self.u_frank.localpart, None, 2)
+        self.get_success(
+            self.store.set_profile_avatar_url(self.u_frank.localpart, None, 1)
         )
 
         self.assertIsNone(
-            (
-                yield defer.ensureDeferred(
-                    self.store.get_profile_avatar_url(self.u_frank.localpart)
-                )
-            )
+            self.get_success(self.store.get_profile_avatar_url(self.u_frank.localpart))
         )
diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py
index 41af8c4847..22a77c3ccc 100644
--- a/tests/storage/test_purge.py
+++ b/tests/storage/test_purge.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +13,7 @@
 # limitations under the License.
 
 from synapse.api.errors import NotFoundError, SynapseError
-from synapse.rest.client.v1 import room
+from synapse.rest.client import room
 
 from tests.unittest import HomeserverTestCase
 
@@ -76,7 +75,7 @@ def test_purge_history_wont_delete_extrems(self):
         token = self.get_success(
             self.store.get_topological_token_for_event(last["event_id"])
         )
-        event = "t{}-{}".format(token.topological + 1, token.stream + 1)
+        event = f"t{token.topological + 1}-{token.stream + 1}"
 
         # Purge everything before this topological token
         f = self.get_failure(
diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index b2a0e60856..8c95a0a2fb 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,11 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import List, Optional
 
 from canonicaljson import json
 
-from twisted.internet import defer
-
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.room_versions import RoomVersions
 from synapse.types import RoomID, UserID
@@ -50,10 +47,15 @@ def prepare(self, reactor, clock, hs):
         self.depth = 1
 
     def inject_room_member(
-        self, room, user, membership, replaces_state=None, extra_content={}
+        self,
+        room,
+        user,
+        membership,
+        replaces_state=None,
+        extra_content: Optional[dict] = None,
     ):
         content = {"membership": membership}
-        content.update(extra_content)
+        content.update(extra_content or {})
         builder = self.event_builder_factory.for_room_version(
             RoomVersions.V1,
             {
@@ -230,10 +232,14 @@ def __init__(self, base_builder, event_id):
                 self._base_builder = base_builder
                 self._event_id = event_id
 
-            @defer.inlineCallbacks
-            def build(self, prev_event_ids, auth_event_ids):
-                built_event = yield defer.ensureDeferred(
-                    self._base_builder.build(prev_event_ids, auth_event_ids)
+            async def build(
+                self,
+                prev_event_ids: List[str],
+                auth_event_ids: Optional[List[str]],
+                depth: Optional[int] = None,
+            ):
+                built_event = await self._base_builder.build(
+                    prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids
                 )
 
                 built_event._event_id = self._event_id
@@ -250,6 +256,10 @@ def room_id(self):
             def type(self):
                 return self._base_builder.type
 
+            @property
+            def internal_metadata(self):
+                return self._base_builder.internal_metadata
+
         event_1, context_1 = self.get_success(
             self.event_creation_handler.create_new_client_event(
                 EventIdManglingBuilder(
diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py
index 4eb41c46e8..9748065282 100644
--- a/tests/storage/test_registration.py
+++ b/tests/storage/test_registration.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,21 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from twisted.internet import defer
-
 from synapse.api.constants import UserTypes
 from synapse.api.errors import ThreepidValidationError
 
-from tests import unittest
-from tests.utils import setup_test_homeserver
-
+from tests.unittest import HomeserverTestCase
 
-class RegistrationStoreTestCase(unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield setup_test_homeserver(self.addCleanup)
 
+class RegistrationStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
 
         self.user_id = "@my-user:test"
@@ -35,9 +27,8 @@ def setUp(self):
         self.pwhash = "{xx1}123456789"
         self.device_id = "akgjhdjklgshg"
 
-    @defer.inlineCallbacks
     def test_register(self):
-        yield defer.ensureDeferred(self.store.register_user(self.user_id, self.pwhash))
+        self.get_success(self.store.register_user(self.user_id, self.pwhash))
 
         self.assertEquals(
             {
@@ -49,93 +40,81 @@ def test_register(self):
                 "consent_version": None,
                 "consent_server_notice_sent": None,
                 "appservice_id": None,
-                "creation_ts": 1000,
+                "creation_ts": 0,
                 "user_type": None,
                 "deactivated": 0,
                 "shadow_banned": 0,
             },
-            (yield defer.ensureDeferred(self.store.get_user_by_id(self.user_id))),
+            (self.get_success(self.store.get_user_by_id(self.user_id))),
         )
 
-    @defer.inlineCallbacks
     def test_add_tokens(self):
-        yield defer.ensureDeferred(self.store.register_user(self.user_id, self.pwhash))
-        yield defer.ensureDeferred(
+        self.get_success(self.store.register_user(self.user_id, self.pwhash))
+        self.get_success(
             self.store.add_access_token_to_user(
                 self.user_id, self.tokens[1], self.device_id, valid_until_ms=None
             )
         )
 
-        result = yield defer.ensureDeferred(
-            self.store.get_user_by_access_token(self.tokens[1])
-        )
+        result = self.get_success(self.store.get_user_by_access_token(self.tokens[1]))
 
         self.assertEqual(result.user_id, self.user_id)
         self.assertEqual(result.device_id, self.device_id)
         self.assertIsNotNone(result.token_id)
 
-    @defer.inlineCallbacks
     def test_user_delete_access_tokens(self):
         # add some tokens
-        yield defer.ensureDeferred(self.store.register_user(self.user_id, self.pwhash))
-        yield defer.ensureDeferred(
+        self.get_success(self.store.register_user(self.user_id, self.pwhash))
+        self.get_success(
             self.store.add_access_token_to_user(
                 self.user_id, self.tokens[0], device_id=None, valid_until_ms=None
             )
         )
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.add_access_token_to_user(
                 self.user_id, self.tokens[1], self.device_id, valid_until_ms=None
             )
         )
 
         # now delete some
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.user_delete_access_tokens(self.user_id, device_id=self.device_id)
         )
 
         # check they were deleted
-        user = yield defer.ensureDeferred(
-            self.store.get_user_by_access_token(self.tokens[1])
-        )
+        user = self.get_success(self.store.get_user_by_access_token(self.tokens[1]))
         self.assertIsNone(user, "access token was not deleted by device_id")
 
         # check the one not associated with the device was not deleted
-        user = yield defer.ensureDeferred(
-            self.store.get_user_by_access_token(self.tokens[0])
-        )
+        user = self.get_success(self.store.get_user_by_access_token(self.tokens[0]))
         self.assertEqual(self.user_id, user.user_id)
 
         # now delete the rest
-        yield defer.ensureDeferred(self.store.user_delete_access_tokens(self.user_id))
+        self.get_success(self.store.user_delete_access_tokens(self.user_id))
 
-        user = yield defer.ensureDeferred(
-            self.store.get_user_by_access_token(self.tokens[0])
-        )
+        user = self.get_success(self.store.get_user_by_access_token(self.tokens[0]))
         self.assertIsNone(user, "access token was not deleted without device_id")
 
-    @defer.inlineCallbacks
     def test_is_support_user(self):
         TEST_USER = "@test:test"
         SUPPORT_USER = "@support:test"
 
-        res = yield defer.ensureDeferred(self.store.is_support_user(None))
+        res = self.get_success(self.store.is_support_user(None))
         self.assertFalse(res)
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.register_user(user_id=TEST_USER, password_hash=None)
         )
-        res = yield defer.ensureDeferred(self.store.is_support_user(TEST_USER))
+        res = self.get_success(self.store.is_support_user(TEST_USER))
         self.assertFalse(res)
 
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.register_user(
                 user_id=SUPPORT_USER, password_hash=None, user_type=UserTypes.SUPPORT
             )
         )
-        res = yield defer.ensureDeferred(self.store.is_support_user(SUPPORT_USER))
+        res = self.get_success(self.store.is_support_user(SUPPORT_USER))
         self.assertTrue(res)
 
-    @defer.inlineCallbacks
     def test_3pid_inhibit_invalid_validation_session_error(self):
         """Tests that enabling the configuration option to inhibit 3PID errors on
         /requestToken also inhibits validation errors caused by an unknown session ID.
@@ -143,30 +122,28 @@ def test_3pid_inhibit_invalid_validation_session_error(self):
 
         # Check that, with the config setting set to false (the default value), a
         # validation error is caused by the unknown session ID.
-        try:
-            yield defer.ensureDeferred(
-                self.store.validate_threepid_session(
-                    "fake_sid",
-                    "fake_client_secret",
-                    "fake_token",
-                    0,
-                )
-            )
-        except ThreepidValidationError as e:
-            self.assertEquals(e.msg, "Unknown session_id", e)
+        e = self.get_failure(
+            self.store.validate_threepid_session(
+                "fake_sid",
+                "fake_client_secret",
+                "fake_token",
+                0,
+            ),
+            ThreepidValidationError,
+        )
+        self.assertEquals(e.value.msg, "Unknown session_id", e)
 
         # Set the config setting to true.
         self.store._ignore_unknown_session_error = True
 
         # Check that now the validation error is caused by the token not matching.
-        try:
-            yield defer.ensureDeferred(
-                self.store.validate_threepid_session(
-                    "fake_sid",
-                    "fake_client_secret",
-                    "fake_token",
-                    0,
-                )
-            )
-        except ThreepidValidationError as e:
-            self.assertEquals(e.msg, "Validation token not found or has expired", e)
+        e = self.get_failure(
+            self.store.validate_threepid_session(
+                "fake_sid",
+                "fake_client_secret",
+                "fake_token",
+                0,
+            ),
+            ThreepidValidationError,
+        )
+        self.assertEquals(e.value.msg, "Validation token not found or has expired", e)
diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py
index bc8400f240..31ce7f6252 100644
--- a/tests/storage/test_room.py
+++ b/tests/storage/test_room.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,22 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from twisted.internet import defer
-
 from synapse.api.constants import EventTypes
 from synapse.api.room_versions import RoomVersions
 from synapse.types import RoomAlias, RoomID, UserID
 
-from tests import unittest
-from tests.utils import setup_test_homeserver
-
+from tests.unittest import HomeserverTestCase
 
-class RoomStoreTestCase(unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield setup_test_homeserver(self.addCleanup)
 
+class RoomStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
         # We can't test RoomStore on its own without the DirectoryStore, for
         # management of the 'room_aliases' table
         self.store = hs.get_datastore()
@@ -37,7 +29,7 @@ def setUp(self):
         self.alias = RoomAlias.from_string("#a-room-name:test")
         self.u_creator = UserID.from_string("@creator:test")
 
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_room(
                 self.room.to_string(),
                 room_creator_user_id=self.u_creator.to_string(),
@@ -46,7 +38,6 @@ def setUp(self):
             )
         )
 
-    @defer.inlineCallbacks
     def test_get_room(self):
         self.assertDictContainsSubset(
             {
@@ -54,16 +45,12 @@ def test_get_room(self):
                 "creator": self.u_creator.to_string(),
                 "is_public": True,
             },
-            (yield defer.ensureDeferred(self.store.get_room(self.room.to_string()))),
+            (self.get_success(self.store.get_room(self.room.to_string()))),
         )
 
-    @defer.inlineCallbacks
     def test_get_room_unknown_room(self):
-        self.assertIsNone(
-            (yield defer.ensureDeferred(self.store.get_room("!uknown:test")))
-        )
+        self.assertIsNone(self.get_success(self.store.get_room("!uknown:test")))
 
-    @defer.inlineCallbacks
     def test_get_room_with_stats(self):
         self.assertDictContainsSubset(
             {
@@ -71,29 +58,17 @@ def test_get_room_with_stats(self):
                 "creator": self.u_creator.to_string(),
                 "public": True,
             },
-            (
-                yield defer.ensureDeferred(
-                    self.store.get_room_with_stats(self.room.to_string())
-                )
-            ),
+            (self.get_success(self.store.get_room_with_stats(self.room.to_string()))),
         )
 
-    @defer.inlineCallbacks
     def test_get_room_with_stats_unknown_room(self):
         self.assertIsNone(
-            (
-                yield defer.ensureDeferred(
-                    self.store.get_room_with_stats("!uknown:test")
-                )
-            ),
+            (self.get_success(self.store.get_room_with_stats("!uknown:test"))),
         )
 
 
-class RoomEventsStoreTestCase(unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = setup_test_homeserver(self.addCleanup)
-
+class RoomEventsStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
         # Room events need the full datastore, for persist_event() and
         # get_room_state()
         self.store = hs.get_datastore()
@@ -102,7 +77,7 @@ def setUp(self):
 
         self.room = RoomID.from_string("!abcde:test")
 
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_room(
                 self.room.to_string(),
                 room_creator_user_id="@creator:text",
@@ -111,23 +86,21 @@ def setUp(self):
             )
         )
 
-    @defer.inlineCallbacks
     def inject_room_event(self, **kwargs):
-        yield defer.ensureDeferred(
+        self.get_success(
             self.storage.persistence.persist_event(
                 self.event_factory.create_event(room_id=self.room.to_string(), **kwargs)
             )
         )
 
-    @defer.inlineCallbacks
     def STALE_test_room_name(self):
         name = "A-Room-Name"
 
-        yield self.inject_room_event(
+        self.inject_room_event(
             etype=EventTypes.Name, name=name, content={"name": name}, depth=1
         )
 
-        state = yield defer.ensureDeferred(
+        state = self.get_success(
             self.store.get_current_state(room_id=self.room.to_string())
         )
 
@@ -137,15 +110,14 @@ def STALE_test_room_name(self):
             state[0],
         )
 
-    @defer.inlineCallbacks
     def STALE_test_room_topic(self):
         topic = "A place for things"
 
-        yield self.inject_room_event(
+        self.inject_room_event(
             etype=EventTypes.Topic, topic=topic, content={"topic": topic}, depth=1
         )
 
-        state = yield defer.ensureDeferred(
+        state = self.get_success(
             self.store.get_current_state(room_id=self.room.to_string())
         )
 
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index d2aed66f6d..c72dc40510 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
@@ -16,7 +15,7 @@
 
 from synapse.api.constants import Membership
 from synapse.rest.admin import register_servlets_for_client_rest_resource
-from synapse.rest.client.v1 import login, room
+from synapse.rest.client import login, room
 from synapse.types import UserID, create_requester
 
 from tests import unittest
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index 2471f1267d..8695264595 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
+# Copyright 2018-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,24 +14,18 @@
 
 import logging
 
-from twisted.internet import defer
-
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.room_versions import RoomVersions
 from synapse.storage.state import StateFilter
 from synapse.types import RoomID, UserID
 
-import tests.unittest
-import tests.utils
+from tests.unittest import HomeserverTestCase
 
 logger = logging.getLogger(__name__)
 
 
-class StateStoreTestCase(tests.unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
-
+class StateStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
         self.storage = hs.get_storage()
         self.state_datastore = self.storage.state.stores.state
@@ -44,7 +37,7 @@ def setUp(self):
 
         self.room = RoomID.from_string("!abc123:test")
 
-        yield defer.ensureDeferred(
+        self.get_success(
             self.store.store_room(
                 self.room.to_string(),
                 room_creator_user_id="@creator:text",
@@ -53,7 +46,6 @@ def setUp(self):
             )
         )
 
-    @defer.inlineCallbacks
     def inject_state_event(self, room, sender, typ, state_key, content):
         builder = self.event_builder_factory.for_room_version(
             RoomVersions.V1,
@@ -66,13 +58,11 @@ def inject_state_event(self, room, sender, typ, state_key, content):
             },
         )
 
-        event, context = yield defer.ensureDeferred(
+        event, context = self.get_success(
             self.event_creation_handler.create_new_client_event(builder)
         )
 
-        yield defer.ensureDeferred(
-            self.storage.persistence.persist_event(event, context)
-        )
+        self.get_success(self.storage.persistence.persist_event(event, context))
 
         return event
 
@@ -82,16 +72,13 @@ def assertStateMapEqual(self, s1, s2):
             self.assertEqual(s1[t].event_id, s2[t].event_id)
         self.assertEqual(len(s1), len(s2))
 
-    @defer.inlineCallbacks
     def test_get_state_groups_ids(self):
-        e1 = yield self.inject_state_event(
-            self.room, self.u_alice, EventTypes.Create, "", {}
-        )
-        e2 = yield self.inject_state_event(
+        e1 = self.inject_state_event(self.room, self.u_alice, EventTypes.Create, "", {})
+        e2 = self.inject_state_event(
             self.room, self.u_alice, EventTypes.Name, "", {"name": "test room"}
         )
 
-        state_group_map = yield defer.ensureDeferred(
+        state_group_map = self.get_success(
             self.storage.state.get_state_groups_ids(self.room, [e2.event_id])
         )
         self.assertEqual(len(state_group_map), 1)
@@ -101,16 +88,13 @@ def test_get_state_groups_ids(self):
             {(EventTypes.Create, ""): e1.event_id, (EventTypes.Name, ""): e2.event_id},
         )
 
-    @defer.inlineCallbacks
     def test_get_state_groups(self):
-        e1 = yield self.inject_state_event(
-            self.room, self.u_alice, EventTypes.Create, "", {}
-        )
-        e2 = yield self.inject_state_event(
+        e1 = self.inject_state_event(self.room, self.u_alice, EventTypes.Create, "", {})
+        e2 = self.inject_state_event(
             self.room, self.u_alice, EventTypes.Name, "", {"name": "test room"}
         )
 
-        state_group_map = yield defer.ensureDeferred(
+        state_group_map = self.get_success(
             self.storage.state.get_state_groups(self.room, [e2.event_id])
         )
         self.assertEqual(len(state_group_map), 1)
@@ -118,32 +102,29 @@ def test_get_state_groups(self):
 
         self.assertEqual({ev.event_id for ev in state_list}, {e1.event_id, e2.event_id})
 
-    @defer.inlineCallbacks
     def test_get_state_for_event(self):
 
         # this defaults to a linear DAG as each new injection defaults to whatever
         # forward extremities are currently in the DB for this room.
-        e1 = yield self.inject_state_event(
-            self.room, self.u_alice, EventTypes.Create, "", {}
-        )
-        e2 = yield self.inject_state_event(
+        e1 = self.inject_state_event(self.room, self.u_alice, EventTypes.Create, "", {})
+        e2 = self.inject_state_event(
             self.room, self.u_alice, EventTypes.Name, "", {"name": "test room"}
         )
-        e3 = yield self.inject_state_event(
+        e3 = self.inject_state_event(
             self.room,
             self.u_alice,
             EventTypes.Member,
             self.u_alice.to_string(),
             {"membership": Membership.JOIN},
         )
-        e4 = yield self.inject_state_event(
+        e4 = self.inject_state_event(
             self.room,
             self.u_bob,
             EventTypes.Member,
             self.u_bob.to_string(),
             {"membership": Membership.JOIN},
         )
-        e5 = yield self.inject_state_event(
+        e5 = self.inject_state_event(
             self.room,
             self.u_bob,
             EventTypes.Member,
@@ -152,9 +133,7 @@ def test_get_state_for_event(self):
         )
 
         # check we get the full state as of the final event
-        state = yield defer.ensureDeferred(
-            self.storage.state.get_state_for_event(e5.event_id)
-        )
+        state = self.get_success(self.storage.state.get_state_for_event(e5.event_id))
 
         self.assertIsNotNone(e4)
 
@@ -170,7 +149,7 @@ def test_get_state_for_event(self):
         )
 
         # check we can filter to the m.room.name event (with a '' state key)
-        state = yield defer.ensureDeferred(
+        state = self.get_success(
             self.storage.state.get_state_for_event(
                 e5.event_id, StateFilter.from_types([(EventTypes.Name, "")])
             )
@@ -179,7 +158,7 @@ def test_get_state_for_event(self):
         self.assertStateMapEqual({(e2.type, e2.state_key): e2}, state)
 
         # check we can filter to the m.room.name event (with a wildcard None state key)
-        state = yield defer.ensureDeferred(
+        state = self.get_success(
             self.storage.state.get_state_for_event(
                 e5.event_id, StateFilter.from_types([(EventTypes.Name, None)])
             )
@@ -188,7 +167,7 @@ def test_get_state_for_event(self):
         self.assertStateMapEqual({(e2.type, e2.state_key): e2}, state)
 
         # check we can grab the m.room.member events (with a wildcard None state key)
-        state = yield defer.ensureDeferred(
+        state = self.get_success(
             self.storage.state.get_state_for_event(
                 e5.event_id, StateFilter.from_types([(EventTypes.Member, None)])
             )
@@ -200,7 +179,7 @@ def test_get_state_for_event(self):
 
         # check we can grab a specific room member without filtering out the
         # other event types
-        state = yield defer.ensureDeferred(
+        state = self.get_success(
             self.storage.state.get_state_for_event(
                 e5.event_id,
                 state_filter=StateFilter(
@@ -220,7 +199,7 @@ def test_get_state_for_event(self):
         )
 
         # check that we can grab everything except members
-        state = yield defer.ensureDeferred(
+        state = self.get_success(
             self.storage.state.get_state_for_event(
                 e5.event_id,
                 state_filter=StateFilter(
@@ -238,17 +217,14 @@ def test_get_state_for_event(self):
         #######################################################
 
         room_id = self.room.to_string()
-        group_ids = yield defer.ensureDeferred(
+        group_ids = self.get_success(
             self.storage.state.get_state_groups_ids(room_id, [e5.event_id])
         )
         group = list(group_ids.keys())[0]
 
         # test _get_state_for_group_using_cache correctly filters out members
         # with types=[]
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_cache,
             group,
             state_filter=StateFilter(
@@ -265,10 +241,7 @@ def test_get_state_for_event(self):
             state_dict,
         )
 
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_members_cache,
             group,
             state_filter=StateFilter(
@@ -281,10 +254,7 @@ def test_get_state_for_event(self):
 
         # test _get_state_for_group_using_cache correctly filters in members
         # with wildcard types
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_cache,
             group,
             state_filter=StateFilter(
@@ -301,10 +271,7 @@ def test_get_state_for_event(self):
             state_dict,
         )
 
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_members_cache,
             group,
             state_filter=StateFilter(
@@ -324,10 +291,7 @@ def test_get_state_for_event(self):
 
         # test _get_state_for_group_using_cache correctly filters in members
         # with specific types
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_cache,
             group,
             state_filter=StateFilter(
@@ -344,10 +308,7 @@ def test_get_state_for_event(self):
             state_dict,
         )
 
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_members_cache,
             group,
             state_filter=StateFilter(
@@ -360,10 +321,7 @@ def test_get_state_for_event(self):
 
         # test _get_state_for_group_using_cache correctly filters in members
         # with specific types
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_members_cache,
             group,
             state_filter=StateFilter(
@@ -413,10 +371,7 @@ def test_get_state_for_event(self):
         # test _get_state_for_group_using_cache correctly filters out members
         # with types=[]
         room_id = self.room.to_string()
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_cache,
             group,
             state_filter=StateFilter(
@@ -428,10 +383,7 @@ def test_get_state_for_event(self):
         self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
 
         room_id = self.room.to_string()
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_members_cache,
             group,
             state_filter=StateFilter(
@@ -444,10 +396,7 @@ def test_get_state_for_event(self):
 
         # test _get_state_for_group_using_cache correctly filters in members
         # wildcard types
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_cache,
             group,
             state_filter=StateFilter(
@@ -458,10 +407,7 @@ def test_get_state_for_event(self):
         self.assertEqual(is_all, False)
         self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
 
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_members_cache,
             group,
             state_filter=StateFilter(
@@ -480,10 +426,7 @@ def test_get_state_for_event(self):
 
         # test _get_state_for_group_using_cache correctly filters in members
         # with specific types
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_cache,
             group,
             state_filter=StateFilter(
@@ -494,10 +437,7 @@ def test_get_state_for_event(self):
         self.assertEqual(is_all, False)
         self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
 
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_members_cache,
             group,
             state_filter=StateFilter(
@@ -510,10 +450,7 @@ def test_get_state_for_event(self):
 
         # test _get_state_for_group_using_cache correctly filters in members
         # with specific types
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_cache,
             group,
             state_filter=StateFilter(
@@ -524,10 +461,7 @@ def test_get_state_for_event(self):
         self.assertEqual(is_all, False)
         self.assertDictEqual({}, state_dict)
 
-        (
-            state_dict,
-            is_all,
-        ) = yield self.state_datastore._get_state_for_group_using_cache(
+        (state_dict, is_all,) = self.state_datastore._get_state_for_group_using_cache(
             self.state_datastore._state_group_members_cache,
             group,
             state_filter=StateFilter(
diff --git a/tests/storage/test_transactions.py b/tests/storage/test_transactions.py
index 8e817e2c7f..bea9091d30 100644
--- a/tests/storage/test_transactions.py
+++ b/tests/storage/test_transactions.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from synapse.storage.databases.main.transactions import DestinationRetryTimings
 from synapse.util.retryutils import MAX_RETRY_INTERVAL
 
 from tests.unittest import HomeserverTestCase
@@ -37,8 +37,11 @@ def test_get_set_transactions(self):
         d = self.store.get_destination_retry_timings("example.com")
         r = self.get_success(d)
 
-        self.assert_dict(
-            {"retry_last_ts": 50, "retry_interval": 100, "failure_ts": 1000}, r
+        self.assertEqual(
+            DestinationRetryTimings(
+                retry_last_ts=50, retry_interval=100, failure_ts=1000
+            ),
+            r,
         )
 
     def test_initial_set_transactions(self):
diff --git a/tests/storage/test_txn_limit.py b/tests/storage/test_txn_limit.py
new file mode 100644
index 0000000000..6ff3ebb137
--- /dev/null
+++ b/tests/storage/test_txn_limit.py
@@ -0,0 +1,36 @@
+# Copyright 2014-2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tests import unittest
+
+
+class SQLTransactionLimitTestCase(unittest.HomeserverTestCase):
+    """Test SQL transaction limit doesn't break transactions."""
+
+    def make_homeserver(self, reactor, clock):
+        return self.setup_test_homeserver(db_txn_limit=1000)
+
+    def test_config(self):
+        db_config = self.hs.config.get_single_database()
+        self.assertEqual(db_config.config["txn_limit"], 1000)
+
+    def test_select(self):
+        def do_select(txn):
+            txn.execute("SELECT 1")
+
+        db_pool = self.hs.get_datastores().databases[0]
+
+        # force txn limit to roll over at least once
+        for _ in range(0, 1001):
+            self.get_success_or_raise(db_pool.runInteraction("test_select", do_select))
diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py
index a6f63f4aaf..222e5d129d 100644
--- a/tests/storage/test_user_directory.py
+++ b/tests/storage/test_user_directory.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
+# Copyright 2018-2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,10 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer
-
-from tests import unittest
-from tests.utils import setup_test_homeserver
+from tests.unittest import HomeserverTestCase, override_config
 
 ALICE = "@alice:a"
 BOB = "@bob:b"
@@ -25,73 +21,52 @@
 BELA = "@somenickname:a"
 
 
-class UserDirectoryStoreTestCase(unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        self.hs = yield setup_test_homeserver(self.addCleanup)
-        self.store = self.hs.get_datastore()
+class UserDirectoryStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
+        self.store = hs.get_datastore()
 
         # alice and bob are both in !room_id. bobby is not but shares
         # a homeserver with alice.
-        yield defer.ensureDeferred(
-            self.store.update_profile_in_user_dir(ALICE, "alice", None)
-        )
-        yield defer.ensureDeferred(
-            self.store.update_profile_in_user_dir(BOB, "bob", None)
-        )
-        yield defer.ensureDeferred(
-            self.store.update_profile_in_user_dir(BOBBY, "bobby", None)
-        )
-        yield defer.ensureDeferred(
-            self.store.update_profile_in_user_dir(BELA, "Bela", None)
-        )
-        yield defer.ensureDeferred(
-            self.store.add_users_in_public_rooms("!room:id", (ALICE, BOB))
-        )
+        self.get_success(self.store.update_profile_in_user_dir(ALICE, "alice", None))
+        self.get_success(self.store.update_profile_in_user_dir(BOB, "bob", None))
+        self.get_success(self.store.update_profile_in_user_dir(BOBBY, "bobby", None))
+        self.get_success(self.store.update_profile_in_user_dir(BELA, "Bela", None))
+        self.get_success(self.store.add_users_in_public_rooms("!room:id", (ALICE, BOB)))
 
-    @defer.inlineCallbacks
     def test_search_user_dir(self):
         # normally when alice searches the directory she should just find
         # bob because bobby doesn't share a room with her.
-        r = yield defer.ensureDeferred(self.store.search_user_dir(ALICE, "bob", 10))
+        r = self.get_success(self.store.search_user_dir(ALICE, "bob", 10))
         self.assertFalse(r["limited"])
         self.assertEqual(1, len(r["results"]))
         self.assertDictEqual(
             r["results"][0], {"user_id": BOB, "display_name": "bob", "avatar_url": None}
         )
 
-    @defer.inlineCallbacks
+    @override_config({"user_directory": {"search_all_users": True}})
     def test_search_user_dir_all_users(self):
-        self.hs.config.user_directory_search_all_users = True
-        try:
-            r = yield defer.ensureDeferred(self.store.search_user_dir(ALICE, "bob", 10))
-            self.assertFalse(r["limited"])
-            self.assertEqual(2, len(r["results"]))
-            self.assertDictEqual(
-                r["results"][0],
-                {"user_id": BOB, "display_name": "bob", "avatar_url": None},
-            )
-            self.assertDictEqual(
-                r["results"][1],
-                {"user_id": BOBBY, "display_name": "bobby", "avatar_url": None},
-            )
-        finally:
-            self.hs.config.user_directory_search_all_users = False
+        r = self.get_success(self.store.search_user_dir(ALICE, "bob", 10))
+        self.assertFalse(r["limited"])
+        self.assertEqual(2, len(r["results"]))
+        self.assertDictEqual(
+            r["results"][0],
+            {"user_id": BOB, "display_name": "bob", "avatar_url": None},
+        )
+        self.assertDictEqual(
+            r["results"][1],
+            {"user_id": BOBBY, "display_name": "bobby", "avatar_url": None},
+        )
 
-    @defer.inlineCallbacks
+    @override_config({"user_directory": {"search_all_users": True}})
     def test_search_user_dir_stop_words(self):
         """Tests that a user can look up another user by searching for the start if its
         display name even if that name happens to be a common English word that would
         usually be ignored in full text searches.
         """
-        self.hs.config.user_directory_search_all_users = True
-        try:
-            r = yield defer.ensureDeferred(self.store.search_user_dir(ALICE, "be", 10))
-            self.assertFalse(r["limited"])
-            self.assertEqual(1, len(r["results"]))
-            self.assertDictEqual(
-                r["results"][0],
-                {"user_id": BELA, "display_name": "Bela", "avatar_url": None},
-            )
-        finally:
-            self.hs.config.user_directory_search_all_users = False
+        r = self.get_success(self.store.search_user_dir(ALICE, "be", 10))
+        self.assertFalse(r["limited"])
+        self.assertEqual(1, len(r["results"]))
+        self.assertDictEqual(
+            r["results"][0],
+            {"user_id": BELA, "display_name": "Bela", "avatar_url": None},
+        )
diff --git a/tests/test_distributor.py b/tests/test_distributor.py
index b57f36e6ac..f8341041ee 100644
--- a/tests/test_distributor.py
+++ b/tests/test_distributor.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -14,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock, patch
+from unittest.mock import Mock, patch
 
 from synapse.util.distributor import Distributor
 
diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py
index 3f2691ee6b..6ebd01bcbe 100644
--- a/tests/test_event_auth.py
+++ b/tests/test_event_auth.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,12 +13,13 @@
 # limitations under the License.
 
 import unittest
+from typing import Optional
 
 from synapse import event_auth
 from synapse.api.errors import AuthError
 from synapse.api.room_versions import RoomVersions
-from synapse.events import make_event_from_dict
-from synapse.types import get_domain_from_id
+from synapse.events import EventBase, make_event_from_dict
+from synapse.types import JsonDict, get_domain_from_id
 
 
 class EventAuthTestCase(unittest.TestCase):
@@ -207,13 +207,302 @@ def test_msc2209(self):
                 do_sig_check=False,
             )
 
+    def test_join_rules_public(self):
+        """
+        Test joining a public room.
+        """
+        creator = "@creator:example.com"
+        pleb = "@joiner:example.com"
+
+        auth_events = {
+            ("m.room.create", ""): _create_event(creator),
+            ("m.room.member", creator): _join_event(creator),
+            ("m.room.join_rules", ""): _join_rules_event(creator, "public"),
+        }
+
+        # Check join.
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user cannot be force-joined to a room.
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _member_event(pleb, "join", sender=creator),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # Banned should be rejected.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "ban")
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user who left can re-join.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "leave")
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user can send a join if they're in the room.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "join")
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user can accept an invite.
+        auth_events[("m.room.member", pleb)] = _member_event(
+            pleb, "invite", sender=creator
+        )
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+    def test_join_rules_invite(self):
+        """
+        Test joining an invite only room.
+        """
+        creator = "@creator:example.com"
+        pleb = "@joiner:example.com"
+
+        auth_events = {
+            ("m.room.create", ""): _create_event(creator),
+            ("m.room.member", creator): _join_event(creator),
+            ("m.room.join_rules", ""): _join_rules_event(creator, "invite"),
+        }
+
+        # A join without an invite is rejected.
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user cannot be force-joined to a room.
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _member_event(pleb, "join", sender=creator),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # Banned should be rejected.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "ban")
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user who left cannot re-join.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "leave")
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user can send a join if they're in the room.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "join")
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user can accept an invite.
+        auth_events[("m.room.member", pleb)] = _member_event(
+            pleb, "invite", sender=creator
+        )
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+    def test_join_rules_msc3083_restricted(self):
+        """
+        Test joining a restricted room from MSC3083.
+
+        This is similar to the public test, but has some additional checks on
+        signatures.
+
+        The checks which care about signatures fake them by simply adding an
+        object of the proper form, not generating valid signatures.
+        """
+        creator = "@creator:example.com"
+        pleb = "@joiner:example.com"
+
+        auth_events = {
+            ("m.room.create", ""): _create_event(creator),
+            ("m.room.member", creator): _join_event(creator),
+            ("m.room.power_levels", ""): _power_levels_event(creator, {"invite": 0}),
+            ("m.room.join_rules", ""): _join_rules_event(creator, "restricted"),
+        }
+
+        # Older room versions don't understand this join rule
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A properly formatted join event should work.
+        authorised_join_event = _join_event(
+            pleb,
+            additional_content={
+                "join_authorised_via_users_server": "@creator:example.com"
+            },
+        )
+        event_auth.check(
+            RoomVersions.V8,
+            authorised_join_event,
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A join issued by a specific user works (i.e. the power level checks
+        # are done properly).
+        pl_auth_events = auth_events.copy()
+        pl_auth_events[("m.room.power_levels", "")] = _power_levels_event(
+            creator, {"invite": 100, "users": {"@inviter:foo.test": 150}}
+        )
+        pl_auth_events[("m.room.member", "@inviter:foo.test")] = _join_event(
+            "@inviter:foo.test"
+        )
+        event_auth.check(
+            RoomVersions.V8,
+            _join_event(
+                pleb,
+                additional_content={
+                    "join_authorised_via_users_server": "@inviter:foo.test"
+                },
+            ),
+            pl_auth_events,
+            do_sig_check=False,
+        )
+
+        # A join which is missing an authorised server is rejected.
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V8,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # An join authorised by a user who is not in the room is rejected.
+        pl_auth_events = auth_events.copy()
+        pl_auth_events[("m.room.power_levels", "")] = _power_levels_event(
+            creator, {"invite": 100, "users": {"@other:example.com": 150}}
+        )
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V8,
+                _join_event(
+                    pleb,
+                    additional_content={
+                        "join_authorised_via_users_server": "@other:example.com"
+                    },
+                ),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user cannot be force-joined to a room. (This uses an event which
+        # *would* be valid, but is sent be a different user.)
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V8,
+                _member_event(
+                    pleb,
+                    "join",
+                    sender=creator,
+                    additional_content={
+                        "join_authorised_via_users_server": "@inviter:foo.test"
+                    },
+                ),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # Banned should be rejected.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "ban")
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V8,
+                authorised_join_event,
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user who left can re-join.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "leave")
+        event_auth.check(
+            RoomVersions.V8,
+            authorised_join_event,
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user can send a join if they're in the room. (This doesn't need to
+        # be authorised since the user is already joined.)
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "join")
+        event_auth.check(
+            RoomVersions.V8,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user can accept an invite. (This doesn't need to be authorised since
+        # the user was invited.)
+        auth_events[("m.room.member", pleb)] = _member_event(
+            pleb, "invite", sender=creator
+        )
+        event_auth.check(
+            RoomVersions.V8,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
 
 # helpers for making events
 
 TEST_ROOM_ID = "!test:room"
 
 
-def _create_event(user_id):
+def _create_event(user_id: str) -> EventBase:
     return make_event_from_dict(
         {
             "room_id": TEST_ROOM_ID,
@@ -225,20 +514,30 @@ def _create_event(user_id):
     )
 
 
-def _join_event(user_id):
+def _member_event(
+    user_id: str,
+    membership: str,
+    sender: Optional[str] = None,
+    additional_content: Optional[dict] = None,
+) -> EventBase:
     return make_event_from_dict(
         {
             "room_id": TEST_ROOM_ID,
             "event_id": _get_event_id(),
             "type": "m.room.member",
-            "sender": user_id,
+            "sender": sender or user_id,
             "state_key": user_id,
-            "content": {"membership": "join"},
+            "content": {"membership": membership, **(additional_content or {})},
+            "prev_events": [],
         }
     )
 
 
-def _power_levels_event(sender, content):
+def _join_event(user_id: str, additional_content: Optional[dict] = None) -> EventBase:
+    return _member_event(user_id, "join", additional_content=additional_content)
+
+
+def _power_levels_event(sender: str, content: JsonDict) -> EventBase:
     return make_event_from_dict(
         {
             "room_id": TEST_ROOM_ID,
@@ -251,7 +550,7 @@ def _power_levels_event(sender, content):
     )
 
 
-def _alias_event(sender, **kwargs):
+def _alias_event(sender: str, **kwargs) -> EventBase:
     data = {
         "room_id": TEST_ROOM_ID,
         "event_id": _get_event_id(),
@@ -264,7 +563,7 @@ def _alias_event(sender, **kwargs):
     return make_event_from_dict(data)
 
 
-def _random_state_event(sender):
+def _random_state_event(sender: str) -> EventBase:
     return make_event_from_dict(
         {
             "room_id": TEST_ROOM_ID,
@@ -277,10 +576,25 @@ def _random_state_event(sender):
     )
 
 
+def _join_rules_event(sender: str, join_rule: str) -> EventBase:
+    return make_event_from_dict(
+        {
+            "room_id": TEST_ROOM_ID,
+            "event_id": _get_event_id(),
+            "type": "m.room.join_rules",
+            "sender": sender,
+            "state_key": "",
+            "content": {
+                "join_rule": join_rule,
+            },
+        }
+    )
+
+
 event_count = 0
 
 
-def _get_event_id():
+def _get_event_id() -> str:
     global event_count
     c = event_count
     event_count += 1
diff --git a/tests/test_federation.py b/tests/test_federation.py
index fc9aab32d0..3785799f46 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.internet.defer import succeed
 
@@ -76,7 +75,7 @@ def setUp(self):
         )
 
         self.handler = self.homeserver.get_federation_handler()
-        self.handler.do_auth = lambda origin, event, context, auth_events: succeed(
+        self.handler._check_event_auth = lambda origin, event, context, state, claimed_auth_event_map, backfilled: succeed(
             context
         )
         self.client = self.homeserver.get_federation_client()
@@ -134,7 +133,7 @@ async def post_json(destination, path, data, headers=None, timeout=0):
             }
         )
 
-        with LoggingContext():
+        with LoggingContext("test-context"):
             failure = self.get_failure(
                 self.handler.on_receive_pdu(
                     "test.serv", lying_event, sent_to_us_directly=True
diff --git a/tests/test_mau.py b/tests/test_mau.py
index 75d28a42df..66111eb367 100644
--- a/tests/test_mau.py
+++ b/tests/test_mau.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,12 +14,10 @@
 
 """Tests REST events for /rooms paths."""
 
-import json
-
-from synapse.api.constants import LoginType
+from synapse.api.constants import APP_SERVICE_REGISTRATION_TYPE, LoginType
 from synapse.api.errors import Codes, HttpResponseException, SynapseError
 from synapse.appservice import ApplicationService
-from synapse.rest.client.v2_alpha import register, sync
+from synapse.rest.client import register, sync
 
 from tests import unittest
 from tests.unittest import override_config
@@ -113,7 +110,7 @@ def test_as_ignores_mau(self):
             )
         )
 
-        self.create_user("as_kermit4", token=as_token)
+        self.create_user("as_kermit4", token=as_token, appservice=True)
 
     def test_allowed_after_a_month_mau(self):
         # Create and sync so that the MAU counts get updated
@@ -232,14 +229,15 @@ def test_tracked_but_not_limited(self):
         self.reactor.advance(100)
         self.assertEqual(2, self.successResultOf(count))
 
-    def create_user(self, localpart, token=None):
-        request_data = json.dumps(
-            {
-                "username": localpart,
-                "password": "monkey",
-                "auth": {"type": LoginType.DUMMY},
-            }
-        )
+    def create_user(self, localpart, token=None, appservice=False):
+        request_data = {
+            "username": localpart,
+            "password": "monkey",
+            "auth": {"type": LoginType.DUMMY},
+        }
+
+        if appservice:
+            request_data["type"] = APP_SERVICE_REGISTRATION_TYPE
 
         channel = self.make_request(
             "POST",
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index f696fcf89e..b4574b2ffe 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
diff --git a/tests/test_phone_home.py b/tests/test_phone_home.py
index e7aed092c2..09707a74d7 100644
--- a/tests/test_phone_home.py
+++ b/tests/test_phone_home.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +13,7 @@
 # limitations under the License.
 
 import resource
-
-import mock
+from unittest import mock
 
 from synapse.app.phone_stats_home import phone_stats_home
 
diff --git a/tests/test_preview.py b/tests/test_preview.py
index ea83299918..48e792b55b 100644
--- a/tests/test_preview.py
+++ b/tests/test_preview.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -326,6 +325,19 @@ def test_meta_charset(self):
         )
         self.assertEqual(encoding, "ascii")
 
+    def test_meta_charset_underscores(self):
+        """A character encoding contains underscore."""
+        encoding = get_html_media_encoding(
+            b"""
+        <html>
+        <head><meta charset="Shift_JIS">
+        </head>
+        </html>
+        """,
+            "text/html",
+        )
+        self.assertEqual(encoding, "Shift_JIS")
+
     def test_xml_encoding(self):
         """A character encoding is found via the meta tag."""
         encoding = get_html_media_encoding(
diff --git a/tests/test_server.py b/tests/test_server.py
index 55cde7f62f..407e172e41 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -202,6 +202,8 @@ def _make_request(self, method, path):
             parse_listener_def({"type": "http", "port": 0}),
             self.resource,
             "1.0",
+            max_request_body_size=1234,
+            reactor=self.reactor,
         )
 
         # render the request and return the channel
diff --git a/tests/test_state.py b/tests/test_state.py
index 6227a3ba95..e5488df1ac 100644
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from mock import Mock
+from typing import List, Optional
+from unittest.mock import Mock
 
 from twisted.internet import defer
 
@@ -37,8 +36,8 @@ def create_event(
     state_key=None,
     depth=2,
     event_id=None,
-    prev_events=[],
-    **kwargs
+    prev_events: Optional[List[str]] = None,
+    **kwargs,
 ):
     global _next_event_id
 
@@ -58,7 +57,7 @@ def create_event(
         "sender": "@user_id:example.com",
         "room_id": "!room_id:example.com",
         "depth": depth,
-        "prev_events": prev_events,
+        "prev_events": prev_events or [],
     }
 
     if state_key is not None:
@@ -169,6 +168,7 @@ def setUp(self):
                 "get_state_handler",
                 "get_clock",
                 "get_state_resolution_handler",
+                "get_account_validity_handler",
                 "hostname",
             ]
         )
@@ -200,7 +200,7 @@ def test_branch_no_conflict(self):
 
         self.store.register_events(graph.walk())
 
-        context_store = {}  # type: dict[str, EventContext]
+        context_store: dict[str, EventContext] = {}
 
         for event in graph.walk():
             context = yield defer.ensureDeferred(
diff --git a/tests/test_terms_auth.py b/tests/test_terms_auth.py
index a743cdc3a9..67dcf567cd 100644
--- a/tests/test_terms_auth.py
+++ b/tests/test_terms_auth.py
@@ -13,12 +13,11 @@
 # limitations under the License.
 
 import json
-
-from mock import Mock
+from unittest.mock import Mock
 
 from twisted.test.proto_helpers import MemoryReactorClock
 
-from synapse.rest.client.v2_alpha.register import register_servlets
+from synapse.rest.client.register import register_servlets
 from synapse.util import Clock
 
 from tests import unittest
diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py
index b921ac52c0..f2ef1c6051 100644
--- a/tests/test_test_utils.py
+++ b/tests/test_test_utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/test_types.py b/tests/test_types.py
index 67ceea6e43..77d12fad4c 100644
--- a/tests/test_types.py
+++ b/tests/test_types.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -111,9 +110,7 @@ def testLeadingUnderscore(self):
     def testNonAscii(self):
         # this should work with either a unicode or a bytes
         self.assertEqual(map_username_to_mxid_localpart("têst"), "t=c3=aast")
-        self.assertEqual(
-            map_username_to_mxid_localpart("têst".encode("utf-8")), "t=c3=aast"
-        )
+        self.assertEqual(map_username_to_mxid_localpart("têst".encode()), "t=c3=aast")
 
 
 class StripInvalidMxidCharactersTestCase(unittest.TestCase):
diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py
index 43898d8142..be6302d170 100644
--- a/tests/test_utils/__init__.py
+++ b/tests/test_utils/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C
 #
@@ -21,8 +20,7 @@
 import warnings
 from asyncio import Future
 from typing import Any, Awaitable, Callable, TypeVar
-
-from mock import Mock
+from unittest.mock import Mock
 
 import attr
 
diff --git a/tests/test_utils/event_injection.py b/tests/test_utils/event_injection.py
index c3c4a93e1f..e9ec9e085b 100644
--- a/tests/test_utils/event_injection.py
+++ b/tests/test_utils/event_injection.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C
 #
@@ -33,7 +32,7 @@ async def inject_member_event(
     membership: str,
     target: Optional[str] = None,
     extra_content: Optional[dict] = None,
-    **kwargs
+    **kwargs,
 ) -> EventBase:
     """Inject a membership event into a room."""
     if target is None:
@@ -58,7 +57,7 @@ async def inject_event(
     hs: synapse.server.HomeServer,
     room_version: Optional[str] = None,
     prev_event_ids: Optional[List[str]] = None,
-    **kwargs
+    **kwargs,
 ) -> EventBase:
     """Inject a generic event into a room
 
@@ -83,7 +82,7 @@ async def create_event(
     hs: synapse.server.HomeServer,
     room_version: Optional[str] = None,
     prev_event_ids: Optional[List[str]] = None,
-    **kwargs
+    **kwargs,
 ) -> Tuple[EventBase, EventContext]:
     if room_version is None:
         room_version = await hs.get_datastore().get_room_version_id(kwargs["room_id"])
diff --git a/tests/test_utils/html_parsers.py b/tests/test_utils/html_parsers.py
index ad563eb3f0..e878af5f12 100644
--- a/tests/test_utils/html_parsers.py
+++ b/tests/test_utils/html_parsers.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,13 +23,13 @@ def __init__(self):
         super().__init__()
 
         # a list of links found in the doc
-        self.links = []  # type: List[str]
+        self.links: List[str] = []
 
         # the values of any hidden <input>s: map from name to value
-        self.hiddens = {}  # type: Dict[str, Optional[str]]
+        self.hiddens: Dict[str, Optional[str]] = {}
 
         # the values of any radio buttons: map from name to list of values
-        self.radios = {}  # type: Dict[str, List[Optional[str]]]
+        self.radios: Dict[str, List[Optional[str]]] = {}
 
     def handle_starttag(
         self, tag: str, attrs: Iterable[Tuple[str, Optional[str]]]
diff --git a/tests/test_utils/logging_setup.py b/tests/test_utils/logging_setup.py
index 74568b34f8..51a197a8c6 100644
--- a/tests/test_utils/logging_setup.py
+++ b/tests/test_utils/logging_setup.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 510b630114..94b19788d7 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-
-from mock import Mock
+from typing import Optional
+from unittest.mock import Mock
 
 from twisted.internet import defer
 from twisted.internet.defer import succeed
@@ -147,9 +146,11 @@ def inject_visibility(self, user_id, visibility):
         return event
 
     @defer.inlineCallbacks
-    def inject_room_member(self, user_id, membership="join", extra_content={}):
+    def inject_room_member(
+        self, user_id, membership="join", extra_content: Optional[dict] = None
+    ):
         content = {"membership": membership}
-        content.update(extra_content)
+        content.update(extra_content or {})
         builder = self.event_builder_factory.for_room_version(
             RoomVersions.V1,
             {
diff --git a/tests/unittest.py b/tests/unittest.py
index 58a4daa1ec..3eec9c4d5b 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector
 # Copyright 2019 Matrix.org Federation C.I.C
@@ -19,10 +18,10 @@
 import hmac
 import inspect
 import logging
+import secrets
 import time
 from typing import Callable, Dict, Iterable, Optional, Tuple, Type, TypeVar, Union
-
-from mock import Mock, patch
+from unittest.mock import Mock, patch
 
 from canonicaljson import json
 
@@ -135,13 +134,13 @@ def tearDown(orig):
     def assertObjectHasAttributes(self, attrs, obj):
         """Asserts that the given object has each of the attributes given, and
         that the value of each matches according to assertEquals."""
-        for (key, value) in attrs.items():
+        for key in attrs.keys():
             if not hasattr(obj, key):
                 raise AssertionError("Expected obj to have a '.%s'" % key)
             try:
                 self.assertEquals(attrs[key], getattr(obj, key))
             except AssertionError as e:
-                raise (type(e))("Assert error for '.{}':".format(key)) from e
+                raise (type(e))(f"Assert error for '.{key}':") from e
 
     def assert_dict(self, required, actual):
         """Does a partial assert of a dict.
@@ -249,6 +248,8 @@ def setUp(self):
             config=self.hs.config.server.listeners[0],
             resource=self.resource,
             server_version_string="1",
+            max_request_body_size=1234,
+            reactor=self.reactor,
         )
 
         from tests.rest.client.v1.utils import RestHelper
@@ -471,7 +472,7 @@ def setup_test_homeserver(self, *args, **kwargs):
         kwargs["config"] = config_obj
 
         async def run_bg_updates():
-            with LoggingContext("run_bg_updates", request="run_bg_updates-1"):
+            with LoggingContext("run_bg_updates"):
                 while not await stor.db_pool.updates.has_completed_background_updates():
                     await stor.db_pool.updates.do_next_background_update(1)
 
@@ -519,7 +520,7 @@ def get_success_or_raise(self, d, by=0.0):
         if not isinstance(deferred, Deferred):
             return d
 
-        results = []  # type: list
+        results: list = []
         deferred.addBoth(results.append)
 
         self.pump(by=by)
@@ -593,7 +594,15 @@ def register_user(
         user_id = channel.json_body["user_id"]
         return user_id
 
-    def login(self, username, password, device_id=None):
+    def login(
+        self,
+        username,
+        password,
+        device_id=None,
+        custom_headers: Optional[
+            Iterable[Tuple[Union[bytes, str], Union[bytes, str]]]
+        ] = None,
+    ):
         """
         Log in a user, and get an access token. Requires the Login API be
         registered.
@@ -604,7 +613,10 @@ def login(self, username, password, device_id=None):
             body["device_id"] = device_id
 
         channel = self.make_request(
-            "POST", "/_matrix/client/r0/login", json.dumps(body).encode("utf8")
+            "POST",
+            "/_matrix/client/r0/login",
+            json.dumps(body).encode("utf8"),
+            custom_headers=custom_headers,
         )
         self.assertEqual(channel.code, 200, channel.result)
 
@@ -626,7 +638,6 @@ def create_and_send_event(
             str: The new event's ID.
         """
         event_creator = self.hs.get_event_creation_handler()
-        secrets = self.hs.get_secrets()
         requester = create_requester(user)
 
         event, context = self.get_success(
diff --git a/tests/util/__init__.py b/tests/util/__init__.py
index bfebb0f644..5e83dba2ed 100644
--- a/tests/util/__init__.py
+++ b/tests/util/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/util/caches/__init__.py b/tests/util/caches/__init__.py
index 451dae3b6c..830e2dfe91 100644
--- a/tests/util/caches/__init__.py
+++ b/tests/util/caches/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/util/caches/test_cached_call.py b/tests/util/caches/test_cached_call.py
index f349b5ced0..80b97167ba 100644
--- a/tests/util/caches/test_cached_call.py
+++ b/tests/util/caches/test_cached_call.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/util/caches/test_deferred_cache.py b/tests/util/caches/test_deferred_cache.py
index c24c33ee91..54a88a8325 100644
--- a/tests/util/caches/test_deferred_cache.py
+++ b/tests/util/caches/test_deferred_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py
index afb11b9caf..39947a166b 100644
--- a/tests/util/caches/test_descriptors.py
+++ b/tests/util/caches/test_descriptors.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
@@ -15,8 +14,7 @@
 # limitations under the License.
 import logging
 from typing import Set
-
-import mock
+from unittest import mock
 
 from twisted.internet import defer, reactor
 
@@ -176,7 +174,7 @@ def fn(self, arg1):
                 return self.result
 
         obj = Cls()
-        callbacks = set()  # type: Set[str]
+        callbacks: Set[str] = set()
 
         # set off an asynchronous request
         obj.result = origin_d = defer.Deferred()
@@ -232,8 +230,7 @@ def inner_fn():
 
         @defer.inlineCallbacks
         def do_lookup():
-            with LoggingContext() as c1:
-                c1.name = "c1"
+            with LoggingContext("c1") as c1:
                 r = yield obj.fn(1)
                 self.assertEqual(current_context(), c1)
             return r
@@ -275,8 +272,7 @@ def inner_fn():
 
         @defer.inlineCallbacks
         def do_lookup():
-            with LoggingContext() as c1:
-                c1.name = "c1"
+            with LoggingContext("c1") as c1:
                 try:
                     d = obj.fn(1)
                     self.assertEqual(
@@ -626,17 +622,17 @@ def func2(self, key, cache_context):
         self.assertEquals(callcount2[0], 1)
 
         a.func2.invalidate(("foo",))
-        self.assertEquals(a.func2.cache.cache.pop.call_count, 1)
+        self.assertEquals(a.func2.cache.cache.del_multi.call_count, 1)
 
         yield a.func2("foo")
         a.func2.invalidate(("foo",))
-        self.assertEquals(a.func2.cache.cache.pop.call_count, 2)
+        self.assertEquals(a.func2.cache.cache.del_multi.call_count, 2)
 
         self.assertEquals(callcount[0], 1)
         self.assertEquals(callcount2[0], 2)
 
         a.func.invalidate(("foo",))
-        self.assertEquals(a.func2.cache.cache.pop.call_count, 3)
+        self.assertEquals(a.func2.cache.cache.del_multi.call_count, 3)
         yield a.func("foo")
 
         self.assertEquals(callcount[0], 2)
@@ -661,28 +657,29 @@ def fn(self, arg1, arg2):
 
             @descriptors.cachedList("fn", "args1")
             async def list_fn(self, args1, arg2):
-                assert current_context().request == "c1"
+                assert current_context().name == "c1"
                 # we want this to behave like an asynchronous function
                 await run_on_reactor()
-                assert current_context().request == "c1"
+                assert current_context().name == "c1"
                 return self.mock(args1, arg2)
 
-        with LoggingContext() as c1:
-            c1.request = "c1"
+        with LoggingContext("c1") as c1:
             obj = Cls()
             obj.mock.return_value = {10: "fish", 20: "chips"}
+
+            # start the lookup off
             d1 = obj.list_fn([10, 20], 2)
             self.assertEqual(current_context(), SENTINEL_CONTEXT)
             r = yield d1
             self.assertEqual(current_context(), c1)
-            obj.mock.assert_called_once_with([10, 20], 2)
+            obj.mock.assert_called_once_with((10, 20), 2)
             self.assertEqual(r, {10: "fish", 20: "chips"})
             obj.mock.reset_mock()
 
             # a call with different params should call the mock again
             obj.mock.return_value = {30: "peas"}
             r = yield obj.list_fn([20, 30], 2)
-            obj.mock.assert_called_once_with([30], 2)
+            obj.mock.assert_called_once_with((30,), 2)
             self.assertEqual(r, {20: "chips", 30: "peas"})
             obj.mock.reset_mock()
 
@@ -697,6 +694,15 @@ async def list_fn(self, args1, arg2):
             obj.mock.assert_not_called()
             self.assertEqual(r, {10: "fish", 20: "chips", 30: "peas"})
 
+            # we should also be able to use a (single-use) iterable, and should
+            # deduplicate the keys
+            obj.mock.reset_mock()
+            obj.mock.return_value = {40: "gravy"}
+            iterable = (x for x in [10, 40, 40])
+            r = yield obj.list_fn(iterable, 2)
+            obj.mock.assert_called_once_with((40,), 2)
+            self.assertEqual(r, {10: "fish", 40: "gravy"})
+
     @defer.inlineCallbacks
     def test_invalidate(self):
         """Make sure that invalidation callbacks are called."""
@@ -722,7 +728,7 @@ async def list_fn(self, args1, arg2):
         # cache miss
         obj.mock.return_value = {10: "fish", 20: "chips"}
         r1 = yield obj.list_fn([10, 20], 2, on_invalidate=invalidate0)
-        obj.mock.assert_called_once_with([10, 20], 2)
+        obj.mock.assert_called_once_with((10, 20), 2)
         self.assertEqual(r1, {10: "fish", 20: "chips"})
         obj.mock.reset_mock()
 
diff --git a/tests/util/caches/test_responsecache.py b/tests/util/caches/test_response_cache.py
similarity index 62%
rename from tests/util/caches/test_responsecache.py
rename to tests/util/caches/test_response_cache.py
index f9a187b8de..1e83ef2f33 100644
--- a/tests/util/caches/test_responsecache.py
+++ b/tests/util/caches/test_response_cache.py
@@ -11,14 +11,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from parameterized import parameterized
 
-from synapse.util.caches.response_cache import ResponseCache
+from twisted.internet import defer
+
+from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext
 
 from tests.server import get_clock
 from tests.unittest import TestCase
 
 
-class DeferredCacheTestCase(TestCase):
+class ResponseCacheTestCase(TestCase):
     """
     A TestCase class for ResponseCache.
 
@@ -48,7 +51,9 @@ def test_cache_hit(self):
 
         expected_result = "howdy"
 
-        wrap_d = cache.wrap(0, self.instant_return, expected_result)
+        wrap_d = defer.ensureDeferred(
+            cache.wrap(0, self.instant_return, expected_result)
+        )
 
         self.assertEqual(
             expected_result,
@@ -66,7 +71,9 @@ def test_cache_miss(self):
 
         expected_result = "howdy"
 
-        wrap_d = cache.wrap(0, self.instant_return, expected_result)
+        wrap_d = defer.ensureDeferred(
+            cache.wrap(0, self.instant_return, expected_result)
+        )
 
         self.assertEqual(
             expected_result,
@@ -80,7 +87,9 @@ def test_cache_expire(self):
 
         expected_result = "howdy"
 
-        wrap_d = cache.wrap(0, self.instant_return, expected_result)
+        wrap_d = defer.ensureDeferred(
+            cache.wrap(0, self.instant_return, expected_result)
+        )
 
         self.assertEqual(expected_result, self.successResultOf(wrap_d))
         self.assertEqual(
@@ -99,7 +108,10 @@ def test_cache_wait_hit(self):
 
         expected_result = "howdy"
 
-        wrap_d = cache.wrap(0, self.delayed_return, expected_result)
+        wrap_d = defer.ensureDeferred(
+            cache.wrap(0, self.delayed_return, expected_result)
+        )
+
         self.assertNoResult(wrap_d)
 
         # function wakes up, returns result
@@ -112,7 +124,9 @@ def test_cache_wait_expire(self):
 
         expected_result = "howdy"
 
-        wrap_d = cache.wrap(0, self.delayed_return, expected_result)
+        wrap_d = defer.ensureDeferred(
+            cache.wrap(0, self.delayed_return, expected_result)
+        )
         self.assertNoResult(wrap_d)
 
         # stop at 1 second to callback cache eviction callLater at that time, then another to set time at 2
@@ -129,3 +143,50 @@ def test_cache_wait_expire(self):
         self.reactor.pump((2,))
 
         self.assertIsNone(cache.get(0), "cache should not have the result now")
+
+    @parameterized.expand([(True,), (False,)])
+    def test_cache_context_nocache(self, should_cache: bool):
+        """If the callback clears the should_cache bit, the result should not be cached"""
+        cache = self.with_cache("medium_cache", ms=3000)
+
+        expected_result = "howdy"
+
+        call_count = 0
+
+        async def non_caching(o: str, cache_context: ResponseCacheContext[int]):
+            nonlocal call_count
+            call_count += 1
+            await self.clock.sleep(1)
+            cache_context.should_cache = should_cache
+            return o
+
+        wrap_d = defer.ensureDeferred(
+            cache.wrap(0, non_caching, expected_result, cache_context=True)
+        )
+        # there should be no result to start with
+        self.assertNoResult(wrap_d)
+
+        # a second call should also return a pending deferred
+        wrap2_d = defer.ensureDeferred(
+            cache.wrap(0, non_caching, expected_result, cache_context=True)
+        )
+        self.assertNoResult(wrap2_d)
+
+        # and there should have been exactly one call
+        self.assertEqual(call_count, 1)
+
+        # let the call complete
+        self.reactor.advance(1)
+
+        # both results should have completed
+        self.assertEqual(expected_result, self.successResultOf(wrap_d))
+        self.assertEqual(expected_result, self.successResultOf(wrap2_d))
+
+        if should_cache:
+            self.assertEqual(
+                expected_result,
+                self.successResultOf(cache.get(0)),
+                "cache should still have the result",
+            )
+        else:
+            self.assertIsNone(cache.get(0), "cache should not have the result")
diff --git a/tests/util/caches/test_ttlcache.py b/tests/util/caches/test_ttlcache.py
index 816795c136..fe8314057d 100644
--- a/tests/util/caches/test_ttlcache.py
+++ b/tests/util/caches/test_ttlcache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+from unittest.mock import Mock
 
 from synapse.util.caches.ttlcache import TTLCache
 
diff --git a/tests/util/test_async_utils.py b/tests/util/test_async_utils.py
index 17fd86d02d..069f875962 100644
--- a/tests/util/test_async_utils.py
+++ b/tests/util/test_async_utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/util/test_batching_queue.py b/tests/util/test_batching_queue.py
new file mode 100644
index 0000000000..07be57d72c
--- /dev/null
+++ b/tests/util/test_batching_queue.py
@@ -0,0 +1,238 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from twisted.internet import defer
+
+from synapse.logging.context import make_deferred_yieldable
+from synapse.util.batching_queue import (
+    BatchingQueue,
+    number_in_flight,
+    number_of_keys,
+    number_queued,
+)
+
+from tests.server import get_clock
+from tests.unittest import TestCase
+
+
+class BatchingQueueTestCase(TestCase):
+    def setUp(self):
+        self.clock, hs_clock = get_clock()
+
+        # We ensure that we remove any existing metrics for "test_queue".
+        try:
+            number_queued.remove("test_queue")
+            number_of_keys.remove("test_queue")
+            number_in_flight.remove("test_queue")
+        except KeyError:
+            pass
+
+        self._pending_calls = []
+        self.queue = BatchingQueue("test_queue", hs_clock, self._process_queue)
+
+    async def _process_queue(self, values):
+        d = defer.Deferred()
+        self._pending_calls.append((values, d))
+        return await make_deferred_yieldable(d)
+
+    def _get_sample_with_name(self, metric, name) -> int:
+        """For a prometheus metric get the value of the sample that has a
+        matching "name" label.
+        """
+        for sample in metric.collect()[0].samples:
+            if sample.labels.get("name") == name:
+                return sample.value
+
+        self.fail("Found no matching sample")
+
+    def _assert_metrics(self, queued, keys, in_flight):
+        """Assert that the metrics are correct"""
+
+        sample = self._get_sample_with_name(number_queued, self.queue._name)
+        self.assertEqual(
+            sample,
+            queued,
+            "number_queued",
+        )
+
+        sample = self._get_sample_with_name(number_of_keys, self.queue._name)
+        self.assertEqual(sample, keys, "number_of_keys")
+
+        sample = self._get_sample_with_name(number_in_flight, self.queue._name)
+        self.assertEqual(
+            sample,
+            in_flight,
+            "number_in_flight",
+        )
+
+    def test_simple(self):
+        """Tests the basic case of calling `add_to_queue` once and having
+        `_process_queue` return.
+        """
+
+        self.assertFalse(self._pending_calls)
+
+        queue_d = defer.ensureDeferred(self.queue.add_to_queue("foo"))
+
+        self._assert_metrics(queued=1, keys=1, in_flight=1)
+
+        # The queue should wait a reactor tick before calling the processing
+        # function.
+        self.assertFalse(self._pending_calls)
+        self.assertFalse(queue_d.called)
+
+        # We should see a call to `_process_queue` after a reactor tick.
+        self.clock.pump([0])
+
+        self.assertEqual(len(self._pending_calls), 1)
+        self.assertEqual(self._pending_calls[0][0], ["foo"])
+        self.assertFalse(queue_d.called)
+        self._assert_metrics(queued=0, keys=0, in_flight=1)
+
+        # Return value of the `_process_queue` should be propagated back.
+        self._pending_calls.pop()[1].callback("bar")
+
+        self.assertEqual(self.successResultOf(queue_d), "bar")
+
+        self._assert_metrics(queued=0, keys=0, in_flight=0)
+
+    def test_batching(self):
+        """Test that multiple calls at the same time get batched up into one
+        call to `_process_queue`.
+        """
+
+        self.assertFalse(self._pending_calls)
+
+        queue_d1 = defer.ensureDeferred(self.queue.add_to_queue("foo1"))
+        queue_d2 = defer.ensureDeferred(self.queue.add_to_queue("foo2"))
+
+        self._assert_metrics(queued=2, keys=1, in_flight=2)
+
+        self.clock.pump([0])
+
+        # We should see only *one* call to `_process_queue`
+        self.assertEqual(len(self._pending_calls), 1)
+        self.assertEqual(self._pending_calls[0][0], ["foo1", "foo2"])
+        self.assertFalse(queue_d1.called)
+        self.assertFalse(queue_d2.called)
+        self._assert_metrics(queued=0, keys=0, in_flight=2)
+
+        # Return value of the `_process_queue` should be propagated back to both.
+        self._pending_calls.pop()[1].callback("bar")
+
+        self.assertEqual(self.successResultOf(queue_d1), "bar")
+        self.assertEqual(self.successResultOf(queue_d2), "bar")
+        self._assert_metrics(queued=0, keys=0, in_flight=0)
+
+    def test_queuing(self):
+        """Test that we queue up requests while a `_process_queue` is being
+        called.
+        """
+
+        self.assertFalse(self._pending_calls)
+
+        queue_d1 = defer.ensureDeferred(self.queue.add_to_queue("foo1"))
+        self.clock.pump([0])
+
+        self.assertEqual(len(self._pending_calls), 1)
+
+        # We queue up work after the process function has been called, testing
+        # that they get correctly queued up.
+        queue_d2 = defer.ensureDeferred(self.queue.add_to_queue("foo2"))
+        queue_d3 = defer.ensureDeferred(self.queue.add_to_queue("foo3"))
+
+        # We should see only *one* call to `_process_queue`
+        self.assertEqual(len(self._pending_calls), 1)
+        self.assertEqual(self._pending_calls[0][0], ["foo1"])
+        self.assertFalse(queue_d1.called)
+        self.assertFalse(queue_d2.called)
+        self.assertFalse(queue_d3.called)
+        self._assert_metrics(queued=2, keys=1, in_flight=3)
+
+        # Return value of the `_process_queue` should be propagated back to the
+        # first.
+        self._pending_calls.pop()[1].callback("bar1")
+
+        self.assertEqual(self.successResultOf(queue_d1), "bar1")
+        self.assertFalse(queue_d2.called)
+        self.assertFalse(queue_d3.called)
+        self._assert_metrics(queued=2, keys=1, in_flight=2)
+
+        # We should now see a second call to `_process_queue`
+        self.clock.pump([0])
+        self.assertEqual(len(self._pending_calls), 1)
+        self.assertEqual(self._pending_calls[0][0], ["foo2", "foo3"])
+        self.assertFalse(queue_d2.called)
+        self.assertFalse(queue_d3.called)
+        self._assert_metrics(queued=0, keys=0, in_flight=2)
+
+        # Return value of the `_process_queue` should be propagated back to the
+        # second.
+        self._pending_calls.pop()[1].callback("bar2")
+
+        self.assertEqual(self.successResultOf(queue_d2), "bar2")
+        self.assertEqual(self.successResultOf(queue_d3), "bar2")
+        self._assert_metrics(queued=0, keys=0, in_flight=0)
+
+    def test_different_keys(self):
+        """Test that calls to different keys get processed in parallel."""
+
+        self.assertFalse(self._pending_calls)
+
+        queue_d1 = defer.ensureDeferred(self.queue.add_to_queue("foo1", key=1))
+        self.clock.pump([0])
+        queue_d2 = defer.ensureDeferred(self.queue.add_to_queue("foo2", key=2))
+        self.clock.pump([0])
+
+        # We queue up another item with key=2 to check that we will keep taking
+        # things off the queue.
+        queue_d3 = defer.ensureDeferred(self.queue.add_to_queue("foo3", key=2))
+
+        # We should see two calls to `_process_queue`
+        self.assertEqual(len(self._pending_calls), 2)
+        self.assertEqual(self._pending_calls[0][0], ["foo1"])
+        self.assertEqual(self._pending_calls[1][0], ["foo2"])
+        self.assertFalse(queue_d1.called)
+        self.assertFalse(queue_d2.called)
+        self.assertFalse(queue_d3.called)
+        self._assert_metrics(queued=1, keys=1, in_flight=3)
+
+        # Return value of the `_process_queue` should be propagated back to the
+        # first.
+        self._pending_calls.pop(0)[1].callback("bar1")
+
+        self.assertEqual(self.successResultOf(queue_d1), "bar1")
+        self.assertFalse(queue_d2.called)
+        self.assertFalse(queue_d3.called)
+        self._assert_metrics(queued=1, keys=1, in_flight=2)
+
+        # Return value of the `_process_queue` should be propagated back to the
+        # second.
+        self._pending_calls.pop()[1].callback("bar2")
+
+        self.assertEqual(self.successResultOf(queue_d2), "bar2")
+        self.assertFalse(queue_d3.called)
+
+        # We should now see a call `_pending_calls` for `foo3`
+        self.clock.pump([0])
+        self.assertEqual(len(self._pending_calls), 1)
+        self.assertEqual(self._pending_calls[0][0], ["foo3"])
+        self.assertFalse(queue_d3.called)
+        self._assert_metrics(queued=0, keys=0, in_flight=1)
+
+        # Return value of the `_process_queue` should be propagated back to the
+        # third deferred.
+        self._pending_calls.pop()[1].callback("bar4")
+
+        self.assertEqual(self.successResultOf(queue_d3), "bar4")
+        self._assert_metrics(queued=0, keys=0, in_flight=0)
diff --git a/tests/util/test_dict_cache.py b/tests/util/test_dict_cache.py
index 2f41333f4c..bee66dee43 100644
--- a/tests/util/test_dict_cache.py
+++ b/tests/util/test_dict_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py
index 49ffeebd0e..e6e13ba06c 100644
--- a/tests/util/test_expiring_cache.py
+++ b/tests/util/test_expiring_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/util/test_file_consumer.py b/tests/util/test_file_consumer.py
index 2012263184..3bb4695405 100644
--- a/tests/util/test_file_consumer.py
+++ b/tests/util/test_file_consumer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,8 +15,7 @@
 
 import threading
 from io import StringIO
-
-from mock import NonCallableMock
+from unittest.mock import NonCallableMock
 
 from twisted.internet import defer, reactor
 
diff --git a/tests/util/test_glob_to_regex.py b/tests/util/test_glob_to_regex.py
new file mode 100644
index 0000000000..220accb92b
--- /dev/null
+++ b/tests/util/test_glob_to_regex.py
@@ -0,0 +1,59 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from synapse.util import glob_to_regex
+
+from tests.unittest import TestCase
+
+
+class GlobToRegexTestCase(TestCase):
+    def test_literal_match(self):
+        """patterns without wildcards should match"""
+        pat = glob_to_regex("foobaz")
+        self.assertTrue(
+            pat.match("FoobaZ"), "patterns should match and be case-insensitive"
+        )
+        self.assertFalse(
+            pat.match("x foobaz"), "pattern should not match at word boundaries"
+        )
+
+    def test_wildcard_match(self):
+        pat = glob_to_regex("f?o*baz")
+
+        self.assertTrue(
+            pat.match("FoobarbaZ"),
+            "* should match string and pattern should be case-insensitive",
+        )
+        self.assertTrue(pat.match("foobaz"), "* should match 0 characters")
+        self.assertFalse(pat.match("fooxaz"), "the character after * must match")
+        self.assertFalse(pat.match("fobbaz"), "? should not match 0 characters")
+        self.assertFalse(pat.match("fiiobaz"), "? should not match 2 characters")
+
+    def test_multi_wildcard(self):
+        """patterns with multiple wildcards in a row should match"""
+        pat = glob_to_regex("**baz")
+        self.assertTrue(pat.match("agsgsbaz"), "** should match any string")
+        self.assertTrue(pat.match("baz"), "** should match the empty string")
+        self.assertEqual(pat.pattern, r"\A.{0,}baz\Z")
+
+        pat = glob_to_regex("*?baz")
+        self.assertTrue(pat.match("agsgsbaz"), "*? should match any string")
+        self.assertTrue(pat.match("abaz"), "*? should match a single char")
+        self.assertFalse(pat.match("baz"), "*? should not match the empty string")
+        self.assertEqual(pat.pattern, r"\A.{1,}baz\Z")
+
+        pat = glob_to_regex("a?*?*?baz")
+        self.assertTrue(pat.match("a g baz"), "?*?*? should match 3 chars")
+        self.assertFalse(pat.match("a..baz"), "?*?*? should not match 2 chars")
+        self.assertTrue(pat.match("a.gg.baz"), "?*?*? should match 4 chars")
+        self.assertEqual(pat.pattern, r"\Aa.{3,}baz\Z")
diff --git a/tests/util/test_itertools.py b/tests/util/test_itertools.py
index e931a7ec18..3c0ddd4f18 100644
--- a/tests/util/test_itertools.py
+++ b/tests/util/test_itertools.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Dict, List
+from typing import Dict, Iterable, List, Sequence
 
 from synapse.util.iterutils import chunk_seq, sorted_topologically
 
@@ -45,7 +44,7 @@ def test_uneven_parts(self):
         )
 
     def test_empty_input(self):
-        parts = chunk_seq([], 5)
+        parts: Iterable[Sequence] = chunk_seq([], 5)
 
         self.assertEqual(
             list(parts),
@@ -57,13 +56,13 @@ class SortTopologically(TestCase):
     def test_empty(self):
         "Test that an empty graph works correctly"
 
-        graph = {}  # type: Dict[int, List[int]]
+        graph: Dict[int, List[int]] = {}
         self.assertEqual(list(sorted_topologically([], graph)), [])
 
     def test_handle_empty_graph(self):
         "Test that a graph where a node doesn't have an entry is treated as empty"
 
-        graph = {}  # type: Dict[int, List[int]]
+        graph: Dict[int, List[int]] = {}
 
         # For disconnected nodes the output is simply sorted.
         self.assertEqual(list(sorted_topologically([1, 2], graph)), [1, 2])
@@ -71,7 +70,7 @@ def test_handle_empty_graph(self):
     def test_disconnected(self):
         "Test that a graph with no edges work"
 
-        graph = {1: [], 2: []}  # type: Dict[int, List[int]]
+        graph: Dict[int, List[int]] = {1: [], 2: []}
 
         # For disconnected nodes the output is simply sorted.
         self.assertEqual(list(sorted_topologically([1, 2], graph)), [1, 2])
@@ -79,19 +78,19 @@ def test_disconnected(self):
     def test_linear(self):
         "Test that a simple `4 -> 3 -> 2 -> 1` graph works"
 
-        graph = {1: [], 2: [1], 3: [2], 4: [3]}  # type: Dict[int, List[int]]
+        graph: Dict[int, List[int]] = {1: [], 2: [1], 3: [2], 4: [3]}
 
         self.assertEqual(list(sorted_topologically([4, 3, 2, 1], graph)), [1, 2, 3, 4])
 
     def test_subset(self):
         "Test that only sorting a subset of the graph works"
-        graph = {1: [], 2: [1], 3: [2], 4: [3]}  # type: Dict[int, List[int]]
+        graph: Dict[int, List[int]] = {1: [], 2: [1], 3: [2], 4: [3]}
 
         self.assertEqual(list(sorted_topologically([4, 3], graph)), [3, 4])
 
     def test_fork(self):
         "Test that a forked graph works"
-        graph = {1: [], 2: [1], 3: [1], 4: [2, 3]}  # type: Dict[int, List[int]]
+        graph: Dict[int, List[int]] = {1: [], 2: [1], 3: [1], 4: [2, 3]}
 
         # Valid orderings are `[1, 3, 2, 4]` or `[1, 2, 3, 4]`, but we should
         # always get the same one.
@@ -99,12 +98,12 @@ def test_fork(self):
 
     def test_duplicates(self):
         "Test that a graph with duplicate edges work"
-        graph = {1: [], 2: [1, 1], 3: [2, 2], 4: [3]}  # type: Dict[int, List[int]]
+        graph: Dict[int, List[int]] = {1: [], 2: [1, 1], 3: [2, 2], 4: [3]}
 
         self.assertEqual(list(sorted_topologically([4, 3, 2, 1], graph)), [1, 2, 3, 4])
 
     def test_multiple_paths(self):
         "Test that a graph with multiple paths between two nodes work"
-        graph = {1: [], 2: [1], 3: [2], 4: [3, 2, 1]}  # type: Dict[int, List[int]]
+        graph: Dict[int, List[int]] = {1: [], 2: [1], 3: [2], 4: [3, 2, 1]}
 
         self.assertEqual(list(sorted_topologically([4, 3, 2, 1], graph)), [1, 2, 3, 4])
diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py
index 0e52811948..c4a3917b23 100644
--- a/tests/util/test_linearizer.py
+++ b/tests/util/test_linearizer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #
diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py
index 58ee918f65..5d9c4665aa 100644
--- a/tests/util/test_logcontext.py
+++ b/tests/util/test_logcontext.py
@@ -17,11 +17,10 @@
 
 class LoggingContextTestCase(unittest.TestCase):
     def _check_test_key(self, value):
-        self.assertEquals(current_context().request, value)
+        self.assertEquals(current_context().name, value)
 
     def test_with_context(self):
-        with LoggingContext() as context_one:
-            context_one.request = "test"
+        with LoggingContext("test"):
             self._check_test_key("test")
 
     @defer.inlineCallbacks
@@ -30,15 +29,13 @@ def test_sleep(self):
 
         @defer.inlineCallbacks
         def competing_callback():
-            with LoggingContext() as competing_context:
-                competing_context.request = "competing"
+            with LoggingContext("competing"):
                 yield clock.sleep(0)
                 self._check_test_key("competing")
 
         reactor.callLater(0, competing_callback)
 
-        with LoggingContext() as context_one:
-            context_one.request = "one"
+        with LoggingContext("one"):
             yield clock.sleep(0)
             self._check_test_key("one")
 
@@ -47,9 +44,7 @@ def _test_run_in_background(self, function):
 
         callback_completed = [False]
 
-        with LoggingContext() as context_one:
-            context_one.request = "one"
-
+        with LoggingContext("one"):
             # fire off function, but don't wait on it.
             d2 = run_in_background(function)
 
@@ -133,9 +128,7 @@ def blocking_function():
 
         sentinel_context = current_context()
 
-        with LoggingContext() as context_one:
-            context_one.request = "one"
-
+        with LoggingContext("one"):
             d1 = make_deferred_yieldable(blocking_function())
             # make sure that the context was reset by make_deferred_yieldable
             self.assertIs(current_context(), sentinel_context)
@@ -149,9 +142,7 @@ def blocking_function():
     def test_make_deferred_yieldable_with_chained_deferreds(self):
         sentinel_context = current_context()
 
-        with LoggingContext() as context_one:
-            context_one.request = "one"
-
+        with LoggingContext("one"):
             d1 = make_deferred_yieldable(_chained_deferred_function())
             # make sure that the context was reset by make_deferred_yieldable
             self.assertIs(current_context(), sentinel_context)
@@ -166,9 +157,7 @@ def test_make_deferred_yieldable_on_non_deferred(self):
         """Check that make_deferred_yieldable does the right thing when its
         argument isn't actually a deferred"""
 
-        with LoggingContext() as context_one:
-            context_one.request = "one"
-
+        with LoggingContext("one"):
             d1 = make_deferred_yieldable("bum")
             self._check_test_key("one")
 
@@ -177,9 +166,9 @@ def test_make_deferred_yieldable_on_non_deferred(self):
             self._check_test_key("one")
 
     def test_nested_logging_context(self):
-        with LoggingContext(request="foo"):
+        with LoggingContext("foo"):
             nested_context = nested_logging_context(suffix="bar")
-            self.assertEqual(nested_context.request, "foo-bar")
+            self.assertEqual(nested_context.name, "foo-bar")
 
     @defer.inlineCallbacks
     def test_make_deferred_yieldable_with_await(self):
@@ -193,9 +182,7 @@ async def blocking_function():
 
         sentinel_context = current_context()
 
-        with LoggingContext() as context_one:
-            context_one.request = "one"
-
+        with LoggingContext("one"):
             d1 = make_deferred_yieldable(blocking_function())
             # make sure that the context was reset by make_deferred_yieldable
             self.assertIs(current_context(), sentinel_context)
diff --git a/tests/util/test_logformatter.py b/tests/util/test_logformatter.py
index 0fb60caacb..a2e08281e6 100644
--- a/tests/util/test_logformatter.py
+++ b/tests/util/test_logformatter.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py
index a739a6aaaf..6578f3411e 100644
--- a/tests/util/test_lrucache.py
+++ b/tests/util/test_lrucache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,9 +13,9 @@
 # limitations under the License.
 
 
-from mock import Mock
+from unittest.mock import Mock
 
-from synapse.util.caches.lrucache import LruCache
+from synapse.util.caches.lrucache import LruCache, setup_expire_lru_cache_entries
 from synapse.util.caches.treecache import TreeCache
 
 from tests import unittest
@@ -60,7 +59,7 @@ def test_pop(self):
         self.assertEquals(cache.pop("key"), None)
 
     def test_del_multi(self):
-        cache = LruCache(4, keylen=2, cache_type=TreeCache)
+        cache = LruCache(4, cache_type=TreeCache)
         cache[("animal", "cat")] = "mew"
         cache[("animal", "dog")] = "woof"
         cache[("vehicles", "car")] = "vroom"
@@ -166,7 +165,7 @@ def test_del_multi(self):
         m2 = Mock()
         m3 = Mock()
         m4 = Mock()
-        cache = LruCache(4, keylen=2, cache_type=TreeCache)
+        cache = LruCache(4, cache_type=TreeCache)
 
         cache.set(("a", "1"), "value", callbacks=[m1])
         cache.set(("a", "2"), "value", callbacks=[m2])
@@ -261,3 +260,47 @@ def test_evict(self):
         self.assertEquals(cache["key3"], [3])
         self.assertEquals(cache["key4"], [4])
         self.assertEquals(cache["key5"], [5, 6])
+
+
+class TimeEvictionTestCase(unittest.HomeserverTestCase):
+    """Test that time based eviction works correctly."""
+
+    def default_config(self):
+        config = super().default_config()
+
+        config.setdefault("caches", {})["expiry_time"] = "30m"
+
+        return config
+
+    def test_evict(self):
+        setup_expire_lru_cache_entries(self.hs)
+
+        cache = LruCache(5, clock=self.hs.get_clock())
+
+        # Check that we evict entries we haven't accessed for 30 minutes.
+        cache["key1"] = 1
+        cache["key2"] = 2
+
+        self.reactor.advance(20 * 60)
+
+        self.assertEqual(cache.get("key1"), 1)
+
+        self.reactor.advance(20 * 60)
+
+        # We have only touched `key1` in the last 30m, so we expect that to
+        # still be in the cache while `key2` should have been evicted.
+        self.assertEqual(cache.get("key1"), 1)
+        self.assertEqual(cache.get("key2"), None)
+
+        # Check that re-adding an expired key works correctly.
+        cache["key2"] = 3
+        self.assertEqual(cache.get("key2"), 3)
+
+        self.reactor.advance(20 * 60)
+
+        self.assertEqual(cache.get("key2"), 3)
+
+        self.reactor.advance(20 * 60)
+
+        self.assertEqual(cache.get("key1"), None)
+        self.assertEqual(cache.get("key2"), 3)
diff --git a/tests/util/test_ratelimitutils.py b/tests/util/test_ratelimitutils.py
index 4d1aee91d5..34aaffe859 100644
--- a/tests/util/test_ratelimitutils.py
+++ b/tests/util/test_ratelimitutils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import Optional
+
 from synapse.config.homeserver import HomeServerConfig
 from synapse.util.ratelimitutils import FederationRateLimiter
 
@@ -89,9 +90,9 @@ def _await_resolution(reactor, d):
     return (reactor.seconds() - start_time) * 1000
 
 
-def build_rc_config(settings={}):
+def build_rc_config(settings: Optional[dict] = None):
     config_dict = default_config("test")
-    config_dict.update(settings)
+    config_dict.update(settings or {})
     config = HomeServerConfig()
     config.parse_config_dict(config_dict, "", "")
     return config.rc_federation
diff --git a/tests/util/test_retryutils.py b/tests/util/test_retryutils.py
index 5f46ed0cef..9e1bebdc83 100644
--- a/tests/util/test_retryutils.py
+++ b/tests/util/test_retryutils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -52,10 +51,12 @@ def test_limiter(self):
         except AssertionError:
             pass
 
+        self.pump()
+
         new_timings = self.get_success(store.get_destination_retry_timings("test_dest"))
-        self.assertEqual(new_timings["failure_ts"], failure_ts)
-        self.assertEqual(new_timings["retry_last_ts"], failure_ts)
-        self.assertEqual(new_timings["retry_interval"], MIN_RETRY_INTERVAL)
+        self.assertEqual(new_timings.failure_ts, failure_ts)
+        self.assertEqual(new_timings.retry_last_ts, failure_ts)
+        self.assertEqual(new_timings.retry_interval, MIN_RETRY_INTERVAL)
 
         # now if we try again we should get a failure
         self.get_failure(
@@ -78,14 +79,16 @@ def test_limiter(self):
         except AssertionError:
             pass
 
+        self.pump()
+
         new_timings = self.get_success(store.get_destination_retry_timings("test_dest"))
-        self.assertEqual(new_timings["failure_ts"], failure_ts)
-        self.assertEqual(new_timings["retry_last_ts"], retry_ts)
+        self.assertEqual(new_timings.failure_ts, failure_ts)
+        self.assertEqual(new_timings.retry_last_ts, retry_ts)
         self.assertGreaterEqual(
-            new_timings["retry_interval"], MIN_RETRY_INTERVAL * RETRY_MULTIPLIER * 0.5
+            new_timings.retry_interval, MIN_RETRY_INTERVAL * RETRY_MULTIPLIER * 0.5
         )
         self.assertLessEqual(
-            new_timings["retry_interval"], MIN_RETRY_INTERVAL * RETRY_MULTIPLIER * 2.0
+            new_timings.retry_interval, MIN_RETRY_INTERVAL * RETRY_MULTIPLIER * 2.0
         )
 
         #
diff --git a/tests/util/test_rwlock.py b/tests/util/test_rwlock.py
index d3dea3b52a..a10071c70f 100644
--- a/tests/util/test_rwlock.py
+++ b/tests/util/test_rwlock.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/util/test_stringutils.py b/tests/util/test_stringutils.py
index 8491f7cc83..ad4dd7f007 100644
--- a/tests/util/test_stringutils.py
+++ b/tests/util/test_stringutils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +13,7 @@
 # limitations under the License.
 
 from synapse.api.errors import SynapseError
-from synapse.util.stringutils import assert_valid_client_secret
+from synapse.util.stringutils import assert_valid_client_secret, base62_encode
 
 from .. import unittest
 
@@ -46,3 +45,9 @@ def test_client_secret_regex(self):
         for client_secret in bad:
             with self.assertRaises(SynapseError):
                 assert_valid_client_secret(client_secret)
+
+    def test_base62_encode(self):
+        self.assertEqual("0", base62_encode(0))
+        self.assertEqual("10", base62_encode(62))
+        self.assertEqual("1c", base62_encode(100))
+        self.assertEqual("001c", base62_encode(100, minwidth=4))
diff --git a/tests/util/test_threepids.py b/tests/util/test_threepids.py
index 5513724d87..d957b953bb 100644
--- a/tests/util/test_threepids.py
+++ b/tests/util/test_threepids.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 Dirk Klimpel
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/util/test_treecache.py b/tests/util/test_treecache.py
index a5f2261208..6066372053 100644
--- a/tests/util/test_treecache.py
+++ b/tests/util/test_treecache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +13,7 @@
 # limitations under the License.
 
 
-from synapse.util.caches.treecache import TreeCache
+from synapse.util.caches.treecache import TreeCache, iterate_tree_cache_entry
 
 from .. import unittest
 
@@ -65,12 +64,14 @@ def test_pop_mixedlevel(self):
         cache[("a", "b")] = "AB"
         cache[("b", "a")] = "BA"
         self.assertEquals(cache.get(("a", "a")), "AA")
-        cache.pop(("a",))
+        popped = cache.pop(("a",))
         self.assertEquals(cache.get(("a", "a")), None)
         self.assertEquals(cache.get(("a", "b")), None)
         self.assertEquals(cache.get(("b", "a")), "BA")
         self.assertEquals(len(cache), 1)
 
+        self.assertEquals({"AA", "AB"}, set(iterate_tree_cache_entry(popped)))
+
     def test_clear(self):
         cache = TreeCache()
         cache[("a",)] = "A"
diff --git a/tests/util/test_wheel_timer.py b/tests/util/test_wheel_timer.py
index 03201a4d9b..0d5039de04 100644
--- a/tests/util/test_wheel_timer.py
+++ b/tests/util/test_wheel_timer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tests/utils.py b/tests/utils.py
index 5d299f766f..ed7aa602a3 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018-2019 New Vector Ltd
 #
@@ -21,10 +20,9 @@
 import uuid
 import warnings
 from typing import Type
+from unittest.mock import Mock, patch
 from urllib import parse as urlparse
 
-from mock import Mock, patch
-
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes
@@ -122,7 +120,6 @@ def default_config(name, parse=False):
         "enable_registration_captcha": False,
         "macaroon_secret_key": "not even a little secret",
         "trusted_third_party_id_servers": [],
-        "room_invite_state_types": [],
         "password_providers": [],
         "worker_replication_url": "",
         "worker_app": None,
@@ -198,7 +195,7 @@ def setup_test_homeserver(
     config=None,
     reactor=None,
     homeserver_to_use: Type[HomeServer] = TestHomeServer,
-    **kwargs
+    **kwargs,
 ):
     """
     Setup a homeserver suitable for running tests against.  Keyword arguments
@@ -244,6 +241,9 @@ def setup_test_homeserver(
             "args": {"database": ":memory:", "cp_min": 1, "cp_max": 1},
         }
 
+    if "db_txn_limit" in kwargs:
+        database_config["txn_limit"] = kwargs["db_txn_limit"]
+
     database = DatabaseConnectionConfig("master", database_config)
     config.database.databases = [database]
 
@@ -312,7 +312,7 @@ def cleanup():
             # database for a few more seconds due to flakiness, preventing
             # us from dropping it when the test is over. If we can't drop
             # it, warn and move on.
-            for x in range(5):
+            for _ in range(5):
                 try:
                     cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
                     db_conn.commit()
diff --git a/tox.ini b/tox.ini
index 5365939e10..d284d8860a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,8 @@
 [tox]
-envlist = packaging, py35, py36, py37, py38, py39, check_codestyle, check_isort
+envlist = packaging, py36, py37, py38, py39, check_codestyle, check_isort
+
+# we require tox>=2.3.2 for the fix to https://github.com/tox-dev/tox/issues/208
+minversion = 2.3.2
 
 [base]
 deps =
@@ -18,13 +21,11 @@ deps =
     # installed on that).
     #
     # anyway, make sure that we have a recent enough setuptools.
-    setuptools>=18.5 ; python_version >= '3.6'
-    setuptools>=18.5,<51.0.0 ; python_version < '3.6'
+    setuptools>=18.5
 
     # we also need a semi-recent version of pip, because old ones fail to
     # install the "enum34" dependency of cryptography.
-    pip>=10 ; python_version >= '3.6'
-    pip>=10,<21.0 ; python_version < '3.6'
+    pip>=10
 
 # directories/files we run the linters on.
 # if you update this list, make sure to do the same in scripts-dev/lint.sh
@@ -33,12 +34,22 @@ lint_targets =
     synapse
     tests
     scripts
+    # annoyingly, black doesn't find these so we have to list them
+    scripts/export_signing_key
+    scripts/generate_config
+    scripts/generate_log_config
+    scripts/hash_password
+    scripts/register_new_matrix_user
+    scripts/synapse_port_db
     scripts-dev
+    scripts-dev/build_debian_packages
+    scripts-dev/sign_json
+    scripts-dev/update_database
     stubs
     contrib
     synctl
     synmark
-    .buildkite
+    .ci
     docker
 
 # default settings for all tox environments
@@ -48,6 +59,7 @@ deps =
 extras =
     # install the optional dependendencies for tox environments without
     # '-noextras' in their name
+    # (this requires tox 3)
     !noextras: all
     test
 
@@ -74,8 +86,6 @@ commands =
     # we use "env" rather than putting a value in `setenv` so that it is not
     # inherited by other tox environments.
     #
-    # keep this in sync with the copy in `testenv:py3-old`.
-    #
     /usr/bin/env COVERAGE_PROCESS_START={toxinidir}/.coveragerc "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
 
 # As of twisted 16.4, trial tries to import the tests as a package (previously
@@ -121,11 +131,7 @@ commands =
     # Install Synapse itself. This won't update any libraries.
     pip install -e ".[test]"
 
-    # we have to duplicate the command from `testenv` rather than refer to it
-    # as `{[testenv]commands}`, because we run on ubuntu xenial, which has
-    # tox 2.3.1, and https://github.com/tox-dev/tox/issues/208.
-    #
-    /usr/bin/env COVERAGE_PROCESS_START={toxinidir}/.coveragerc "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
+    {[testenv]commands}
 
 [testenv:benchmark]
 deps =
@@ -137,7 +143,8 @@ commands =
     python -m synmark {posargs:}
 
 [testenv:packaging]
-skip_install=True
+skip_install = true
+usedevelop = false
 deps =
     check-manifest
 commands =
@@ -155,7 +162,8 @@ extras = lint
 commands = isort -c --df --sp setup.cfg {[base]lint_targets}
 
 [testenv:check-newsfragment]
-skip_install = True
+skip_install = true
+usedevelop = false
 deps = towncrier>=18.6.0rc1
 commands =
    python -m towncrier.check --compare-with=origin/dinsic
@@ -164,24 +172,26 @@ commands =
 commands = {toxinidir}/scripts-dev/generate_sample_config --check
 
 [testenv:combine]
-skip_install = True
+skip_install = true
+usedevelop = false
 deps =
     coverage
-    pip>=10 ; python_version >= '3.6'
-    pip>=10,<21.0 ; python_version < '3.6'
+    pip>=10
 commands=
     coverage combine
     coverage report
 
 [testenv:cov-erase]
-skip_install = True
+skip_install = true
+usedevelop = false
 deps =
     coverage
 commands=
     coverage erase
 
 [testenv:cov-html]
-skip_install = True
+skip_install = true
+usedevelop = false
 deps =
     coverage
 commands=