Merge branch 'rav/room_invite_state' into rav/create_in_room_invite

changelog.d/9610.docker

@@ -0,0 +1 @@
+Speed up Docker builds and make it nicer to test against Complement while developing (install all dependencies before copying the project).

changelog.d/9699.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.30.1 which meant the suggested `pip` incantation to install an updated `cryptography` was incorrect.

changelog.d/9703.misc

@@ -0,0 +1 @@
+Fix undetected mypy error when using Python 3.6.

changelog.d/9709.misc

@@ -0,0 +1 @@
+Fix type-checking CI on develop.

docker/Dockerfile

@@ -25,42 +25,40 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
 
 # install the OS build deps
 RUN apt-get update && apt-get install -y \
-        build-essential \
-        libffi-dev \
-        libjpeg-dev \
-        libpq-dev \
-        libssl-dev \
-        libwebp-dev \
-        libxml++2.6-dev \
-        libxslt1-dev \
-        openssl \
-        rustc \
-        zlib1g-dev \
-        && rm -rf /var/lib/apt/lists/*
-
-# Build dependencies that are not available as wheels, to speed up rebuilds
-RUN pip install --prefix="/install" --no-warn-script-location \
-        cryptography \
-        frozendict \
-        jaeger-client \
-        opentracing \
-        # Match the version constraints of Synapse
-        "prometheus_client>=0.4.0" \
-        psycopg2 \
-        pycparser \
-        pyrsistent \
-        pyyaml \
-        simplejson \
-        threadloop \
-        thrift
-
-# now install synapse and all of the python deps to /install.
-COPY synapse /synapse/synapse/
+    build-essential \
+    libffi-dev \
+    libjpeg-dev \
+    libpq-dev \
+    libssl-dev \
+    libwebp-dev \
+    libxml++2.6-dev \
+    libxslt1-dev \
+    openssl \
+    rustc \
+    zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy just what we need to pip install
 COPY scripts /synapse/scripts/
 COPY MANIFEST.in README.rst setup.py synctl /synapse/
+COPY synapse/__init__.py /synapse/synapse/__init__.py
+COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
 
+# To speed up rebuilds, install all of the dependencies before we copy over
+# the whole synapse project so that we this layer in the Docker cache can be
+# used while you develop on the source
+#
+# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
 RUN pip install --prefix="/install" --no-warn-script-location \
-        /synapse[all]
+    /synapse[all]
+
+# Copy over the rest of the project
+COPY synapse /synapse/synapse/
+
+# Install the synapse package itself and all of its children packages.
+#
+# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
+RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 ###
 ### Stage 1: runtime
@@ -69,16 +67,16 @@ RUN pip install --prefix="/install" --no-warn-script-location \
 FROM docker.io/python:${PYTHON_VERSION}-slim
 
 RUN apt-get update && apt-get install -y \
-        curl \
-        gosu \
-        libjpeg62-turbo \
-        libpq5 \
-        libwebp6 \
-        xmlsec1 \
-        libjemalloc2 \
-        libssl-dev \
-        openssl \
-        && rm -rf /var/lib/apt/lists/*
+    curl \
+    gosu \
+    libjpeg62-turbo \
+    libpq5 \
+    libwebp6 \
+    xmlsec1 \
+    libjemalloc2 \
+    libssl-dev \
+    openssl \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py
@@ -91,4 +89,4 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp
 ENTRYPOINT ["/start.py"]
 
 HEALTHCHECK --interval=1m --timeout=5s \
-        CMD curl -fSs http://localhost:8008/health || exit 1
+    CMD curl -fSs http://localhost:8008/health || exit 1

synapse/logging/opentracing.py

@@ -169,7 +169,7 @@ def set_fates(clotho, lachesis, atropos, father="Zues", mother="Themis"):
 import logging
 import re
 from functools import wraps
-from typing import TYPE_CHECKING, Dict, Optional, Type
+from typing import TYPE_CHECKING, Dict, Optional, Pattern, Type
 
 import attr
 
@@ -262,7 +262,7 @@ def report_span(self, span):
 # Block everything by default
 # A regex which matches the server_names to expose traces for.
 # None means 'block everything'.
-_homeserver_whitelist = None  # type: Optional[re.Pattern[str]]
+_homeserver_whitelist = None  # type: Optional[Pattern[str]]
 
 # Util methods
 

synapse/python_dependencies.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import itertools
 import logging
 from typing import List, Set
 
@@ -101,7 +102,7 @@
         "txacme>=0.9.2",
         # txacme depends on eliot. Eliot 1.8.0 is incompatible with
         # python 3.5.2, as per https://github.com/itamarst/eliot/issues/418
-        'eliot<1.8.0;python_version<"3.5.3"',
+        "eliot<1.8.0;python_version<'3.5.3'",
     ],
     "saml2": [
         # pysaml2 6.4.0 is incompatible with Python 3.5 (see https://github.com/IdentityPython/pysaml2/issues/749)
@@ -131,6 +132,18 @@
         ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS
 
 
+# ensure there are no double-quote characters in any of the deps (otherwise the
+# 'pip install' incantation in DependencyException will break)
+for dep in itertools.chain(
+    REQUIREMENTS,
+    *CONDITIONAL_REQUIREMENTS.values(),
+):
+    if '"' in dep:
+        raise Exception(
+            "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,)
+        )
+
+
 def list_requirements():
     return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
 
@@ -150,7 +163,7 @@ def message(self):
     @property
     def dependencies(self):
         for i in self.args[0]:
-            yield "'" + i + "'"
+            yield '"' + i + '"'
 
 
 def check_requirements(for_feature=None):

tests/replication/_base.py

@@ -44,7 +44,7 @@
 try:
     import hiredis
 except ImportError:
-    hiredis = None
+    hiredis = None  # type: ignore
 
 logger = logging.getLogger(__name__)