diff --git a/changelog.d/9610.docker b/changelog.d/9610.docker new file mode 100644 index 000000000000..056252a66963 --- /dev/null +++ b/changelog.d/9610.docker @@ -0,0 +1 @@ +Speed up Docker builds and make it nicer to test against Complement while developing (install all dependencies before copying the project). diff --git a/changelog.d/9699.bugfix b/changelog.d/9699.bugfix new file mode 100644 index 000000000000..e871825b334e --- /dev/null +++ b/changelog.d/9699.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.30.1 which meant the suggested `pip` incantation to install an updated `cryptography` was incorrect. diff --git a/changelog.d/9703.misc b/changelog.d/9703.misc new file mode 100644 index 000000000000..8dda739228bd --- /dev/null +++ b/changelog.d/9703.misc @@ -0,0 +1 @@ +Fix undetected mypy error when using Python 3.6. \ No newline at end of file diff --git a/changelog.d/9709.misc b/changelog.d/9709.misc new file mode 100644 index 000000000000..10542fdf38c1 --- /dev/null +++ b/changelog.d/9709.misc @@ -0,0 +1 @@ +Fix type-checking CI on develop. \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index a442b345984f..5b7bf02776d6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,42 +25,40 @@ LABEL org.opencontainers.image.licenses='Apache-2.0' # install the OS build deps RUN apt-get update && apt-get install -y \ - build-essential \ - libffi-dev \ - libjpeg-dev \ - libpq-dev \ - libssl-dev \ - libwebp-dev \ - libxml++2.6-dev \ - libxslt1-dev \ - openssl \ - rustc \ - zlib1g-dev \ - && rm -rf /var/lib/apt/lists/* - -# Build dependencies that are not available as wheels, to speed up rebuilds -RUN pip install --prefix="/install" --no-warn-script-location \ - cryptography \ - frozendict \ - jaeger-client \ - opentracing \ - # Match the version constraints of Synapse - "prometheus_client>=0.4.0" \ - psycopg2 \ - pycparser \ - pyrsistent \ - pyyaml \ - simplejson \ - threadloop \ - thrift - -# now install synapse and all of the python deps to /install. -COPY synapse /synapse/synapse/ + build-essential \ + libffi-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libwebp-dev \ + libxml++2.6-dev \ + libxslt1-dev \ + openssl \ + rustc \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +# Copy just what we need to pip install COPY scripts /synapse/scripts/ COPY MANIFEST.in README.rst setup.py synctl /synapse/ +COPY synapse/__init__.py /synapse/synapse/__init__.py +COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py +# To speed up rebuilds, install all of the dependencies before we copy over +# the whole synapse project so that we this layer in the Docker cache can be +# used while you develop on the source +# +# This is aiming at installing the `install_requires` and `extras_require` from `setup.py` RUN pip install --prefix="/install" --no-warn-script-location \ - /synapse[all] + /synapse[all] + +# Copy over the rest of the project +COPY synapse /synapse/synapse/ + +# Install the synapse package itself and all of its children packages. +# +# This is aiming at installing only the `packages=find_packages(...)` from `setup.py +RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse ### ### Stage 1: runtime @@ -69,16 +67,16 @@ RUN pip install --prefix="/install" --no-warn-script-location \ FROM docker.io/python:${PYTHON_VERSION}-slim RUN apt-get update && apt-get install -y \ - curl \ - gosu \ - libjpeg62-turbo \ - libpq5 \ - libwebp6 \ - xmlsec1 \ - libjemalloc2 \ - libssl-dev \ - openssl \ - && rm -rf /var/lib/apt/lists/* + curl \ + gosu \ + libjpeg62-turbo \ + libpq5 \ + libwebp6 \ + xmlsec1 \ + libjemalloc2 \ + libssl-dev \ + openssl \ + && rm -rf /var/lib/apt/lists/* COPY --from=builder /install /usr/local COPY ./docker/start.py /start.py @@ -91,4 +89,4 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp ENTRYPOINT ["/start.py"] HEALTHCHECK --interval=1m --timeout=5s \ - CMD curl -fSs http://localhost:8008/health || exit 1 + CMD curl -fSs http://localhost:8008/health || exit 1 diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index c6e63357401d..aa146e8bb8b2 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -169,7 +169,7 @@ def set_fates(clotho, lachesis, atropos, father="Zues", mother="Themis"): import logging import re from functools import wraps -from typing import TYPE_CHECKING, Dict, Optional, Type +from typing import TYPE_CHECKING, Dict, Optional, Pattern, Type import attr @@ -262,7 +262,7 @@ def report_span(self, span): # Block everything by default # A regex which matches the server_names to expose traces for. # None means 'block everything'. -_homeserver_whitelist = None # type: Optional[re.Pattern[str]] +_homeserver_whitelist = None # type: Optional[Pattern[str]] # Util methods diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 14ddaed026e0..2a1c925ee8b9 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -15,6 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools import logging from typing import List, Set @@ -101,7 +102,7 @@ "txacme>=0.9.2", # txacme depends on eliot. Eliot 1.8.0 is incompatible with # python 3.5.2, as per https://github.com/itamarst/eliot/issues/418 - 'eliot<1.8.0;python_version<"3.5.3"', + "eliot<1.8.0;python_version<'3.5.3'", ], "saml2": [ # pysaml2 6.4.0 is incompatible with Python 3.5 (see https://github.com/IdentityPython/pysaml2/issues/749) @@ -131,6 +132,18 @@ ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS +# ensure there are no double-quote characters in any of the deps (otherwise the +# 'pip install' incantation in DependencyException will break) +for dep in itertools.chain( + REQUIREMENTS, + *CONDITIONAL_REQUIREMENTS.values(), +): + if '"' in dep: + raise Exception( + "Dependency `%s` contains double-quote; use single-quotes instead" % (dep,) + ) + + def list_requirements(): return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS) @@ -150,7 +163,7 @@ def message(self): @property def dependencies(self): for i in self.args[0]: - yield "'" + i + "'" + yield '"' + i + '"' def check_requirements(for_feature=None): diff --git a/tests/replication/_base.py b/tests/replication/_base.py index 67b7913666fc..1d4a59286241 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -44,7 +44,7 @@ try: import hiredis except ImportError: - hiredis = None + hiredis = None # type: ignore logger = logging.getLogger(__name__)