[WIP] External sharded cache

changelog.d/11046.feature

@@ -0,0 +1 @@
+Add new external sharded Redis cache. Contributed by Nick @ Beeper.

pyproject.toml

@@ -178,6 +178,7 @@ jaeger-client = { version = ">=4.0.0", optional = true }
 pyjwt = { version = ">=1.6.4", optional = true }
 txredisapi = { version = ">=1.4.7", optional = true }
 hiredis = { version = "*", optional = true }
+jump-consistent-hash = { version = ">=3.2.0", optional = true }
 Pympler = { version = "*", optional = true }
 parameterized = { version = ">=0.7.4", optional = true }
 idna = { version = ">=2.5", optional = true }
@@ -199,7 +200,7 @@ opentracing = ["jaeger-client", "opentracing"]
 jwt = ["pyjwt"]
 # hiredis is not a *strict* dependency, but it makes things much faster.
 # (if it is not installed, we fall back to slow code.)
-redis = ["txredisapi", "hiredis"]
+redis = ["txredisapi", "hiredis", "jump-consistent-hash"]
 # Required to use experimental `caches.track_memory_usage` config option.
 cache_memory = ["pympler"]
 test = ["parameterized", "idna"]
@@ -233,7 +234,7 @@ all = [
  # jwt
  "pyjwt",
  # redis
- "txredisapi", "hiredis",
+ "txredisapi", "hiredis", "jump-consistent-hash",
  # cache_memory
  "pympler",
  # omitted:

stubs/jump.pyi

@@ -0,0 +1,15 @@
+# Copyright 2022 Beeper
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def hash(key: int, buckets: int) -> int: ...

stubs/txredisapi.pyi

@@ -33,7 +33,9 @@ class RedisProtocol(protocol.Protocol):
  only_if_not_exists: bool = False,
  only_if_exists: bool = False,
  ) -> "Deferred[None]": ...
+ def mset(self, values: dict[str, Any]) -> "Deferred[Any]": ...
  def get(self, key: str) -> "Deferred[Any]": ...
+ def mget(self, keys: list[str]) -> "Deferred[Any]": ...
 
 class SubscriberProtocol(RedisProtocol):
  def __init__(self, *args: object, **kwargs: object): ...

synapse/config/redis.py

@@ -35,6 +35,12 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
  self.redis_port = redis_config.get("port", 6379)
  self.redis_password = redis_config.get("password")
 
+ cache_shard_config = redis_config.get("cache_shards")
+ if cache_shard_config:
+ self.cache_shard_hosts = cache_shard_config.get("hosts", [])
+ self.cache_shard_expire = cache_shard_config.get("expire_caches", False)
+ self.cache_shard_ttl = cache_shard_config.get("cache_entry_ttl", False)
+
  def generate_config_section(self, **kwargs: Any) -> str:
  return """\
  # Configuration for Redis when using workers. This *must* be enabled when
@@ -54,4 +60,16 @@ def generate_config_section(self, **kwargs: Any) -> str:
  # Optional password if configured on the Redis instance
  #
  #password: <secret_password>
+
+ # Optional one or more Redis hosts to use for long term shared caches.
+ # Should be configured to automatically expire records when out of
+ # memory, and not be the same instance as used for replication.
+ #
+ #cache_shards:
+ # enabled: false
+ # expire_caches: false
+ # cache_entry_ttl: 30m
+ # hosts:
+ # - host: localhost
+ # port: 6379
  """