Fixes to the federation rate limiter

changelog.d/5621.bugfix

@@ -0,0 +1 @@
+Various minor fixes to the federation request rate limiter.

synapse/config/ratelimiting.py

@@ -23,7 +23,7 @@ def __init__(self, config, defaults={"per_second": 0.17, "burst_count": 3.0}):
 
 class FederationRateLimitConfig(object):
  _items_and_default = {
- "window_size": 10000,
+ "window_size": 1000,
  "sleep_limit": 10,
  "sleep_delay": 500,
  "reject_limit": 50,
@@ -54,7 +54,7 @@ def read_config(self, config, **kwargs):
 
  # Load the new-style federation config, if it exists. Otherwise, fall
  # back to the old method.
- if "federation_rc" in config:
+ if "rc_federation" in config:
  self.rc_federation = FederationRateLimitConfig(**config["rc_federation"])
  else:
  self.rc_federation = FederationRateLimitConfig(

synapse/util/ratelimitutils.py

@@ -36,9 +36,11 @@ def __init__(self, clock, config):
  clock (Clock)
  config (FederationRateLimitConfig)
  """
- self.clock = clock
- self._config = config
- self.ratelimiters = {}
+
+ def new_limiter():
+ return _PerHostRatelimiter(clock=clock, config=config)
+
+ self.ratelimiters = collections.defaultdict(new_limiter)
 
  def ratelimit(self, host):
  """Used to ratelimit an incoming request from given host
@@ -53,11 +55,9 @@ def ratelimit(self, host):
  host (str): Origin of incoming request.
 
  Returns:
- _PerHostRatelimiter
+ context manager which returns a deferred.
  """
- return self.ratelimiters.setdefault(
- host, _PerHostRatelimiter(clock=self.clock, config=self._config)
- ).ratelimit()
+ return self.ratelimiters[host].ratelimit()
 
 
 class _PerHostRatelimiter(object):
@@ -122,7 +122,7 @@ def _on_enter(self, request_id):
  self.request_times.append(time_now)
 
  def queue_request():
- if len(self.current_processing) > self.concurrent_requests:
+ if len(self.current_processing) >= self.concurrent_requests:
  queue_defer = defer.Deferred()
  self.ready_request_queue[request_id] = queue_defer
  logger.info(

tests/config/test_ratelimiting.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from synapse.config.homeserver import HomeServerConfig
+
+from tests.unittest import TestCase
+from tests.utils import default_config
+
+
+class RatelimitConfigTestCase(TestCase):
+ def test_parse_rc_federation(self):
+ config_dict = default_config("test")
+ config_dict["rc_federation"] = {
+ "window_size": 20000,
+ "sleep_limit": 693,
+ "sleep_delay": 252,
+ "reject_limit": 198,
+ "concurrent": 7,
+ }
+
+ config = HomeServerConfig()
+ config.parse_config_dict(config_dict, "", "")
+ config_obj = config.rc_federation
+
+ self.assertEqual(config_obj.window_size, 20000)
+ self.assertEqual(config_obj.sleep_limit, 693)
+ self.assertEqual(config_obj.sleep_delay, 252)
+ self.assertEqual(config_obj.reject_limit, 198)
+ self.assertEqual(config_obj.concurrent, 7)

tests/util/test_ratelimitutils.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from synapse.config.homeserver import HomeServerConfig
+from synapse.util.ratelimitutils import FederationRateLimiter
+
+from tests.server import get_clock
+from tests.unittest import TestCase
+from tests.utils import default_config
+
+
+class FederationRateLimiterTestCase(TestCase):
+ def test_ratelimit(self):
+ """A simple test with the default values"""
+ reactor, clock = get_clock()
+ rc_config = build_rc_config()
+ ratelimiter = FederationRateLimiter(clock, rc_config)
+
+ with ratelimiter.ratelimit("testhost") as d1:
+ # shouldn't block
+ self.successResultOf(d1)
+
+ def test_concurrent_limit(self):
+ """Test what happens when we hit the concurrent limit"""
+ reactor, clock = get_clock()
+ rc_config = build_rc_config({"rc_federation": {"concurrent": 2}})
+ ratelimiter = FederationRateLimiter(clock, rc_config)
+
+ with ratelimiter.ratelimit("testhost") as d1:
+ # shouldn't block
+ self.successResultOf(d1)
+
+ cm2 = ratelimiter.ratelimit("testhost")
+ d2 = cm2.__enter__()
+ # also shouldn't block
+ self.successResultOf(d2)
+
+ cm3 = ratelimiter.ratelimit("testhost")
+ d3 = cm3.__enter__()
+ # this one should block, though ...
+ self.assertNoResult(d3)
+
+ # ... until we complete an earlier request
+ cm2.__exit__(None, None, None)
+ self.successResultOf(d3)
+
+ def test_sleep_limit(self):
+ """Test what happens when we hit the sleep limit"""
+ reactor, clock = get_clock()
+ rc_config = build_rc_config(
+ {"rc_federation": {"sleep_limit": 2, "sleep_delay": 500}}
+ )
+ ratelimiter = FederationRateLimiter(clock, rc_config)
+
+ with ratelimiter.ratelimit("testhost") as d1:
+ # shouldn't block
+ self.successResultOf(d1)
+
+ with ratelimiter.ratelimit("testhost") as d2:
+ # nor this
+ self.successResultOf(d2)
+
+ with ratelimiter.ratelimit("testhost") as d3:
+ # this one should block, though ...
+ self.assertNoResult(d3)
+ sleep_time = _await_resolution(reactor, d3)
+ self.assertAlmostEqual(sleep_time, 500, places=3)
+
+
+def _await_resolution(reactor, d):
+ """advance the clock until the deferred completes.
+
+ Returns the number of milliseconds it took to complete.
+ """
+ start_time = reactor.seconds()
+ while not d.called:
+ reactor.advance(0.01)
+ return (reactor.seconds() - start_time) * 1000
+
+
+def build_rc_config(settings={}):
+ config_dict = default_config("test")
+ config_dict.update(settings)
+ config = HomeServerConfig()
+ config.parse_config_dict(config_dict, "", "")
+ return config.rc_federation

tests/utils.py

@@ -152,12 +152,6 @@ def default_config(name, parse=False):
  "mau_stats_only": False,
  "mau_limits_reserved_threepids": [],
  "admin_contact": None,
- "rc_federation": {
- "reject_limit": 10,
- "sleep_limit": 10,
- "sleep_delay": 10,
- "concurrent": 10,
- },
  "rc_message": {"per_second": 10000, "burst_count": 10000},
  "rc_registration": {"per_second": 10000, "burst_count": 10000},
  "rc_login": {