Fix #742 - Add setting to limit NRTM response size

docs/admins/configuration.rst

@@ -639,6 +639,16 @@ Sources
   |br| **Default**: not defined, all access denied. Clients in
   ``nrtm_access_list``, if defined, have filtered access.
   |br| **Change takes effect**: after SIGHUP, upon next request.
+* ``sources.{name}.nrtm_query_serial_range_limit``: the maximum number of
+  serials a client may request in one NRTM query, if otherwise permitted.
+  This is intended to limit the maximum load of NRTM queries - it is checked
+  before IRRd runs any heavy database queries. The limit is applied to the
+  requested range regardless of any gaps, i.e. a range of ``10-20`` requires
+  this setting to be at least 10, even if there are no entries for some of
+  these serials. IRRd is aware of the serial ``LAST`` refers to and will take
+  that into account.
+  |br| **Default**: not defined, no limits on NRTM query size.
+  |br| **Change takes effect**: after SIGHUP, upon next request.
 * ``sources.{name}.strict_import_keycert_objects``: a setting used when
   migrating authoritative data that may contain `key-cert` objects.
   See the :doc:`data migration guide </admins/availability-and-migration>`

docs/releases/4.3.0.rst

@@ -119,6 +119,8 @@ significantly faster.
 
 Other changes
 -------------
+* The ``sources.{name}.nrtm_query_serial_range_limit`` setting was
+  added to limit the load of NRTM queries.
 * IRRd processes will now log a traceback of all their threads when
   receiving a SIGUSR1 signal. This can be helpful when debugging
   hanging workers or other complex issues.

irrd/conf/__init__.py

@@ -352,6 +352,8 @@ def _validate_subconfig(key, value):
                 errors.append(f'Setting import_timer for source {name} must be a number.')
             if not str(details.get('export_timer', '0')).isnumeric():
                 errors.append(f'Setting export_timer for source {name} must be a number.')
+            if not str(details.get('nrtm_query_serial_range_limit', '0')).isnumeric():
+                errors.append(f'Setting nrtm_query_serial_range_limit for source {name} must be a number.')
 
             if details.get('nrtm_access_list'):
                 expected_access_lists.add(details.get('nrtm_access_list'))

irrd/conf/known_keys.py

@@ -90,6 +90,7 @@
     'export_timer',
     'nrtm_access_list',
     'nrtm_access_list_unfiltered',
+    'nrtm_query_serial_range_limit',
     'strict_import_keycert_objects',
     'rpki_excluded',
     'scopefilter_excluded',

irrd/conf/test_conf.py

@@ -95,6 +95,7 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp
                         'authoritative': True,
                         'keep_journal': True,
                         'suspension_enabled': True,
+                        'nrtm_query_serial_range_limit': 10,
                     },
                     'TESTDB2': {
                         'nrtm_host': '192.0.2.1',
@@ -290,6 +291,7 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
                         'nrtm_host': '192.0.2.1',
                         'unknown': True,
                         'suspension_enabled': True,
+                        'nrtm_query_serial_range_limit': 'not-a-number',
                     },
                     'TESTDB2': {
                         'authoritative': True,
@@ -353,6 +355,7 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
         assert 'Setting rpki.notify_invalid_header must be a string, if defined.' in str(ce.value)
         assert 'Setting import_timer for source TESTDB must be a number.' in str(ce.value)
         assert 'Setting export_timer for source TESTDB must be a number.' in str(ce.value)
+        assert 'Setting nrtm_query_serial_range_limit for source TESTDB must be a number.' in str(ce.value)
         assert 'Invalid source name: lowercase' in str(ce.value)
         assert 'Invalid source name: invalid char' in str(ce.value)
         assert 'but rpki.notify_invalid_enabled is not set' in str(ce.value)

irrd/mirroring/nrtm_generator.py

@@ -57,6 +57,11 @@ def generate(self, source: str, version: str,
 
         serial_end_display = serial_end_available if serial_end_requested is None else serial_end_requested
 
+        range_limit = get_setting(f'sources.{source}.nrtm_query_serial_range_limit')
+        print(f'{range_limit=} {(serial_end_display - serial_start_requested)=}')
+        if range_limit and int(range_limit) < (serial_end_display - serial_start_requested):
+            raise NRTMGeneratorException(f'Serial range requested exceeds maximum range of {range_limit}')
+
         q = RPSLDatabaseJournalQuery().sources([source]).serial_nrtm_range(serial_start_requested, serial_end_requested)
         operations = list(database_handler.execute_query(q))
 

irrd/mirroring/tests/test_nrtm_generator.py

@@ -15,6 +15,7 @@ def prepare_generator(monkeypatch, config_override):
         'sources': {
             'TEST': {
                 'keep_journal': True,
+                'nrtm_query_serial_range_limit': 200,
             }
         }
     })
@@ -84,7 +85,7 @@ def test_generate_serial_range_v1(self, prepare_generator):
 
         %END TEST""").strip()
 
-    def test_generate_until_last(self, prepare_generator):
+    def test_generate_until_last(self, prepare_generator, config_override):
         generator, mock_dh = prepare_generator
         result = generator.generate('TEST', '3', 110, None, mock_dh)
 
@@ -169,6 +170,47 @@ def test_no_source_status_entry(self, prepare_generator, config_override):
             generator.generate('TEST', '3', 110, 300, mock_dh)
         assert 'There are no journal entries for this source.' in str(nge.value)
 
+    def test_v3_range_limit_not_set(self, prepare_generator, config_override):
+        generator, mock_dh = prepare_generator
+        config_override({
+            'sources': {
+                'TEST': {
+                    'keep_journal': True,
+                }
+            }
+        })
+
+        result = generator.generate('TEST', '3', 110, 190, mock_dh)
+
+        assert result == textwrap.dedent("""
+        %START Version: 3 TEST 110-190
+
+        ADD 120
+
+        object 1 🦄
+        auth: CRYPT-PW DummyValue  # Filtered for security
+
+        DEL 180
+
+        object 2 🌈
+
+        %END TEST""").strip()
+
+    def test_range_limit_exceeded(self, prepare_generator, config_override):
+        generator, mock_dh = prepare_generator
+        config_override({
+            'sources': {
+                'TEST': {
+                    'keep_journal': True,
+                    'nrtm_query_serial_range_limit': 50,
+                }
+            }
+        })
+
+        with pytest.raises(NRTMGeneratorException) as nge:
+            generator.generate('TEST', '3', 110, 190, mock_dh)
+        assert 'Serial range requested exceeds maximum range of 50' in str(nge.value)
+
     def test_include_auth_hash(self, prepare_generator):
         generator, mock_dh = prepare_generator
         result = generator.generate('TEST', '3', 110, 190, mock_dh, False)