Revert "Reduce RPKI memory use by not querying object_text"

This reverts commit dced78b.
irrdnet · Jun 25, 2021 · 43ce5e0 · 43ce5e0
1 parent dced78b
commit 43ce5e0
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 20 deletions.
diff --git a/irrd/rpki/notifications.py b/irrd/rpki/notifications.py
@@ -27,22 +27,16 @@ def notify_rpki_invalid_owners(database_handler: DatabaseHandler,
  if not get_setting('rpki.notify_invalid_enabled'):
  return 0
 
- pks_now_invalid: List[str] = []
+ rpsl_objs = []
  for obj in rpsl_dicts_now_invalid:
  source = obj['source']
  authoritative = get_setting(f'sources.{source}.authoritative')
  if authoritative and obj['rpki_status'] == RPKIStatus.invalid:
- pks_now_invalid.append(obj['pk'])
+ rpsl_objs.append(rpsl_object_from_text(obj['object_text']))
 
- if not pks_now_invalid:
+ if not rpsl_objs:
  return 0
 
- query = RPSLDatabaseQuery(['object_text']).pks(pks_now_invalid)
- rpsl_objs = [
- rpsl_object_from_text(obj['object_text'], strict_validation=False)
- for obj in database_handler.execute_query(query)
- ]
-
  sources = set([obj.parsed_data['source'] for obj in rpsl_objs])
  mntner_emails_by_source = {}
  for source in sources:

diff --git a/irrd/rpki/tests/test_notifications.py b/irrd/rpki/tests/test_notifications.py
@@ -24,15 +24,12 @@ def test_notify_regular(self, monkeypatch, config_override):
  monkeypatch.setattr('irrd.rpki.notifications.send_email', mock_email)
 
  rpsl_dicts_now_invalid = [
- {'source': 'TEST', 'pk': 'uuid-pk', 'rpki_status': RPKIStatus.invalid},
+ {'source': 'TEST', 'object_text': SAMPLE_ROUTE + 'mnt-by: DOESNOTEXIST-MNT\nMISSING-DATA-MNT\n', 'rpki_status': RPKIStatus.invalid},
  {'source': 'TEST', 'object_text': SAMPLE_ROUTE6, 'rpki_status': RPKIStatus.valid}, # should be ignored
  {'source': 'TEST2', 'object_text': SAMPLE_ROUTE6, 'rpki_status': RPKIStatus.invalid}, # should be ignored
  ]
 
  query_results = iter([
- [
- {'object_text': SAMPLE_ROUTE + 'mnt-by: DOESNOTEXIST-MNT\nMISSING-DATA-MNT\n','rpki_status': RPKIStatus.invalid},
- ],
  [
  {'rpsl_pk': 'TEST-MNT', 'parsed_data': {
  'mnt-nfy': ['mnt-nfy@example.com'],
@@ -50,7 +47,6 @@ def test_notify_regular(self, monkeypatch, config_override):
  assert notified == 3
 
  assert flatten_mock_calls(mock_dq) == [
- ['pks', (['uuid-pk'],), {}],
  ['sources', (['TEST'],), {}],
  ['rpsl_pks', ({'TEST-MNT', 'DOESNOTEXIST-MNT'},), {}],
  ['object_classes', (['mntner'],), {}],

diff --git a/irrd/storage/queries.py b/irrd/storage/queries.py
@@ -33,10 +33,6 @@ def pk(self, pk: str):
  """Filter on an exact object PK (UUID)."""
  return self._filter(self.columns.pk == pk)
 
- def pks(self, pks: List[str]):
- """Filter on exact object PKs (UUID)."""
- return self._filter(self.columns.pk.in_(pks))
-
  def rpsl_pk(self, rpsl_pk: str):
  """Filter on an exact RPSL PK (e.g. 192.0.2.0/24,AS65537)."""
  return self.rpsl_pks([rpsl_pk])

diff --git a/irrd/storage/tests/test_database.py b/irrd/storage/tests/test_database.py
@@ -523,13 +523,11 @@ def test_chained_filters(self, irrd_database, database_handler_with_route):
  pk = result[0]['pk']
 
  self._assert_match(RPSLDatabaseQuery().pk(pk))
- self._assert_match(RPSLDatabaseQuery().pks([pk]))
 
  def test_non_matching_filters(self, irrd_database, database_handler_with_route):
  self.dh = database_handler_with_route
  # None of these should match
  self._assert_no_match(RPSLDatabaseQuery().pk(str(uuid.uuid4())))
- self._assert_no_match(RPSLDatabaseQuery().pks([str(uuid.uuid4())]))
  self._assert_no_match(RPSLDatabaseQuery().rpsl_pk('foo'))
  self._assert_no_match(RPSLDatabaseQuery().sources(['TEST3']))
  self._assert_no_match(RPSLDatabaseQuery().object_classes(['route6']))