Skip to content

Commit

Permalink
Reduce RPKI memory use by not querying object_text
Browse files Browse the repository at this point in the history
  • Loading branch information
@mxsasha
mxsasha committed Jun 27, 2021
1 parent 224f3a3 commit d12fb31
Show file tree
Hide file tree
Showing 4 changed files with 187 additions and 102 deletions.
272 changes: 171 additions & 101 deletions irrd/rpki/tests/test_validators.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,105 +21,163 @@ def test_validate_routes_from_roa_objs(self, monkeypatch, config_override):
monkeypatch.setattr('irrd.rpki.validators.RPSLDatabaseQuery',
lambda column_names, enable_ordering: mock_dq)

mock_query_result = [
{
'rpsl_pk': 'pk_route_v4_d0_l24',
'ip_version': 4,
'ip_first': '192.0.2.0',
'prefix_length': 24,
'asn_first': 65546,
'rpki_status': RPKIStatus.not_found,
'source': 'TEST1',
},
{
'rpsl_pk': 'pk_route_v4_d0_l25',
'ip_version': 4,
'ip_first': '192.0.2.0',
'prefix_length': 25,
'asn_first': 65546,
'rpki_status': RPKIStatus.not_found,
'source': 'TEST1',
},
{
# This route is valid, but as the state is already valid,
# it should not be included in the response.
'rpsl_pk': 'pk_route_v4_d0_l28',
'ip_version': 4,
'ip_first': '192.0.2.0',
'prefix_length': 27,
'asn_first': 65546,
'rpki_status': RPKIStatus.valid,
'source': 'TEST1',
},
{
'rpsl_pk': 'pk_route_v4_d64_l32',
'ip_version': 4,
'ip_first': '192.0.2.64',
'prefix_length': 32,
'asn_first': 65546,
'rpki_status': RPKIStatus.valid,
'source': 'TEST1',
},
{
'rpsl_pk': 'pk_route_v4_d128_l25',
'ip_version': 4,
'ip_first': '192.0.2.128',
'prefix_length': 25,
'asn_first': 65547,
'rpki_status': RPKIStatus.valid,
'source': 'TEST1',
},
{
# RPKI invalid, but should be ignored.
'rpsl_pk': 'pk_route_v4_d128_l26',
'ip_version': 4,
'ip_first': '192.0.2.128',
'prefix_length': 26,
'asn_first': 65547,
'rpki_status': RPKIStatus.invalid,
'source': RPKI_IRR_PSEUDO_SOURCE,
},
{
# RPKI invalid, but should be not_found because of source.
'rpsl_pk': 'pk_route_v4_d128_l26_excluded',
'ip_version': 4,
'ip_first': '192.0.2.128',
'prefix_length': 26,
'asn_first': 65547,
'rpki_status': RPKIStatus.valid,
'source': 'SOURCE-EXCLUDED',
},
{
'rpsl_pk': 'pk_route_v6',
'ip_version': 6,
'ip_first': '2001:db8::',
'prefix_length': 32,
'asn_first': 65547,
'rpki_status': RPKIStatus.invalid,
'source': 'TEST1',
},
{
# Should not match any ROA - ROAs for a subset
# exist, but those should not be included
'rpsl_pk': 'pk_route_v4_no_roa',
'ip_version': 4,
'ip_first': '192.0.2.0',
'prefix_length': 23,
'asn_first': 65549,
'rpki_status': RPKIStatus.valid,
'source': 'TEST1',
},
{
'rpsl_pk': 'pk_route_v4_roa_as0',
'ip_version': 4,
'ip_first': '203.0.113.1',
'prefix_length': 32,
'asn_first': 65547,
'rpki_status': RPKIStatus.not_found,
'source': 'TEST1',
},
]
mock_dh.execute_query = lambda query: mock_query_result
mock_query_result = iter([
[
{
'pk': 'pk_route_v4_d0_l24',
'rpsl_pk': 'pk_route_v4_d0_l24',
'ip_version': 4,
'ip_first': '192.0.2.0',
'prefix_length': 24,
'asn_first': 65546,
'rpki_status': RPKIStatus.not_found,
'source': 'TEST1',
},
{
'pk': 'pk_route_v4_d0_l25',
'rpsl_pk': 'pk_route_v4_d0_l25',
'ip_version': 4,
'ip_first': '192.0.2.0',
'prefix_length': 25,
'asn_first': 65546,
'rpki_status': RPKIStatus.not_found,
'source': 'TEST1',
},
{
# This route is valid, but as the state is already valid,
# it should not be included in the response.
'pk': 'pk_route_v4_d0_l28',
'rpsl_pk': 'pk_route_v4_d0_l28',
'ip_version': 4,
'ip_first': '192.0.2.0',
'prefix_length': 27,
'asn_first': 65546,
'rpki_status': RPKIStatus.valid,
'source': 'TEST1',
},
{
'pk': 'pk_route_v4_d64_l32',
'rpsl_pk': 'pk_route_v4_d64_l32',
'ip_version': 4,
'ip_first': '192.0.2.64',
'prefix_length': 32,
'asn_first': 65546,
'rpki_status': RPKIStatus.valid,
'source': 'TEST1',
},
{
'pk': 'pk_route_v4_d128_l25',
'rpsl_pk': 'pk_route_v4_d128_l25',
'ip_version': 4,
'ip_first': '192.0.2.128',
'prefix_length': 25,
'asn_first': 65547,
'rpki_status': RPKIStatus.valid,
'source': 'TEST1',
},
{
# RPKI invalid, but should be ignored.
'pk': 'pk_route_v4_d128_l26_rpki',
'rpsl_pk': 'pk_route_v4_d128_l26',
'ip_version': 4,
'ip_first': '192.0.2.128',
'prefix_length': 26,
'asn_first': 65547,
'rpki_status': RPKIStatus.invalid,
'source': RPKI_IRR_PSEUDO_SOURCE,
},
{
# RPKI invalid, but should be not_found because of source.
'pk': 'pk_route_v4_d128_l26_excluded',
'rpsl_pk': 'pk_route_v4_d128_l26_excluded',
'ip_version': 4,
'ip_first': '192.0.2.128',
'prefix_length': 26,
'asn_first': 65547,
'rpki_status': RPKIStatus.valid,
'source': 'SOURCE-EXCLUDED',
},
{
'pk': 'pk_route_v6',
'rpsl_pk': 'pk_route_v6',
'ip_version': 6,
'ip_first': '2001:db8::',
'prefix_length': 32,
'asn_first': 65547,
'rpki_status': RPKIStatus.invalid,
'source': 'TEST1',
},
{
# Should not match any ROA - ROAs for a subset
# exist, but those should not be included
'pk': 'pk_route_v4_no_roa',
'rpsl_pk': 'pk_route_v4_no_roa',
'ip_version': 4,
'ip_first': '192.0.2.0',
'prefix_length': 23,
'asn_first': 65549,
'rpki_status': RPKIStatus.valid,
'source': 'TEST1',
},
{
'pk': 'pk_route_v4_roa_as0',
'rpsl_pk': 'pk_route_v4_roa_as0',
'ip_version': 4,
'ip_first': '203.0.113.1',
'prefix_length': 32,
'asn_first': 65547,
'rpki_status': RPKIStatus.not_found,
'source': 'TEST1',
},
], [
{
'pk': 'pk_route_v4_d0_l24',
'object_text': 'object text',
'object_class': 'route',
},
{
'pk': 'pk_route_v4_d0_l25',
'object_text': 'object text',
'object_class': 'route',
},
{
'pk': 'pk_route_v4_d64_l32',
'object_text': 'object text',
'object_class': 'route',
},
{
'pk': 'pk_route_v4_d128_l25',
'object_text': 'object text',
'object_class': 'route',
},
{
'pk': 'pk_route_v4_d128_l26_rpki',
'object_text': 'object text',
'object_class': 'route',
},
{
'pk': 'pk_route_v4_d128_l26_excluded',
'object_text': 'object text',
'object_class': 'route',
},
{
'pk': 'pk_route_v6',
'object_text': 'object text',
'object_class': 'route',
},
{
'pk': 'pk_route_v4_no_roa',
'object_text': 'object text',
'object_class': 'route',
},
{
'pk': 'pk_route_v4_roa_as0',
'object_text': 'object text',
'object_class': 'route',
},
]
])
mock_dh.execute_query = lambda query: next(mock_query_result)

roas = [
# Valid for pk_route_v4_d0_l25 and pk_route_v4_d0_l24
Expand All @@ -142,12 +200,15 @@ def test_validate_routes_from_roa_objs(self, monkeypatch, config_override):
result = BulkRouteROAValidator(mock_dh, roas).validate_all_routes(sources=['TEST1'])
new_valid_objs, new_invalid_objs, new_unknown_objs = result
assert {o['rpsl_pk'] for o in new_valid_objs} == {'pk_route_v6', 'pk_route_v4_d0_l25', 'pk_route_v4_d0_l24'}
assert [o['object_class'] for o in new_valid_objs] == ['route', 'route', 'route']
assert [o['object_text'] for o in new_valid_objs] == ['object text', 'object text', 'object text']
assert {o['rpsl_pk'] for o in new_invalid_objs} == {'pk_route_v4_d64_l32', 'pk_route_v4_d128_l25', 'pk_route_v4_roa_as0'}
assert {o['rpsl_pk'] for o in new_unknown_objs} == {'pk_route_v4_no_roa', 'pk_route_v4_d128_l26_excluded'}

assert flatten_mock_calls(mock_dq) == [
['object_classes', (['route', 'route6'],), {}],
['sources', (['TEST1'],), {}]
['sources', (['TEST1'],), {}],
['pks', (['pk_route_v4_d0_l24', 'pk_route_v4_d0_l25', 'pk_route_v6', 'pk_route_v4_d64_l32', 'pk_route_v4_d128_l25', 'pk_route_v4_roa_as0', 'pk_route_v4_d128_l26_excluded', 'pk_route_v4_no_roa'], ), {}],
]

def test_validate_routes_with_roa_from_database(self, monkeypatch, config_override):
Expand Down Expand Up @@ -178,6 +239,7 @@ def test_validate_routes_with_roa_from_database(self, monkeypatch, config_overri
},
], [ # RPSL objects:
{
'pk': 'pk1',
'rpsl_pk': 'pk_route_v4_d0_l25',
'ip_version': 4,
'ip_first': '192.0.2.0',
Expand All @@ -186,19 +248,27 @@ def test_validate_routes_with_roa_from_database(self, monkeypatch, config_overri
'rpki_status': RPKIStatus.not_found,
'source': 'TEST1',
},
], [
{
'pk': 'pk1',
'object_class': 'route',
'object_text': 'object text',
},
]
])
mock_dh.execute_query = lambda query: next(mock_query_result)

result = BulkRouteROAValidator(mock_dh).validate_all_routes(sources=['TEST1'])
new_valid_pks, new_invalid_pks, new_unknown_pks = result
assert {o['rpsl_pk'] for o in new_valid_pks} == {'pk_route_v4_d0_l25'}
assert {o['object_text'] for o in new_valid_pks} == {'object text'}
assert new_invalid_pks == list()
assert new_unknown_pks == list()

assert flatten_mock_calls(mock_dq) == [
['object_classes', (['route', 'route6'],), {}],
['sources', (['TEST1'],), {}]
['sources', (['TEST1'],), {}],
['pks', (['pk1'],), {}],
]
assert flatten_mock_calls(mock_rq) == [] # No filters applied

Expand Down
11 changes: 10 additions & 1 deletion irrd/rpki/validators.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def validate_all_routes(self, sources: List[str]=None) -> \
validation result, are not included in the return value.
"""
columns = ['pk', 'rpsl_pk', 'ip_first', 'prefix_length', 'asn_first', 'source',
'object_class', 'object_text', 'rpki_status']
'rpki_status']
q = RPSLDatabaseQuery(column_names=columns, enable_ordering=False)
q = q.object_classes(['route', 'route6'])
if sources:
Expand All @@ -107,6 +107,15 @@ def validate_all_routes(self, sources: List[str]=None) -> \
result['rpki_status'] = new_status
objs_changed[new_status].append(result)

# Object text and class are only retrieved for objects with state changes
pks_to_enrich = [obj['pk'] for objs in objs_changed.values() for obj in objs]
query = RPSLDatabaseQuery(['pk', 'object_text', 'object_class'], enable_ordering=False).pks(pks_to_enrich)
rows_per_pk = {row['pk']: row for row in self.database_handler.execute_query(query)}

for rpsl_objs in objs_changed.values():
for rpsl_obj in rpsl_objs:
rpsl_obj.update(rows_per_pk[rpsl_obj['pk']])

return objs_changed[RPKIStatus.valid], objs_changed[RPKIStatus.invalid], objs_changed[RPKIStatus.not_found]

def validate_route(self, prefix_ip: str, prefix_length: int, prefix_asn: int, source: str) -> RPKIStatus:
Expand Down
4 changes: 4 additions & 0 deletions irrd/storage/queries.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ def pk(self, pk: str):
"""Filter on an exact object PK (UUID)."""
return self._filter(self.columns.pk == pk)

def pks(self, pks: List[str]):
"""Filter on exact object PKs (UUID)."""
return self._filter(self.columns.pk.in_(pks))

def rpsl_pk(self, rpsl_pk: str):
"""Filter on an exact RPSL PK (e.g. 192.0.2.0/24,AS65537)."""
return self.rpsl_pks([rpsl_pk])
Expand Down
2 changes: 2 additions & 0 deletions irrd/storage/tests/test_database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -523,11 +523,13 @@ def test_chained_filters(self, irrd_database, database_handler_with_route):
pk = result[0]['pk']

self._assert_match(RPSLDatabaseQuery().pk(pk))
self._assert_match(RPSLDatabaseQuery().pks([pk]))

def test_non_matching_filters(self, irrd_database, database_handler_with_route):
self.dh = database_handler_with_route
# None of these should match
self._assert_no_match(RPSLDatabaseQuery().pk(str(uuid.uuid4())))
self._assert_no_match(RPSLDatabaseQuery().pks([str(uuid.uuid4())]))
self._assert_no_match(RPSLDatabaseQuery().rpsl_pk('foo'))
self._assert_no_match(RPSLDatabaseQuery().sources(['TEST3']))
self._assert_no_match(RPSLDatabaseQuery().object_classes(['route6']))
Expand Down

0 comments on commit d12fb31

Please sign in to comment.