Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix #525 - Reduce scope filter memory use (ref #497) #530

Merged
merged 1 commit into from
Jul 29, 2021
+93 −57
Merged

Fix #525 - Reduce scope filter memory use (ref #497) #530

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 82 additions & 56 deletions irrd/scopefilter/tests/test_scopefilter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,62 +111,83 @@ def test_validate_all_rpsl_objects(self, config_override, monkeypatch):
},
})

mock_query_result = [
{
# Should become in_scope
'rpsl_pk': '192.0.2.128/25,AS65547',
'ip_first': '192.0.2.128',
'prefix_length': 25,
'asn_first': 65547,
'source': 'TEST',
'object_class': 'route',
'object_text': 'text',
'scopefilter_status': ScopeFilterStatus.out_scope_prefix,
},
{
# Should become out_scope_prefix
'rpsl_pk': '192.0.2.0/25,AS65547',
'ip_first': '192.0.2.0',
'prefix_length': 25,
'asn_first': 65547,
'source': 'TEST',
'object_class': 'route',
'object_text': 'text',
'scopefilter_status': ScopeFilterStatus.in_scope,
},
{
# Should become out_scope_as
'rpsl_pk': '192.0.2.128/25,AS65547',
'ip_first': '192.0.2.128',
'prefix_length': 25,
'asn_first': 23456,
'source': 'TEST',
'object_class': 'route',
'object_text': 'text',
'scopefilter_status': ScopeFilterStatus.out_scope_prefix,
},
{
# Should become out_scope_as
'rpsl_pk': 'AS65547',
'asn_first': 23456,
'source': 'TEST',
'object_class': 'aut-num',
'object_text': 'text',
'scopefilter_status': ScopeFilterStatus.in_scope,
},
{
# Should not change
'rpsl_pk': '192.0.2.128/25,AS65548',
'ip_first': '192.0.2.128',
'prefix_length': 25,
'asn_first': 65548,
'source': 'TEST',
'object_class': 'route',
'object_text': 'text',
'scopefilter_status': ScopeFilterStatus.in_scope,
},
]
mock_dh.execute_query = lambda query: mock_query_result
mock_query_result = iter([
[
{
# Should become in_scope
'pk': '192.0.2.128/25,AS65547',
'rpsl_pk': '192.0.2.128/25,AS65547',
'ip_first': '192.0.2.128',
'prefix_length': 25,
'asn_first': 65547,
'source': 'TEST',
'object_class': 'route',
'scopefilter_status': ScopeFilterStatus.out_scope_prefix,
},
{
# Should become out_scope_prefix
'pk': '192.0.2.0/25,AS65547',
'rpsl_pk': '192.0.2.0/25,AS65547',
'ip_first': '192.0.2.0',
'prefix_length': 25,
'asn_first': 65547,
'source': 'TEST',
'object_class': 'route',
'scopefilter_status': ScopeFilterStatus.in_scope,
},
{
# Should become out_scope_as
'pk': '192.0.2.128/25,AS65547',
'rpsl_pk': '192.0.2.128/25,AS65547',
'ip_first': '192.0.2.128',
'prefix_length': 25,
'asn_first': 23456,
'source': 'TEST',
'object_class': 'route',
'scopefilter_status': ScopeFilterStatus.out_scope_prefix,
},
{
# Should become out_scope_as
'pk': 'AS65547',
'rpsl_pk': 'AS65547',
'asn_first': 23456,
'source': 'TEST',
'object_class': 'aut-num',
'object_text': 'text',
'scopefilter_status': ScopeFilterStatus.in_scope,
},
{
# Should not change
'pk': '192.0.2.128/25,AS65548',
'rpsl_pk': '192.0.2.128/25,AS65548',
'ip_first': '192.0.2.128',
'prefix_length': 25,
'asn_first': 65548,
'source': 'TEST',
'object_class': 'route',
'scopefilter_status': ScopeFilterStatus.in_scope,
},
],
[
{
'pk': '192.0.2.128/25,AS65547',
'object_text': 'text-192.0.2.128/25,AS65547',
},
{
'pk': '192.0.2.0/25,AS65547',
'object_text': 'text-192.0.2.0/25,AS65547',
},
{
'pk': '192.0.2.128/25,AS65547',
'object_text': 'text-192.0.2.128/25,AS65547',
},
{
'pk': 'AS65547',
'object_text': 'text-AS65547',
},
]
])
mock_dh.execute_query = lambda query: next(mock_query_result)

validator = ScopeFilterValidator()
result = validator.validate_all_rpsl_objects(mock_dh)
Expand All @@ -178,15 +199,20 @@ def test_validate_all_rpsl_objects(self, config_override, monkeypatch):

assert now_in_scope[0]['rpsl_pk'] == '192.0.2.128/25,AS65547'
assert now_in_scope[0]['old_status'] == ScopeFilterStatus.out_scope_prefix
assert now_in_scope[0]['object_text'] == 'text-192.0.2.128/25,AS65547'

assert now_out_scope_as[0]['rpsl_pk'] == '192.0.2.128/25,AS65547'
assert now_out_scope_as[0]['old_status'] == ScopeFilterStatus.out_scope_prefix
assert now_out_scope_as[0]['object_text'] == 'text-192.0.2.128/25,AS65547'
assert now_out_scope_as[1]['rpsl_pk'] == 'AS65547'
assert now_out_scope_as[1]['old_status'] == ScopeFilterStatus.in_scope
assert now_out_scope_as[1]['object_text'] == 'text-AS65547'

assert now_out_scope_prefix[0]['rpsl_pk'] == '192.0.2.0/25,AS65547'
assert now_out_scope_prefix[0]['old_status'] == ScopeFilterStatus.in_scope
assert now_out_scope_prefix[0]['object_text'] == 'text-192.0.2.0/25,AS65547'

assert flatten_mock_calls(mock_dq) == [
['object_classes', (['route', 'route6', 'aut-num'],), {}],
['pks', (['192.0.2.128/25,AS65547', '192.0.2.0/25,AS65547', '192.0.2.128/25,AS65547', 'AS65547'],), {}],
]
12 changes: 11 additions & 1 deletion irrd/scopefilter/validators.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ def validate_all_rpsl_objects(self, database_handler: DatabaseHandler) -> \
validation result, are not included in the return value.
"""
columns = ['rpsl_pk', 'ip_first', 'prefix_length', 'asn_first', 'source', 'object_class',
'object_text', 'scopefilter_status', 'rpki_status']
'scopefilter_status', 'rpki_status']

objs_changed: Dict[ScopeFilterStatus, List[Dict[str, str]]] = defaultdict(list)

Expand All @@ -136,6 +136,16 @@ def validate_all_rpsl_objects(self, database_handler: DatabaseHandler) -> \
if new_status != current_status:
result['scopefilter_status'] = new_status
objs_changed[new_status].append(result)

# Object text is only retrieved for objects with state changes
pks_to_enrich = [obj['pk'] for objs in objs_changed.values() for obj in objs]
query = RPSLDatabaseQuery(['pk', 'object_text'], enable_ordering=False).pks(pks_to_enrich)
rows_per_pk = {row['pk']: row for row in database_handler.execute_query(query)}

for rpsl_objs in objs_changed.values():
for rpsl_obj in rpsl_objs:
rpsl_obj.update(rows_per_pk[rpsl_obj['pk']])

return (objs_changed[ScopeFilterStatus.in_scope],
objs_changed[ScopeFilterStatus.out_scope_as],
objs_changed[ScopeFilterStatus.out_scope_prefix])