From dc6e931ecb1db237d9588e4a08aef242cf2c6055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Herr=C3=A1n?= <106821523+MiguelAHM@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:39:42 +0200 Subject: [PATCH] Schedule ROA load each 15 minutes (#1444) * feat: create new warning for roas check and test for legacy * feat: add test for REST API calls and little refactor to specify rest messages * feat: fix ITs * feat: add more tests * feat: create the validator * feat: fix compile error due to previous code * feat: renaming and deleting white spaces * feat: fix tests according to last changes * feat: remove already migrated classes * feat: change generator by validator * feat: fix compilation errors * feat: increase testperformance to 3s * feat: put timeout back to 2s and fix compilation issue in whoisFixture * feat: fix compilation issues * feat: fix ITs * feat: refactor messages and PR comments * feat: fix ITs * feat: refactor * feat: rename * feat: rename * Return 404 on invalid NRTMv4 notification file path (#1422) * _check_valid_notification_file * refactor --------- Co-authored-by: Ed Shryane * feat: fix logger issue (#1427) * remove_unused_code (#1429) * remove_unused_code * fix_tests * Revert changes to thread configuration in tests (#1415) * Handle Multiple Delivery Status Notification Failure Responses for the Same Email Address (#1417) * feat: remove message after error and catch DuplicateKeyException * feat: split the changes, just leave Catch in this PR * feat: catch the exception by email, no for all the emails. Turn the warn into debug * feat: put white space after try and before bracket * feat: just loop once * Added test to make sure the message doesn't get stuck * Fix test (rs.next() behaves the same if there is a result row or not) --------- Co-authored-by: Ed Shryane * Fix tests (#1433) * fix tests * fix tests * reduce mariadb innodb memory usage to 1GB (#1439) * Set memory limit for test elasticsearch instance (#1436) * Set memory limit for test elasticsearch instance * Move ES memlimit to ENV variable * Reduce elasticsearch memory limit --------- Co-authored-by: Ed Shryane * feat: renaming, revert ErrorMessages changes, change the query flag, remove @Service * feat: use a queryParameter instead of queryFlag * feat: add stopwatch to monitor the performance of RPKI service * feat: add missing return * feat: rpki service should be a @Service and RpkiRoaChecker a component because performance issues, pulling all roas each call * feat: schedule roas * feat: remove blank lines * use SynchonizedIntervalMap to avoid concurrent problems * feat: remove unused import * feat: fix cron expresion * feat: intantiate new interval instead using clean * feat: fix each seconds issue * feat: use nestedIntervalMap instead async map * feat: remove initial inizialisation * feat: change message error and refactor * feat: rename * feat: avoid returning a pair --------- Co-authored-by: maggarwal13 <50230916+maggarwal13@users.noreply.github.com> Co-authored-by: Ed Shryane Co-authored-by: Firdavs Murodov --- .../rest/search/RpkiRoaMessageGenerator.java | 4 ++-- .../WhoisSearchServiceTestIntegration.java | 17 ++++++++------ .../ripe/db/whois/query/QueryMessages.java | 12 +++++----- .../db/whois/common/rpki/RpkiService.java | 23 ++++++++++++++----- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/whois-api/src/main/java/net/ripe/db/whois/api/rest/search/RpkiRoaMessageGenerator.java b/whois-api/src/main/java/net/ripe/db/whois/api/rest/search/RpkiRoaMessageGenerator.java index 8b1c618993..579ca612e9 100644 --- a/whois-api/src/main/java/net/ripe/db/whois/api/rest/search/RpkiRoaMessageGenerator.java +++ b/whois-api/src/main/java/net/ripe/db/whois/api/rest/search/RpkiRoaMessageGenerator.java @@ -55,8 +55,8 @@ private RpslMessage validateRoa(final RpslObject rpslObject){ } return switch (invalidRpkiRoa.getValue()) { - case INVALID_ORIGIN -> new RpslMessage(QueryMessages.roaRouteOriginConflicts(rpslObject.getType().getName(), invalidRpkiRoa.getKey().getAsn())); - case INVALID_PREFIX_LENGTH -> new RpslMessage(QueryMessages.roaRoutePrefixLengthConflicts(rpslObject.getType().getName(), invalidRpkiRoa.getKey().getMaxLength())); + case INVALID_ORIGIN -> new RpslMessage(QueryMessages.roaRouteOriginConflicts(rpslObject.getType().getName(), invalidRpkiRoa.getKey().getMaxLength(), invalidRpkiRoa.getKey().getAsn())); + case INVALID_PREFIX_LENGTH -> new RpslMessage(QueryMessages.roaRoutePrefixLengthConflicts(rpslObject.getType().getName(), invalidRpkiRoa.getKey().getMaxLength(), invalidRpkiRoa.getKey().getAsn())); default -> new RpslMessage(QueryMessages.roaRouteConflicts(rpslObject.getType().getName(), invalidRpkiRoa.getKey().getMaxLength(), invalidRpkiRoa.getKey().getAsn())); }; } diff --git a/whois-api/src/test/java/net/ripe/db/whois/api/rest/WhoisSearchServiceTestIntegration.java b/whois-api/src/test/java/net/ripe/db/whois/api/rest/WhoisSearchServiceTestIntegration.java index 4b5217b3e1..b5eb4b4a50 100644 --- a/whois-api/src/test/java/net/ripe/db/whois/api/rest/WhoisSearchServiceTestIntegration.java +++ b/whois-api/src/test/java/net/ripe/db/whois/api/rest/WhoisSearchServiceTestIntegration.java @@ -1860,7 +1860,7 @@ public void search_less_specific_route_existing_roa_validation_enabled_as_json() assertThat(whoisResources.getWhoisObjects(), hasSize(1)); assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages().get(0).getText(), is("" + - "Warning: this %s object conflicts with an overlapping RPKI ROA with a less specific prefix %s.\n" + + "Warning: this %s object conflicts with an overlapping RPKI ROA with a less specific prefix %s but same origin AS%s.\n" + "As a result an announcement for this prefix may be rejected by many autonomous systems. You should " + "either remove this route: object or update or delete the ROA.\n")); } @@ -1906,7 +1906,7 @@ public void search_route6_roa_origin_mismatch_validation_enabled_as_json() { assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages(), hasSize(1)); assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages().get(0).getText(), is("" + - "Warning: this %s object conflicts with an overlapping RPKI ROA with a different origin AS%s.\n" + + "Warning: this %s object conflicts with an overlapping RPKI ROA with prefix %s but different origin AS%s.\n" + "As a result an announcement for this prefix may be rejected by many autonomous systems. You should " + "either remove this route: object or update or delete the ROA.\n")); } @@ -1931,7 +1931,7 @@ public void search_route_roa_origin_mismatch_validation_enabled_as_json() { assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages(), hasSize(1)); assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages().get(0).getText(), is("" + - "Warning: this %s object conflicts with an overlapping RPKI ROA with a different origin AS%s.\n" + + "Warning: this %s object conflicts with an overlapping RPKI ROA with prefix %s but different origin AS%s.\n" + "As a result an announcement for this prefix may be rejected by many autonomous systems. You should " + "either remove this route: object or update or delete the ROA.\n")); } @@ -1977,7 +1977,7 @@ public void search_route_roa_origin_mismatch_validation_enabled_as_xml() { assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages(), hasSize(1)); assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages().get(0).getText(), is("" + - "Warning: this %s object conflicts with an overlapping RPKI ROA with a different origin AS%s.\n" + + "Warning: this %s object conflicts with an overlapping RPKI ROA with prefix %s but different origin AS%s.\n" + "As a result an announcement for this prefix may be rejected by many autonomous systems. You should " + "either remove this route: object or update or delete the ROA.\n")); } @@ -2063,10 +2063,11 @@ public void search_route6_roa_mismatch_less_specific_as_xml_strings() { " \n" + " \n" + " \n" + " \n" + " \n" + + " \n" + " \n" + " \n" + "\n" + @@ -2271,9 +2272,10 @@ public void search_route6_roa_mismatch_origin_as_xml_strings() { " \n" + " \n" + " \n" + " \n" + + " \n" + " \n" + " \n" + " \n" + @@ -2343,9 +2345,10 @@ public void search_route_roa_mismatch_validation_enabled_as_xml_strings() { " \n" + " \n" + " \n" + " \n" + + " \n" + " \n" + " \n" + " \n" + diff --git a/whois-client/src/main/java/net/ripe/db/whois/query/QueryMessages.java b/whois-client/src/main/java/net/ripe/db/whois/query/QueryMessages.java index 7da3663cfb..25862e5a90 100644 --- a/whois-client/src/main/java/net/ripe/db/whois/query/QueryMessages.java +++ b/whois-client/src/main/java/net/ripe/db/whois/query/QueryMessages.java @@ -60,20 +60,20 @@ public static Message unvalidatedAbuseCShown(final CharSequence key, final CharS "\nAbuse-mailbox validation failed. Please refer to %s for further information.", key, value, orgId); } - public static Message roaRouteOriginConflicts(final String objectType, final long asn){ + public static Message roaRouteOriginConflicts(final String objectType, final int prefix, final long asn){ return new QueryMessage(Type.WARNING, "" - + "Warning: this %s object conflicts with an overlapping RPKI ROA with a different origin AS%s." + + "Warning: this %s object conflicts with an overlapping RPKI ROA with prefix %s but different origin AS%s." + "\n" + "As a result an announcement for this prefix may be rejected by many autonomous systems. You should" + - " either remove this route: object or update or delete the ROA.", objectType, asn); + " either remove this route: object or update or delete the ROA.", objectType, prefix, asn); } - public static Message roaRoutePrefixLengthConflicts(final String objectType, final int prefix){ + public static Message roaRoutePrefixLengthConflicts(final String objectType, final int prefix, final long asn){ return new QueryMessage(Type.WARNING, "" - + "Warning: this %s object conflicts with an overlapping RPKI ROA with a less specific prefix %s." + + "Warning: this %s object conflicts with an overlapping RPKI ROA with a less specific prefix %s but same origin AS%s." + "\n" + "As a result an announcement for this prefix may be rejected by many autonomous systems. You should" + - " either remove this route: object or update or delete the ROA.", objectType, prefix); + " either remove this route: object or update or delete the ROA.", objectType, prefix, asn); } public static Message roaRouteConflicts(final String objectType, final int prefix, final long asn){ diff --git a/whois-commons/src/main/java/net/ripe/db/whois/common/rpki/RpkiService.java b/whois-commons/src/main/java/net/ripe/db/whois/common/rpki/RpkiService.java index ef89ad8adb..3f3b1904b0 100644 --- a/whois-commons/src/main/java/net/ripe/db/whois/common/rpki/RpkiService.java +++ b/whois-commons/src/main/java/net/ripe/db/whois/common/rpki/RpkiService.java @@ -8,6 +8,7 @@ import net.ripe.db.whois.common.ip.Ipv6Resource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import java.util.Collection; @@ -20,24 +21,32 @@ public class RpkiService { private static final Logger LOGGER = LoggerFactory.getLogger(RpkiService.class); - private final NestedIntervalMap> ipv4Tree = new NestedIntervalMap<>(); - private final NestedIntervalMap> ipv6Tree = new NestedIntervalMap<>(); + private NestedIntervalMap> ipv4Tree; + private NestedIntervalMap> ipv6Tree; + + private final RpkiDataProvider rpkiDataProvider; public RpkiService(final RpkiDataProvider rpkiDataProvider) { + this.rpkiDataProvider = rpkiDataProvider; + loadRoas(); + } + + @Scheduled(cron = "0 */15 * * * *") + private void loadRoas() { final List loadedRoas = rpkiDataProvider.loadRoas(); if (loadedRoas != null && !loadedRoas.isEmpty()){ final List roas = loadedRoas.stream() .filter(roa -> roa.getTrustAnchor() != TrustAnchor.UNSUPPORTED) .collect(Collectors.toList()); + buildTrees(roas); LOGGER.info("downloaded {} roas from rpki", roas.size()); - buildTrees(roas, ipv4Tree, ipv6Tree); } } - private void buildTrees(final List roas, - final NestedIntervalMap> ipv4Tree, - final NestedIntervalMap> ipv6Tree) { + private void buildTrees(final List roas) { + final NestedIntervalMap> ipv4Tree = new NestedIntervalMap<>(); + final NestedIntervalMap> ipv6Tree = new NestedIntervalMap<>(); for (Roa roa : roas) { if (isIpv4(roa.getPrefix())) { addRoaToTree(ipv4Tree, Ipv4Resource.parse(roa.getPrefix()), roa); @@ -45,6 +54,8 @@ private void buildTrees(final List roas, addRoaToTree(ipv6Tree, Ipv6Resource.parse(roa.getPrefix()), roa); } } + this.ipv4Tree = ipv4Tree; + this.ipv6Tree = ipv6Tree; } private > void addRoaToTree(final NestedIntervalMap> tree,