Schedule ROA load each 15 minutes (#1444)

* feat: create new warning for roas check and test for legacy

* feat: add test for REST API calls and little refactor to specify rest messages

* feat: fix ITs

* feat: add more tests

* feat: create the validator

* feat: fix compile error due to previous code

* feat: renaming and deleting white spaces

* feat: fix tests according to last changes

* feat: remove already migrated classes

* feat: change generator by validator

* feat: fix compilation errors

* feat: increase testperformance to 3s

* feat: put timeout back to 2s and fix compilation issue in whoisFixture

* feat: fix compilation issues

* feat: fix ITs

* feat: refactor messages and PR comments

* feat: fix ITs

* feat: refactor

* feat: rename

* feat: rename

* Return 404 on invalid NRTMv4 notification file path (#1422)

* _check_valid_notification_file

* refactor

---------

Co-authored-by: Ed Shryane <eshryane@ripe.net>

* feat: fix logger issue (#1427)

* remove_unused_code (#1429)

* remove_unused_code

* fix_tests

* Revert changes to thread configuration in tests (#1415)

* Handle Multiple Delivery Status Notification Failure Responses for the Same Email Address (#1417)

* feat: remove message after error and catch DuplicateKeyException

* feat: split the changes, just leave Catch in this PR

* feat: catch the exception by email, no for all the emails. Turn the warn into debug

* feat: put white space after try and before bracket

* feat: just loop once

* Added test to make sure the message doesn't get stuck

* Fix test (rs.next() behaves the same if there is a result row or not)

---------

Co-authored-by: Ed Shryane <eshryane@ripe.net>

* Fix tests (#1433)

* fix tests

* fix tests

* reduce mariadb innodb memory usage to 1GB (#1439)

* Set memory limit for test elasticsearch instance (#1436)

* Set memory limit for test elasticsearch instance

* Move ES memlimit to ENV variable

* Reduce elasticsearch memory limit

---------

Co-authored-by: Ed Shryane <eshryane@ripe.net>

* feat: renaming, revert ErrorMessages changes, change the query flag, remove @service

* feat: use a queryParameter instead of queryFlag

* feat: add stopwatch to monitor the performance of RPKI service

* feat: add missing return

* feat: rpki service should be a @service and RpkiRoaChecker a component because performance issues, pulling all roas each call

* feat: schedule roas

* feat: remove blank lines

* use SynchonizedIntervalMap to avoid concurrent problems

* feat: remove unused import

* feat: fix cron expresion

* feat: intantiate new interval instead using clean

* feat: fix each seconds issue

* feat: use nestedIntervalMap instead async map

* feat: remove initial inizialisation

* feat: change message error and refactor

* feat: rename

* feat: avoid returning a pair

---------

Co-authored-by: maggarwal13 <50230916+maggarwal13@users.noreply.github.com>
Co-authored-by: Ed Shryane <eshryane@ripe.net>
Co-authored-by: Firdavs Murodov <fmurodov@ripe.net>

whois-api/src/main/java/net/ripe/db/whois/api/rest/search/RpkiRoaMessageGenerator.java

@@ -55,8 +55,8 @@ private RpslMessage validateRoa(final RpslObject rpslObject){
         }
 
         return switch (invalidRpkiRoa.getValue()) {
-            case INVALID_ORIGIN -> new RpslMessage(QueryMessages.roaRouteOriginConflicts(rpslObject.getType().getName(), invalidRpkiRoa.getKey().getAsn()));
-            case INVALID_PREFIX_LENGTH -> new RpslMessage(QueryMessages.roaRoutePrefixLengthConflicts(rpslObject.getType().getName(), invalidRpkiRoa.getKey().getMaxLength()));
+            case INVALID_ORIGIN -> new RpslMessage(QueryMessages.roaRouteOriginConflicts(rpslObject.getType().getName(), invalidRpkiRoa.getKey().getMaxLength(), invalidRpkiRoa.getKey().getAsn()));
+            case INVALID_PREFIX_LENGTH -> new RpslMessage(QueryMessages.roaRoutePrefixLengthConflicts(rpslObject.getType().getName(), invalidRpkiRoa.getKey().getMaxLength(), invalidRpkiRoa.getKey().getAsn()));
             default -> new RpslMessage(QueryMessages.roaRouteConflicts(rpslObject.getType().getName(), invalidRpkiRoa.getKey().getMaxLength(), invalidRpkiRoa.getKey().getAsn()));
         };
     }

whois-api/src/test/java/net/ripe/db/whois/api/rest/WhoisSearchServiceTestIntegration.java

@@ -1860,7 +1860,7 @@ public void search_less_specific_route_existing_roa_validation_enabled_as_json()
         assertThat(whoisResources.getWhoisObjects(), hasSize(1));
 
         assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages().get(0).getText(), is("" +
-                "Warning: this %s object conflicts with an overlapping RPKI ROA with a less specific prefix %s.\n" +
+                "Warning: this %s object conflicts with an overlapping RPKI ROA with a less specific prefix %s but same origin AS%s.\n" +
                 "As a result an announcement for this prefix may be rejected by many autonomous systems. You should " +
                 "either remove this route: object or update or delete the ROA.\n"));
     }
@@ -1906,7 +1906,7 @@ public void search_route6_roa_origin_mismatch_validation_enabled_as_json() {
 
         assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages(), hasSize(1));
         assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages().get(0).getText(), is("" +
-                "Warning: this %s object conflicts with an overlapping RPKI ROA with a different origin AS%s.\n" +
+                "Warning: this %s object conflicts with an overlapping RPKI ROA with prefix %s but different origin AS%s.\n" +
                 "As a result an announcement for this prefix may be rejected by many autonomous systems. You should " +
                 "either remove this route: object or update or delete the ROA.\n"));
     }
@@ -1931,7 +1931,7 @@ public void search_route_roa_origin_mismatch_validation_enabled_as_json() {
 
         assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages(), hasSize(1));
         assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages().get(0).getText(), is("" +
-                "Warning: this %s object conflicts with an overlapping RPKI ROA with a different origin AS%s.\n" +
+                "Warning: this %s object conflicts with an overlapping RPKI ROA with prefix %s but different origin AS%s.\n" +
                 "As a result an announcement for this prefix may be rejected by many autonomous systems. You should " +
                 "either remove this route: object or update or delete the ROA.\n"));
     }
@@ -1977,7 +1977,7 @@ public void search_route_roa_origin_mismatch_validation_enabled_as_xml() {
 
         assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages(), hasSize(1));
         assertThat(whoisResources.getWhoisObjects().get(0).getObjectMessages().getMessages().get(0).getText(), is("" +
-                "Warning: this %s object conflicts with an overlapping RPKI ROA with a different origin AS%s.\n" +
+                "Warning: this %s object conflicts with an overlapping RPKI ROA with prefix %s but different origin AS%s.\n" +
                 "As a result an announcement for this prefix may be rejected by many autonomous systems. You should " +
                 "either remove this route: object or update or delete the ROA.\n"));
     }
@@ -2063,10 +2063,11 @@ public void search_route6_roa_mismatch_less_specific_as_xml_strings() {
                 "    </attributes>\n" +
                 "    <objectmessages>\n" +
                 "        <objectmessage severity=\"Warning\" text=\"Warning: this %s object conflicts with an " +
-                "overlapping RPKI ROA with a less specific prefix %s.&#xA;As a result an announcement for this prefix may be" +
+                "overlapping RPKI ROA with a less specific prefix %s but same origin AS%s.&#xA;As a result an announcement for this prefix may be" +
                 " rejected by many autonomous systems. You should either remove this route: object or update or delete the ROA.&#xA;\">\n" +
                 "            <args value=\"route6\"/>\n" +
                 "            <args value=\"32\"/>\n" +
+                "            <args value=\"52511\"/>\n" +
                 "        </objectmessage>\n" +
                 "    </objectmessages>\n" +
                 "</object>\n" +
@@ -2271,9 +2272,10 @@ public void search_route6_roa_mismatch_origin_as_xml_strings() {
                 "    </attributes>\n" +
                 "    <objectmessages>\n" +
                 "        <objectmessage severity=\"Warning\" text=\"Warning: this %s object conflicts with an " +
-                "overlapping RPKI ROA with a different origin AS%s.&#xA;As a result an announcement for this prefix " +
+                "overlapping RPKI ROA with prefix %s but different origin AS%s.&#xA;As a result an announcement for this prefix " +
                 "may be rejected by many autonomous systems. You should either remove this route: object or update or delete the ROA.&#xA;\">\n" +
                 "            <args value=\"route6\"/>\n" +
+                "            <args value=\"32\"/>\n" +
                 "            <args value=\"52511\"/>\n" +
                 "        </objectmessage>\n" +
                 "    </objectmessages>\n" +
@@ -2343,9 +2345,10 @@ public void search_route_roa_mismatch_validation_enabled_as_xml_strings() {
                 "    </attributes>\n" +
                 "    <objectmessages>\n" +
                 "        <objectmessage severity=\"Warning\" text=\"Warning: this %s object conflicts with an " +
-                "overlapping RPKI ROA with a different origin AS%s.&#xA;As a result an announcement for this prefix " +
+                "overlapping RPKI ROA with prefix %s but different origin AS%s.&#xA;As a result an announcement for this prefix " +
                 "may be rejected by many autonomous systems. You should either remove this route: object or update or delete the ROA.&#xA;\">\n" +
                 "            <args value=\"route\"/>\n" +
+                "            <args value=\"16\"/>\n" +
                 "            <args value=\"6505\"/>\n" +
                 "        </objectmessage>\n" +
                 "    </objectmessages>\n" +

whois-client/src/main/java/net/ripe/db/whois/query/QueryMessages.java

@@ -60,20 +60,20 @@ public static Message unvalidatedAbuseCShown(final CharSequence key, final CharS
                 "\nAbuse-mailbox validation failed. Please refer to %s for further information.", key, value, orgId);
     }
 
-    public static Message roaRouteOriginConflicts(final String objectType, final long asn){
+    public static Message roaRouteOriginConflicts(final String objectType, final int prefix, final long asn){
         return new QueryMessage(Type.WARNING, ""
-                + "Warning: this %s object conflicts with an overlapping RPKI ROA with a different origin AS%s."
+                + "Warning: this %s object conflicts with an overlapping RPKI ROA with prefix %s but different origin AS%s."
                 + "\n"
                 + "As a result an announcement for this prefix may be rejected by many autonomous systems. You should" +
-                " either remove this route: object or update or delete the ROA.", objectType, asn);
+                " either remove this route: object or update or delete the ROA.", objectType, prefix, asn);
     }
 
-    public static Message roaRoutePrefixLengthConflicts(final String objectType, final int prefix){
+    public static Message roaRoutePrefixLengthConflicts(final String objectType, final int prefix, final long asn){
         return new QueryMessage(Type.WARNING, ""
-                + "Warning: this %s object conflicts with an overlapping RPKI ROA with a less specific prefix %s."
+                + "Warning: this %s object conflicts with an overlapping RPKI ROA with a less specific prefix %s but same origin AS%s."
                 + "\n"
                 + "As a result an announcement for this prefix may be rejected by many autonomous systems. You should" +
-                " either remove this route: object or update or delete the ROA.", objectType, prefix);
+                " either remove this route: object or update or delete the ROA.", objectType, prefix, asn);
     }
 
     public static Message roaRouteConflicts(final String objectType, final int prefix, final long asn){

whois-commons/src/main/java/net/ripe/db/whois/common/rpki/RpkiService.java

@@ -8,6 +8,7 @@
 import net.ripe.db.whois.common.ip.Ipv6Resource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
 import java.util.Collection;
@@ -20,31 +21,41 @@ public class RpkiService {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RpkiService.class);
 
-    private final NestedIntervalMap<Ipv4Resource, Set<Roa>> ipv4Tree = new NestedIntervalMap<>();
-    private final NestedIntervalMap<Ipv6Resource, Set<Roa>> ipv6Tree = new NestedIntervalMap<>();
+    private NestedIntervalMap<Ipv4Resource, Set<Roa>> ipv4Tree;
+    private NestedIntervalMap<Ipv6Resource, Set<Roa>> ipv6Tree;
+
+    private final RpkiDataProvider rpkiDataProvider;
 
     public RpkiService(final RpkiDataProvider rpkiDataProvider) {
+        this.rpkiDataProvider = rpkiDataProvider;
+        loadRoas();
+    }
+
+    @Scheduled(cron = "0 */15 * * * *")
+    private void loadRoas() {
         final List<Roa> loadedRoas = rpkiDataProvider.loadRoas();
         if (loadedRoas != null && !loadedRoas.isEmpty()){
             final List<Roa> roas = loadedRoas.stream()
                     .filter(roa -> roa.getTrustAnchor() != TrustAnchor.UNSUPPORTED)
                     .collect(Collectors.toList());
 
+            buildTrees(roas);
             LOGGER.info("downloaded {} roas from rpki", roas.size());
-            buildTrees(roas, ipv4Tree, ipv6Tree);
         }
     }
 
-    private void buildTrees(final List<Roa> roas,
-                            final NestedIntervalMap<Ipv4Resource, Set<Roa>> ipv4Tree,
-                            final NestedIntervalMap<Ipv6Resource, Set<Roa>> ipv6Tree) {
+    private void buildTrees(final List<Roa> roas) {
+        final NestedIntervalMap<Ipv4Resource, Set<Roa>> ipv4Tree = new NestedIntervalMap<>();
+        final NestedIntervalMap<Ipv6Resource, Set<Roa>> ipv6Tree = new NestedIntervalMap<>();
         for (Roa roa : roas) {
             if (isIpv4(roa.getPrefix())) {
                 addRoaToTree(ipv4Tree, Ipv4Resource.parse(roa.getPrefix()), roa);
             } else {
                 addRoaToTree(ipv6Tree, Ipv6Resource.parse(roa.getPrefix()), roa);
             }
         }
+        this.ipv4Tree = ipv4Tree;
+        this.ipv6Tree = ipv6Tree;
     }
 
     private <T extends IpInterval<T>> void addRoaToTree(final NestedIntervalMap<T, Set<Roa>> tree,