Remove Message If Bounce Detection Or Unsubscribe Fails (#1418)

* feat: remove the email if bounce message failed

* feat: refactor to just return and delete the message when it is multipart or textplain message and doesn't meet the requirements

* Allow MimeMessageHelper as parameter in MailGatewaySmtp (#1420)

* feat: little refactor in mailGatewaySmtp

* feat: from can be different from whois-internal as well

* feat: from should be always from mail.smtp

* Roa Rpki Validator From Internal (#1419)

* feat: move rao rpki validator common code from whois internal to whois

* feat: move code into common module, add component and service and return an error if optional property is empty

* feat: return null and add nullable annotation

* feat: put less roas

* Allow MimeMessageHelper as parameter in MailGatewaySmtp (#1420)

* feat: little refactor in mailGatewaySmtp

* feat: from can be different from whois-internal as well

* feat: from should be always from mail.smtp

* feat: fix a TODO typo and IT should work even if roas are not specified

* Move RPKI Validator Dummifier to Correct Module (#1421)

* feat: move dummifier class to correct module

* feat: move roas.json to common

* Final-Recipient: RFC822 is case insensitive (#1423)

* Final-Recipient: RFC822 is case insensitive.

* There may also be a space separator

* feat: move all message logic from whois-internal to whois (#1424)

* feat: runtime exception fix when using this module (#1425)

* feat: do no return error if roas are not loaded (#1426)

* Return 404 on invalid NRTMv4 notification file path (#1422)

* _check_valid_notification_file

* refactor

---------

Co-authored-by: Ed Shryane <eshryane@ripe.net>

* feat: fix logger issue (#1427)

* remove_unused_code (#1429)

* remove_unused_code

* fix_tests

* Revert changes to thread configuration in tests (#1415)

* Handle Multiple Delivery Status Notification Failure Responses for the Same Email Address (#1417)

* feat: remove message after error and catch DuplicateKeyException

* feat: split the changes, just leave Catch in this PR

* feat: catch the exception by email, no for all the emails. Turn the warn into debug

* feat: put white space after try and before bracket

* feat: just loop once

* Added test to make sure the message doesn't get stuck

* Fix test (rs.next() behaves the same if there is a result row or not)

---------

Co-authored-by: Ed Shryane <eshryane@ripe.net>

* feat: remove disabled and catch illegalStateException

* feat: remove unused imports and add serialId to the exception

* feat: return null in case unsubscription parsing do not succed

* feat: fix tests

* feat: log as info instead error when bouce detection or unsubscribing message fails, log as error when there is some unexpected error in the message

---------

Co-authored-by: Ed Shryane <eshryane@ripe.net>
Co-authored-by: maggarwal13 <50230916+maggarwal13@users.noreply.github.com>

whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParser.java

@@ -8,6 +8,7 @@
 import jakarta.mail.internet.InternetHeaders;
 import jakarta.mail.internet.MimeMessage;
 import net.ripe.db.whois.api.mail.EmailMessageInfo;
+import net.ripe.db.whois.api.mail.exception.MailParsingException;
 import org.apache.commons.compress.utils.Lists;
 import org.eclipse.angus.mail.dsn.DeliveryStatus;
 import org.eclipse.angus.mail.dsn.MultipartReport;
@@ -43,8 +44,12 @@ public BouncedMessageParser(@Value("${mail.smtp.from:}") final String smtpFrom)
     }
 
     @Nullable
-    public EmailMessageInfo parse(final MimeMessage message) throws MessagingException, IOException {
-        if (enabled && isMultipartReport(message)) {
+    public EmailMessageInfo parse(final MimeMessage message) throws MessagingException, MailParsingException {
+        if (!enabled || !isMultipartReport(message) ){
+            return null;
+        }
+
+        try {
             final MultipartReport multipartReport = multipartReport(message.getContent());
             if (isReportDeliveryStatus(multipartReport)) {
                 final DeliveryStatus deliveryStatus = deliveryStatus(message);
@@ -55,8 +60,10 @@ public EmailMessageInfo parse(final MimeMessage message) throws MessagingExcepti
                     return new EmailMessageInfo(recipient, messageId);
                 }
             }
+        } catch (MessagingException | IOException | IllegalStateException ex){
+            throw new MailParsingException("Error parsing multipart report");
         }
-        return null;
+        throw new MailParsingException("MultiPart message without failure report");
     }
 
     private boolean isMultipartReport(final MimeMessage message) throws MessagingException {

whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageDequeue.java

@@ -7,6 +7,7 @@
 import net.ripe.db.whois.api.mail.EmailMessageInfo;
 import net.ripe.db.whois.api.mail.MailMessage;
 import net.ripe.db.whois.api.mail.dao.MailMessageDao;
+import net.ripe.db.whois.api.mail.exception.MailParsingException;
 import net.ripe.db.whois.common.ApplicationService;
 import net.ripe.db.whois.common.DateTimeProvider;
 import net.ripe.db.whois.common.MaintenanceMode;
@@ -205,7 +206,15 @@ private void handleMessage(final String messageId) {
                 mailMessageDao.deleteMessage(messageId);
                 return;
             }
+        } catch (MailParsingException e){
+            LOGGER.info("Error detecting bounce detection or unsubscribing for messageId {}", messageId, e);
+            mailMessageDao.deleteMessage(messageId);
+            return;
+        } catch (MessagingException ex) {
+            LOGGER.error("There was some kind of error processing the message {}", messageId, ex);
+        }
 
+        try {
             loggerContext.init(getMessageIdLocalPart(message));
             try {
                 handleMessageInContext(messageId, message);

whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageService.java

@@ -4,6 +4,7 @@
 import jakarta.mail.MessagingException;
 import jakarta.mail.internet.MimeMessage;
 import net.ripe.db.whois.api.mail.EmailMessageInfo;
+import net.ripe.db.whois.api.mail.exception.MailParsingException;
 import net.ripe.db.whois.common.dao.EmailStatusDao;
 import net.ripe.db.whois.common.dao.OutgoingMessageDao;
 import net.ripe.db.whois.common.mail.EmailStatus;
@@ -14,7 +15,6 @@
 import org.springframework.dao.DuplicateKeyException;
 import org.springframework.stereotype.Service;
 
-import java.io.IOException;
 import java.util.List;
 
 @Service
@@ -39,10 +39,10 @@ public MessageService(
         this.emailStatusDao = emailStatusDao;
     }
 
-    public EmailMessageInfo getBouncedMessageInfo(final MimeMessage message) throws MessagingException, IOException {
+    public EmailMessageInfo getBouncedMessageInfo(final MimeMessage message) throws MessagingException, MailParsingException {
         return bouncedMessageParser.parse(message);
     }
-    public EmailMessageInfo getUnsubscribedMessageInfo(final MimeMessage message) throws MessagingException {
+    public EmailMessageInfo getUnsubscribedMessageInfo(final MimeMessage message) throws MessagingException, MailParsingException {
         return unsubscribeMessageParser.parse(message);
     }
 

whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/UnsubscribeMessageParser.java

@@ -7,6 +7,7 @@
 import jakarta.mail.internet.InternetAddress;
 import jakarta.mail.internet.MimeMessage;
 import net.ripe.db.whois.api.mail.EmailMessageInfo;
+import net.ripe.db.whois.api.mail.exception.MailParsingException;
 import org.elasticsearch.common.Strings;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -32,14 +33,21 @@ public UnsubscribeMessageParser(@Value("${mail.smtp.from:}") final String smtpFr
     }
 
     @Nullable
-    public EmailMessageInfo parse(final MimeMessage message) throws MessagingException {
-        if (enabled && isTextPlain(message)) {
+    public EmailMessageInfo parse(final MimeMessage message) throws MessagingException, MailParsingException {
+        if (!enabled || !isTextPlain(message)){
+            return null;
+        }
+
+        try {
             final String messageId = getMessageIdFromSubject(message);
             final String from = getFrom(message);
             if ((messageId != null) && (from != null)) {
                 return new EmailMessageInfo(List.of(from), messageId);
             }
+        } catch (MessagingException | IllegalStateException ex){
+            throw new MailParsingException("Error parsing text plain unsubscribe message");
         }
+
         return null;
     }
 

whois-api/src/main/java/net/ripe/db/whois/api/mail/exception/MailParsingException.java

@@ -0,0 +1,16 @@
+package net.ripe.db.whois.api.mail.exception;
+
+import org.eclipse.angus.mail.iap.ParsingException;
+
+import java.io.Serial;
+
+public class MailParsingException extends ParsingException {
+
+    @Serial
+    private static final long serialVersionUID = -1263525433251664387L;
+
+    public MailParsingException(final String message) {
+        super(message);
+    }
+
+}

whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParserTest.java

@@ -2,6 +2,7 @@
 
 import net.ripe.db.whois.api.MimeMessageProvider;
 import net.ripe.db.whois.api.mail.EmailMessageInfo;
+import net.ripe.db.whois.api.mail.exception.MailParsingException;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -67,16 +68,19 @@ public void parse_permanent_delivery_failure_message_rfc822_headers_real() throw
 
     @Test
     public void parse_permanent_delivery_failure_without_message_id() {
-        final IllegalStateException e = assertThrows(IllegalStateException.class, () -> {
+        final MailParsingException e = assertThrows(MailParsingException.class, () -> {
             subject.parse(MimeMessageProvider.getUpdateMessage("permanentFailureWithoutMessageId.mail"));
         });
 
-        assertThat(e.getMessage(), is("No Message-Id header"));
+        assertThat(e.getMessage(), is("Error parsing multipart report"));
     }
 
     @Test
-    public void parse_delayed_delivery() throws Exception {
-        assertThat(subject.parse(MimeMessageProvider.getUpdateMessage("messageDelayedRfc822Headers.mail")), is(nullValue()));
+    public void parse_delayed_delivery() {
+        final MailParsingException exception = assertThrows(MailParsingException.class, () -> {
+            subject.parse(MimeMessageProvider.getUpdateMessage("messageDelayedRfc822Headers.mail"));
+        });
+        assertThat(exception.getMessage(), is("MultiPart message without failure report"));
     }
 
     @Test

whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/MailBounceTestIntegration.java

@@ -7,7 +7,6 @@
 import org.awaitility.Awaitility;
 import org.hamcrest.Matchers;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -231,7 +230,6 @@ public void full_email_address_is_normalised() throws MessagingException, IOExce
         assertThat(mailSenderStub.anyMoreMessages(), is(false));
     }
 
-    @Disabled("TODO: [ES] message gets stuck")
     @Test
     public void invalid_email_do_not_causes_address_to_be_marked_as_undeliverable() {
         final MimeMessage message = MimeMessageProvider.getUpdateMessage("permanentFailureWithoutMessageId.mail");

whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/MessageServiceTestIntegration.java

@@ -4,14 +4,12 @@
 import jakarta.mail.internet.MimeMessage;
 import net.ripe.db.whois.api.MimeMessageProvider;
 import net.ripe.db.whois.api.mail.EmailMessageInfo;
+import net.ripe.db.whois.api.mail.exception.MailParsingException;
 import net.ripe.db.whois.update.mail.MailSenderStub;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
-
-import java.io.IOException;
-
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.CoreMatchers.nullValue;
@@ -44,7 +42,7 @@ public void setup() {
     }
 
     @Test
-    public void testNoBouncedEmailFromCorrectEmail() throws MessagingException, IOException {
+    public void testNoBouncedEmailFromCorrectEmail() throws MessagingException, MailParsingException {
         final String role =
                 "role:        dummy role\n" +
                         "address:       Singel 258\n" +
@@ -67,7 +65,7 @@ public void testNoBouncedEmailFromCorrectEmail() throws MessagingException, IOEx
     }
 
     @Test
-    public void testBouncedEmailFromIncorrectEmail() throws MessagingException, IOException {
+    public void testBouncedEmailFromIncorrectEmail() throws MessagingException, MailParsingException {
         insertOutgoingMessageId("XXXXXXXX-5AE3-4C58-8E3F-860327BA955D@ripe.net", "enduser@ripe.net");
         final MimeMessage message = MimeMessageProvider.getUpdateMessage("permanentFailureMessageRfc822.mail");
 
@@ -79,7 +77,7 @@ public void testBouncedEmailFromIncorrectEmail() throws MessagingException, IOEx
     }
 
     @Test
-    public void testBouncedEmailFromCorrectEmail() throws MessagingException, IOException {
+    public void testBouncedEmailFromCorrectEmail() throws MessagingException, MailParsingException {
         insertOutgoingMessageId("XXXXXXXX-5AE3-4C58-8E3F-860327BA955D@ripe.net", BOUNCED_MAIL_RECIPIENT);
         final MimeMessage message = MimeMessageProvider.getUpdateMessage("permanentFailureMessageRfc822.mail");
 
@@ -91,7 +89,7 @@ public void testBouncedEmailFromCorrectEmail() throws MessagingException, IOExce
     }
 
     @Test
-    public void testBouncedFailureRecipientFromCorrectEmail() throws MessagingException, IOException {
+    public void testBouncedFailureRecipientFromCorrectEmail() throws MessagingException, MailParsingException {
         insertOutgoingMessageId("XXXXXXXX-5AE3-4C58-8E3F-860327BA955D@ripe.net", BOUNCED_MAIL_RECIPIENT);
         insertOutgoingMessageId("XXXXXXXX-5AE3-4C58-8E3F-860327BA955D@ripe.net", ANOTHER_BOUNCED_MAIL_RECIPIENT);
 
@@ -106,7 +104,7 @@ public void testBouncedFailureRecipientFromCorrectEmail() throws MessagingExcept
     }
 
     @Test
-    public void testBouncedMultipleFailurePerRecipientFromCorrectEmail() throws MessagingException, IOException {
+    public void testBouncedMultipleFailurePerRecipientFromCorrectEmail() throws MessagingException, MailParsingException {
         insertOutgoingMessageId("XXXXXXXX-8553-47AB-A79B-A9896A2DFBAC@ripe.net", BOUNCED_MAIL_RECIPIENT);
         insertOutgoingMessageId("XXXXXXXX-8553-47AB-A79B-A9896A2DFBAC@ripe.net", ANOTHER_BOUNCED_MAIL_RECIPIENT);
 
@@ -121,7 +119,7 @@ public void testBouncedMultipleFailurePerRecipientFromCorrectEmail() throws Mess
     }
 
     @Test
-    public void testUnsubscribedEmailFromCorrectEmail() throws MessagingException, IOException {
+    public void testUnsubscribedEmailFromCorrectEmail() throws MessagingException, MailParsingException {
         insertOutgoingMessageId("8b8ed6c0-f9cc-4a5f-afbb-fde079b94f44@ripe.net", UNSUBSCRIBED_MAIL_RECIPIENT);
         final MimeMessage message = MimeMessageProvider.getUpdateMessage("unsubscribeAppleMail.mail");
 
@@ -136,7 +134,7 @@ public void testUnsubscribedEmailFromCorrectEmail() throws MessagingException, I
     }
 
     @Test
-    public void testUnsubscribedEmailFromCorrectEmailIsCaseInsensitive() throws MessagingException, IOException {
+    public void testUnsubscribedEmailFromCorrectEmailIsCaseInsensitive() throws MessagingException, MailParsingException {
         insertOutgoingMessageId("8b8ed6c0-f9cc-4a5f-afbb-fde079b94f44@ripe.net", "EnDuseR@ripe.net");
         final MimeMessage message = MimeMessageProvider.getUpdateMessage("unsubscribeAppleMail.mail");