diff --git a/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/AutoSubmittedMessageParser.java b/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/AutoSubmittedMessageParser.java new file mode 100644 index 0000000000..4452ad2ada --- /dev/null +++ b/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/AutoSubmittedMessageParser.java @@ -0,0 +1,64 @@ +package net.ripe.db.whois.api.mail.dequeue; + +import jakarta.mail.MessagingException; +import jakarta.mail.internet.MimeMessage; +import net.ripe.db.whois.api.mail.EmailMessageInfo; +import net.ripe.db.whois.api.mail.exception.MailParsingException; +import org.elasticsearch.common.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.Nullable; +import java.util.Collections; + +// Detect automated responses and mark them for deletion. Do not try to parse them as Whois updates (for now). +// TODO: attempt to find the outgoing message-id and failed recipient if possible by parsing the plaintext body +@Component +public class AutoSubmittedMessageParser { + + private static final Logger LOGGER = LoggerFactory.getLogger(AutoSubmittedMessageParser.class); + + private final boolean enabled; + + @Autowired + public AutoSubmittedMessageParser(@Value("${mail.smtp.from:}") final String smtpFrom) { + this.enabled = !Strings.isNullOrEmpty(smtpFrom); + } + + @Nullable + public EmailMessageInfo parse(final MimeMessage message) throws MessagingException, MailParsingException { + if (!enabled) { + return null; + } + + final String autoSubmitted = getHeader(message, "Auto-Submitted"); + if (autoSubmitted != null) { + if (autoSubmitted.contains("auto-generated") || autoSubmitted.contains("auto-replied")) { + return new EmailMessageInfo(Collections.emptyList(), null); + } else { + LOGGER.info("Unexpected Auto-Submitted value {}", autoSubmitted); + } + } + + final String from = getHeader(message, "From"); + if (from != null) { + if (from.toUpperCase().contains("MAILER-DAEMON")) { + return new EmailMessageInfo(Collections.emptyList(), null); + } + } + + return null; + } + + @Nullable + private String getHeader(final MimeMessage message, final String name) throws MessagingException { + final String[] headers = message.getHeader(name); + if (headers != null && headers.length > 0) { + return headers[0]; + } + return null; + } +} diff --git a/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParser.java b/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParser.java index bfe67a9df5..5217be8f55 100644 --- a/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParser.java +++ b/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParser.java @@ -1,5 +1,6 @@ package net.ripe.db.whois.api.mail.dequeue; +import jakarta.mail.BodyPart; import jakarta.mail.MessagingException; import jakarta.mail.Part; import jakarta.mail.internet.AddressException; @@ -7,6 +8,7 @@ import jakarta.mail.internet.InternetAddress; import jakarta.mail.internet.InternetHeaders; import jakarta.mail.internet.MimeMessage; +import jakarta.mail.internet.MimeMultipart; import net.ripe.db.whois.api.mail.EmailMessageInfo; import net.ripe.db.whois.api.mail.exception.MailParsingException; import org.apache.commons.compress.utils.Lists; @@ -32,6 +34,7 @@ public class BouncedMessageParser { private static final Logger LOGGER = LoggerFactory.getLogger(BouncedMessageParser.class); private static final ContentType MULTIPART_REPORT = contentType("multipart/report"); + private static final ContentType MULTIPART_MIXED = contentType("multipart/mixed"); private final boolean enabled; @@ -45,25 +48,52 @@ public BouncedMessageParser(@Value("${mail.smtp.from:}") final String smtpFrom) @Nullable public EmailMessageInfo parse(final MimeMessage message) throws MessagingException, MailParsingException { - if (!enabled || !isMultipartReport(message) ){ + if (!enabled) { return null; } - try { - final MultipartReport multipartReport = multipartReport(message.getContent()); - if (isReportDeliveryStatus(multipartReport)) { - final DeliveryStatus deliveryStatus = deliveryStatus(message); - if (isFailed(deliveryStatus)) { - final MimeMessage returnedMessage = multipartReport.getReturnedMessage(); - final String messageId = getMessageId(returnedMessage.getMessageID()); - final List recipient = extractRecipients(deliveryStatus); - return new EmailMessageInfo(recipient, messageId); + // TODO: refactor to remove duplicate code + + if (isMultipartReport(message)) { + try { + final MultipartReport multipartReport = multipartReport(message.getContent()); + if (isReportDeliveryStatus(multipartReport)) { + final DeliveryStatus deliveryStatus = deliveryStatus(message); + if (isFailed(deliveryStatus)) { + final MimeMessage returnedMessage = getReturnedMessage(multipartReport); + final String messageId = getMessageId(returnedMessage.getMessageID()); + final List recipient = extractRecipients(deliveryStatus); + return new EmailMessageInfo(recipient, messageId); + } + } + } catch (MessagingException | IOException | IllegalStateException ex){ + throw new MailParsingException("Error parsing multipart report"); + } + // multipart report can *only* be a failure + throw new MailParsingException("MultiPart message without failure report"); + } + + if (isMultipartMixed(message)) { + try { + final MimeMultipart multipart = multipart(message.getContent()); + if (isReportDeliveryStatus(multipart)) { + final DeliveryStatus deliveryStatus = deliveryStatus(message); + if (isFailed(deliveryStatus)) { + final MimeMessage returnedMessage = getReturnedMessage(multipart); + final String messageId = getMessageId(returnedMessage.getMessageID()); + final List recipient = extractRecipients(deliveryStatus); + return new EmailMessageInfo(recipient, messageId); + } } + } catch (MessagingException | IOException | IllegalStateException ex) { + LOGGER.error(String.format("%s: %s", ex.getClass().getName(), ex.getMessage()), ex); + throw new MailParsingException("Error parsing multipart report"); } - } catch (MessagingException | IOException | IllegalStateException ex){ - throw new MailParsingException("Error parsing multipart report"); + // do not throw an exception, as whois updates can be multipart/mixed } - throw new MailParsingException("MultiPart message without failure report"); + + // fall through: message is not bounced message + return null; } private boolean isMultipartReport(final MimeMessage message) throws MessagingException { @@ -71,9 +101,50 @@ private boolean isMultipartReport(final MimeMessage message) throws MessagingExc return MULTIPART_REPORT.match(contentType); } - private boolean isReportDeliveryStatus(final MultipartReport multipartReport) throws MessagingException { - final Report report = multipartReport.getReport(); - return ("delivery-status".equals(report.getType())); + private boolean isMultipartMixed(final MimeMessage message) throws MessagingException { + final ContentType contentType = contentType(message.getContentType()); + return MULTIPART_MIXED.match(contentType); + } + + private boolean isReportDeliveryStatus(final MimeMultipart mimeMultipart) throws MessagingException { + final Report report = getReport(mimeMultipart); + return ((report != null) && "delivery-status".equals(report.getType())); + } + + @Nullable + private Report getReport(final MimeMultipart mimeMultipart) throws MessagingException { + if (mimeMultipart.getCount() < 2) { + return null; + } + final BodyPart bodyPart = mimeMultipart.getBodyPart(1); + try { + final Object content = bodyPart.getContent(); + if (!(content instanceof Report)) { + return null; + } else { + return (Report) content; + } + } catch (IOException ex) { + return null; + } + } + + @Nullable + private MimeMessage getReturnedMessage(final MimeMultipart mimeMultipart) throws MessagingException { + if (mimeMultipart.getCount() < 3) { + return null; + } + final BodyPart bodyPart = mimeMultipart.getBodyPart(2); + if (!bodyPart.isMimeType("message/rfc822") && + !bodyPart.isMimeType("text/rfc822-headers")) { + return null; + } else { + try { + return (MimeMessage) bodyPart.getContent(); + } catch (IOException ex) { + return null; + } + } } private MultipartReport multipartReport(final Object content) throws MessagingException { @@ -84,6 +155,14 @@ private MultipartReport multipartReport(final Object content) throws MessagingEx } } + private MimeMultipart multipart(final Object content) throws MessagingException { + if (content instanceof MimeMultipart) { + return (MimeMultipart)content; + } else { + throw new MessagingException("Unexpected content was not multipart/mixed"); + } + } + private DeliveryStatus deliveryStatus(final Part part) throws MessagingException { try { return new DeliveryStatus(part.getInputStream()); diff --git a/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageDequeue.java b/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageDequeue.java index 6e8c0dbf71..c20e380fa5 100644 --- a/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageDequeue.java +++ b/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageDequeue.java @@ -28,7 +28,6 @@ import org.springframework.dao.DataAccessException; import org.springframework.stereotype.Component; -import java.io.IOException; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -206,6 +205,13 @@ private void handleMessage(final String messageId) { mailMessageDao.deleteMessage(messageId); return; } + + final EmailMessageInfo automatedFailureMessage = messageService.getAutomatedFailureMessageInfo(message); + if (automatedFailureMessage != null) { + // TODO: verify and set as undeliverable + mailMessageDao.deleteMessage(messageId); + } + } catch (MailParsingException e){ LOGGER.info("Error detecting bounce detection or unsubscribing for messageId {}", messageId, e); mailMessageDao.deleteMessage(messageId); diff --git a/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageService.java b/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageService.java index d52f1bbae2..80ca641458 100644 --- a/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageService.java +++ b/whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageService.java @@ -15,6 +15,7 @@ import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; +import javax.annotation.Nullable; import java.util.List; @Service @@ -22,6 +23,7 @@ public class MessageService { private static final Logger LOGGER = LoggerFactory.getLogger(MessageService.class); + private final AutoSubmittedMessageParser autoSubmittedMessageParser; private final UnsubscribeMessageParser unsubscribeMessageParser; private final OutgoingMessageDao outgoingMessageDao; private final EmailStatusDao emailStatusDao; @@ -29,27 +31,37 @@ public class MessageService { @Autowired public MessageService( + final AutoSubmittedMessageParser autoSubmittedMessageParser, final UnsubscribeMessageParser unsubscribeMessageParser, final BouncedMessageParser bouncedMessageParser, final OutgoingMessageDao outgoingMessageDao, final EmailStatusDao emailStatusDao) { + this.autoSubmittedMessageParser = autoSubmittedMessageParser; this.unsubscribeMessageParser = unsubscribeMessageParser; this.bouncedMessageParser = bouncedMessageParser; this.outgoingMessageDao = outgoingMessageDao; this.emailStatusDao = emailStatusDao; } + @Nullable public EmailMessageInfo getBouncedMessageInfo(final MimeMessage message) throws MessagingException, MailParsingException { return bouncedMessageParser.parse(message); } + + @Nullable public EmailMessageInfo getUnsubscribedMessageInfo(final MimeMessage message) throws MessagingException, MailParsingException { return unsubscribeMessageParser.parse(message); } - public void verifyAndSetAsUndeliverable(final EmailMessageInfo message){ + @Nullable + public EmailMessageInfo getAutomatedFailureMessageInfo(final MimeMessage message) throws MessagingException, MailParsingException { + return autoSubmittedMessageParser.parse(message); + } + + public void verifyAndSetAsUndeliverable(final EmailMessageInfo message) { final List outgoingEmail = outgoingMessageDao.getEmails(message.messageId()); - if (!isValidMessage(message, outgoingEmail)){ + if (!isValidMessage(message, outgoingEmail)) { return; } @@ -63,8 +75,8 @@ public void verifyAndSetAsUndeliverable(final EmailMessageInfo message){ }); } - public void verifyAndSetAsUnsubscribed(final EmailMessageInfo message){ - if (message.emailAddresses() != null && message.emailAddresses().size() != 1){ + public void verifyAndSetAsUnsubscribed(final EmailMessageInfo message) { + if (message.emailAddresses() != null && message.emailAddresses().size() != 1) { LOGGER.warn("This can not happen, unsubscribe with multiple recipients. messageId: {}", message.messageId()); return; } @@ -72,7 +84,7 @@ public void verifyAndSetAsUnsubscribed(final EmailMessageInfo message){ final String unsubscribeRequestEmail = message.emailAddresses().get(0); final List emails = outgoingMessageDao.getEmails(message.messageId()); - if (emails.stream().noneMatch(email -> email.equalsIgnoreCase(unsubscribeRequestEmail))){ + if (emails.stream().noneMatch(email -> email.equalsIgnoreCase(unsubscribeRequestEmail))) { LOGGER.warn("Couldn't find outgoing message matching {}", message.messageId()); return; } @@ -81,8 +93,8 @@ public void verifyAndSetAsUnsubscribed(final EmailMessageInfo message){ emailStatusDao.createEmailStatus(unsubscribeRequestEmail, EmailStatus.UNSUBSCRIBE); } - private boolean isValidMessage(final EmailMessageInfo message, final List outgoingEmail){ - if (message.messageId() == null || message.emailAddresses() == null || message.emailAddresses().isEmpty()){ + private boolean isValidMessage(final EmailMessageInfo message, final List outgoingEmail) { + if (message.messageId() == null || message.emailAddresses() == null || message.emailAddresses().isEmpty()) { LOGGER.warn("Incorrect message {}", message.messageId()); return false; } @@ -99,7 +111,7 @@ private boolean isValidMessage(final EmailMessageInfo message, final List messageRecipients, final List storedEmails){ + private boolean containsAllCaseInsensitive(final List messageRecipients, final List storedEmails) { final List emailsInLowerCase = storedEmails .stream() .map(String::toLowerCase) diff --git a/whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParserTest.java b/whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParserTest.java index 7479a04133..7818e46bd4 100644 --- a/whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParserTest.java +++ b/whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParserTest.java @@ -24,6 +24,21 @@ public void setup() { this.subject = new BouncedMessageParser("bounce-handler@ripe.net"); } + @Test + public void parse_permanent_delivery_failure_multipart_mixed_rfc822() throws Exception { + final EmailMessageInfo bouncedMessage = subject.parse(MimeMessageProvider.getUpdateMessage("permanentFailureMessageMultipartMixedRfc822.mail")); + + assertThat(bouncedMessage.messageId(), is("XXXXXXXX-64b6-476a-9670-13576e4223c8@ripe.net")); + assertThat(bouncedMessage.emailAddresses(), containsInAnyOrder("lir@example.com")); + } + + @Test + public void parse_multipart_mixed_unsigned() throws Exception { + final EmailMessageInfo bouncedMessage = subject.parse(MimeMessageProvider.getUpdateMessage("multipartMixedUnsigned.mail")); + + assertThat(bouncedMessage, is(nullValue())); + } + @Test public void parse_permanent_delivery_failure_message_rfc822() throws Exception { final EmailMessageInfo bouncedMessage = subject.parse(MimeMessageProvider.getUpdateMessage("permanentFailureMessageRfc822.mail")); diff --git a/whois-api/src/test/resources/testMail/permanentFailureMessageMultipartMixedRfc822.mail b/whois-api/src/test/resources/testMail/permanentFailureMessageMultipartMixedRfc822.mail new file mode 100644 index 0000000000..e230780f89 --- /dev/null +++ b/whois-api/src/test/resources/testMail/permanentFailureMessageMultipartMixedRfc822.mail @@ -0,0 +1,72 @@ +Received: from mahimahi.ripe.net ([193.0.19.1]) + by allealle.ripe.net with esmtps (TLS1.2) tls TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + (Exim 4.94) + id xxxxxx-0004d4-OF + for test-dbm@ripe.net; Fri, 23 Feb 2024 17:29:29 +0200 +Received: from abc1.host.org ([68.232.147.154]:21686) + by mahimahi.ripe.net with esmtps (TLS1.2) tls TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + (Exim 4.94.2) + id xxxxxx-0009P6-50 + for test-dbm@ripe.net; Fri, 23 Feb 2024 17:29:29 +0200 +Subject: Delivery status notification: failed +From: Mailer Daemon +To: test-dbm@ripe.net +Date: Thu, 11 Apr 2024 16:30:54 +0200 (CEST) +MIME-Version: 1.0 +Content-Type: multipart/mixed;boundary="4667782431312076979/mx1.example.com" +Message-ID: <6a8c666fc2acd464@mx1.example.com> + +This is a MIME-encapsulated message. + +--4667782431312076979/mx1.example.com +Content-Description: Notification +Content-Type: text/plain; charset=us-ascii + + Hi! + + This is the MAILER-DAEMON, please DO NOT REPLY to this email. + + An error has occurred while attempting to deliver a message for + the following list of recipients: + +lir@example.com: 524 5.2.4 Mailing list expansion problem: + + Below is a copy of the original message: + +--4667782431312076979/mx1.example.com +Content-Description: Delivery Report +Content-Type: message/delivery-status + +Reporting-MTA: dns; mx1.example.com + +Final-Recipient: rfc822; lir@example.com +Action: failed +Status: 5.0.0 + +--4667782431312076979/mx1.example.com +Content-Description: Message headers +Content-Type: text/rfc822-headers + +Received: from pikapika.ripe.net (pikapika.ripe.net [2001:67c:2e8:11::c100:1315]) + by mx1.example.com (OpenSMTPD) with ESMTPS id 51020dc7 (TLSv1.2:ECDHE-RSA-AES256-GCM-SHA384:256:NO) + for ; + Thu, 11 Apr 2024 16:30:52 +0200 (CEST) +Received: from [193.0.7.249] (helo=busa.ripe.net) + by pikapika.ripe.net with esmtp (Exim 4.97.1) + (envelope-from ) + id 1ruvRv-0000000035A-3tjs + for lir@example.com; + Thu, 11 Apr 2024 14:30:47 +0000 +Date: Thu, 11 Apr 2024 14:30:47 +0000 (UTC) +From: RIPE Database Administration local +Reply-To: lir@example.com +To: lir@example.com +Message-ID: XXXXXXXX-64b6-476a-9670-13576e4223c8@ripe.net +Subject: Notification of RIPE Database changes +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit +Precedence: bulk +Auto-Submitted: auto-generated + +--4667782431312076979/mx1.example.com--