Skip to content

Commit

Permalink
Handle Multipart/mixed Bounces and Auto-submitted Messages (#1490)
Browse files Browse the repository at this point in the history 
* Multipart/mixed messages may also contain a delivery failure report

* Detect auto-generated responses and delete them, do not attempt to parse them as Whois Update messages

* Correct Auto-Submitted value

* Fix formatting, updated TODO

* Added TODO refactor BouncedMessageParser
  • Loading branch information
@eshryane
eshryane committed Jun 28, 2024
1 parent 3ad74d4 commit 30e26dc
Show file tree
Hide file tree
Showing 6 changed files with 273 additions and 25 deletions.
64 changes: 64 additions & 0 deletions whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/AutoSubmittedMessageParser.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package net.ripe.db.whois.api.mail.dequeue;

import jakarta.mail.MessagingException;
import jakarta.mail.internet.MimeMessage;
import net.ripe.db.whois.api.mail.EmailMessageInfo;
import net.ripe.db.whois.api.mail.exception.MailParsingException;
import org.elasticsearch.common.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import javax.annotation.Nullable;
import java.util.Collections;

// Detect automated responses and mark them for deletion. Do not try to parse them as Whois updates (for now).
// TODO: attempt to find the outgoing message-id and failed recipient if possible by parsing the plaintext body
@Component
public class AutoSubmittedMessageParser {

private static final Logger LOGGER = LoggerFactory.getLogger(AutoSubmittedMessageParser.class);

private final boolean enabled;

@Autowired
public AutoSubmittedMessageParser(@Value("${mail.smtp.from:}") final String smtpFrom) {
this.enabled = !Strings.isNullOrEmpty(smtpFrom);
}

@Nullable
public EmailMessageInfo parse(final MimeMessage message) throws MessagingException, MailParsingException {
if (!enabled) {
return null;
}

final String autoSubmitted = getHeader(message, "Auto-Submitted");
if (autoSubmitted != null) {
if (autoSubmitted.contains("auto-generated") || autoSubmitted.contains("auto-replied")) {
return new EmailMessageInfo(Collections.emptyList(), null);
} else {
LOGGER.info("Unexpected Auto-Submitted value {}", autoSubmitted);
}
}

final String from = getHeader(message, "From");
if (from != null) {
if (from.toUpperCase().contains("MAILER-DAEMON")) {
return new EmailMessageInfo(Collections.emptyList(), null);
}
}

return null;
}

@Nullable
private String getHeader(final MimeMessage message, final String name) throws MessagingException {
final String[] headers = message.getHeader(name);
if (headers != null && headers.length > 0) {
return headers[0];
}
return null;
}
}
111 changes: 95 additions & 16 deletions whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParser.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
package net.ripe.db.whois.api.mail.dequeue;

import jakarta.mail.BodyPart;
import jakarta.mail.MessagingException;
import jakarta.mail.Part;
import jakarta.mail.internet.AddressException;
import jakarta.mail.internet.ContentType;
import jakarta.mail.internet.InternetAddress;
import jakarta.mail.internet.InternetHeaders;
import jakarta.mail.internet.MimeMessage;
import jakarta.mail.internet.MimeMultipart;
import net.ripe.db.whois.api.mail.EmailMessageInfo;
import net.ripe.db.whois.api.mail.exception.MailParsingException;
import org.apache.commons.compress.utils.Lists;
Expand All @@ -32,6 +34,7 @@ public class BouncedMessageParser {
private static final Logger LOGGER = LoggerFactory.getLogger(BouncedMessageParser.class);

private static final ContentType MULTIPART_REPORT = contentType("multipart/report");
private static final ContentType MULTIPART_MIXED = contentType("multipart/mixed");

private final boolean enabled;

Expand All @@ -45,35 +48,103 @@ public BouncedMessageParser(@Value("${mail.smtp.from:}") final String smtpFrom)

@Nullable
public EmailMessageInfo parse(final MimeMessage message) throws MessagingException, MailParsingException {
if (!enabled || !isMultipartReport(message) ){
if (!enabled) {
return null;
}

try {
final MultipartReport multipartReport = multipartReport(message.getContent());
if (isReportDeliveryStatus(multipartReport)) {
final DeliveryStatus deliveryStatus = deliveryStatus(message);
if (isFailed(deliveryStatus)) {
final MimeMessage returnedMessage = multipartReport.getReturnedMessage();
final String messageId = getMessageId(returnedMessage.getMessageID());
final List<String> recipient = extractRecipients(deliveryStatus);
return new EmailMessageInfo(recipient, messageId);
// TODO: refactor to remove duplicate code

if (isMultipartReport(message)) {
try {
final MultipartReport multipartReport = multipartReport(message.getContent());
if (isReportDeliveryStatus(multipartReport)) {
final DeliveryStatus deliveryStatus = deliveryStatus(message);
if (isFailed(deliveryStatus)) {
final MimeMessage returnedMessage = getReturnedMessage(multipartReport);
final String messageId = getMessageId(returnedMessage.getMessageID());
final List<String> recipient = extractRecipients(deliveryStatus);
return new EmailMessageInfo(recipient, messageId);
}
}
} catch (MessagingException | IOException | IllegalStateException ex){
throw new MailParsingException("Error parsing multipart report");
}
// multipart report can *only* be a failure
throw new MailParsingException("MultiPart message without failure report");
}

if (isMultipartMixed(message)) {
try {
final MimeMultipart multipart = multipart(message.getContent());
if (isReportDeliveryStatus(multipart)) {
final DeliveryStatus deliveryStatus = deliveryStatus(message);
if (isFailed(deliveryStatus)) {
final MimeMessage returnedMessage = getReturnedMessage(multipart);
final String messageId = getMessageId(returnedMessage.getMessageID());
final List<String> recipient = extractRecipients(deliveryStatus);
return new EmailMessageInfo(recipient, messageId);
}
}
} catch (MessagingException | IOException | IllegalStateException ex) {
LOGGER.error(String.format("%s: %s", ex.getClass().getName(), ex.getMessage()), ex);
throw new MailParsingException("Error parsing multipart report");
}
} catch (MessagingException | IOException | IllegalStateException ex){
throw new MailParsingException("Error parsing multipart report");
// do not throw an exception, as whois updates can be multipart/mixed
}
throw new MailParsingException("MultiPart message without failure report");

// fall through: message is not bounced message
return null;
}

private boolean isMultipartReport(final MimeMessage message) throws MessagingException {
final ContentType contentType = contentType(message.getContentType());
return MULTIPART_REPORT.match(contentType);
}

private boolean isReportDeliveryStatus(final MultipartReport multipartReport) throws MessagingException {
final Report report = multipartReport.getReport();
return ("delivery-status".equals(report.getType()));
private boolean isMultipartMixed(final MimeMessage message) throws MessagingException {
final ContentType contentType = contentType(message.getContentType());
return MULTIPART_MIXED.match(contentType);
}

private boolean isReportDeliveryStatus(final MimeMultipart mimeMultipart) throws MessagingException {
final Report report = getReport(mimeMultipart);
return ((report != null) && "delivery-status".equals(report.getType()));
}

@Nullable
private Report getReport(final MimeMultipart mimeMultipart) throws MessagingException {
if (mimeMultipart.getCount() < 2) {
return null;
}
final BodyPart bodyPart = mimeMultipart.getBodyPart(1);
try {
final Object content = bodyPart.getContent();
if (!(content instanceof Report)) {
return null;
} else {
return (Report) content;
}
} catch (IOException ex) {
return null;
}
}

@Nullable
private MimeMessage getReturnedMessage(final MimeMultipart mimeMultipart) throws MessagingException {
if (mimeMultipart.getCount() < 3) {
return null;
}
final BodyPart bodyPart = mimeMultipart.getBodyPart(2);
if (!bodyPart.isMimeType("message/rfc822") &&
!bodyPart.isMimeType("text/rfc822-headers")) {
return null;
} else {
try {
return (MimeMessage) bodyPart.getContent();
} catch (IOException ex) {
return null;
}
}
}

private MultipartReport multipartReport(final Object content) throws MessagingException {
Expand All @@ -84,6 +155,14 @@ private MultipartReport multipartReport(final Object content) throws MessagingEx
}
}

private MimeMultipart multipart(final Object content) throws MessagingException {
if (content instanceof MimeMultipart) {
return (MimeMultipart)content;
} else {
throw new MessagingException("Unexpected content was not multipart/mixed");
}
}

private DeliveryStatus deliveryStatus(final Part part) throws MessagingException {
try {
return new DeliveryStatus(part.getInputStream());
Expand Down
8 changes: 7 additions & 1 deletion whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageDequeue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import org.springframework.dao.DataAccessException;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
Expand Down Expand Up @@ -206,6 +205,13 @@ private void handleMessage(final String messageId) {
mailMessageDao.deleteMessage(messageId);
return;
}

final EmailMessageInfo automatedFailureMessage = messageService.getAutomatedFailureMessageInfo(message);
if (automatedFailureMessage != null) {
// TODO: verify and set as undeliverable
mailMessageDao.deleteMessage(messageId);
}

} catch (MailParsingException e){
LOGGER.info("Error detecting bounce detection or unsubscribing for messageId {}", messageId, e);
mailMessageDao.deleteMessage(messageId);
Expand Down
28 changes: 20 additions & 8 deletions whois-api/src/main/java/net/ripe/db/whois/api/mail/dequeue/MessageService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,53 @@
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;

import javax.annotation.Nullable;
import java.util.List;

@Service
public class MessageService {

private static final Logger LOGGER = LoggerFactory.getLogger(MessageService.class);

private final AutoSubmittedMessageParser autoSubmittedMessageParser;
private final UnsubscribeMessageParser unsubscribeMessageParser;
private final OutgoingMessageDao outgoingMessageDao;
private final EmailStatusDao emailStatusDao;
private final BouncedMessageParser bouncedMessageParser;

@Autowired
public MessageService(
final AutoSubmittedMessageParser autoSubmittedMessageParser,
final UnsubscribeMessageParser unsubscribeMessageParser,
final BouncedMessageParser bouncedMessageParser,
final OutgoingMessageDao outgoingMessageDao,
final EmailStatusDao emailStatusDao) {
this.autoSubmittedMessageParser = autoSubmittedMessageParser;
this.unsubscribeMessageParser = unsubscribeMessageParser;
this.bouncedMessageParser = bouncedMessageParser;
this.outgoingMessageDao = outgoingMessageDao;
this.emailStatusDao = emailStatusDao;
}

@Nullable
public EmailMessageInfo getBouncedMessageInfo(final MimeMessage message) throws MessagingException, MailParsingException {
return bouncedMessageParser.parse(message);
}

@Nullable
public EmailMessageInfo getUnsubscribedMessageInfo(final MimeMessage message) throws MessagingException, MailParsingException {
return unsubscribeMessageParser.parse(message);
}

public void verifyAndSetAsUndeliverable(final EmailMessageInfo message){
@Nullable
public EmailMessageInfo getAutomatedFailureMessageInfo(final MimeMessage message) throws MessagingException, MailParsingException {
return autoSubmittedMessageParser.parse(message);
}

public void verifyAndSetAsUndeliverable(final EmailMessageInfo message) {
final List<String> outgoingEmail = outgoingMessageDao.getEmails(message.messageId());

if (!isValidMessage(message, outgoingEmail)){
if (!isValidMessage(message, outgoingEmail)) {
return;
}

Expand All @@ -63,16 +75,16 @@ public void verifyAndSetAsUndeliverable(final EmailMessageInfo message){
});
}

public void verifyAndSetAsUnsubscribed(final EmailMessageInfo message){
if (message.emailAddresses() != null && message.emailAddresses().size() != 1){
public void verifyAndSetAsUnsubscribed(final EmailMessageInfo message) {
if (message.emailAddresses() != null && message.emailAddresses().size() != 1) {
LOGGER.warn("This can not happen, unsubscribe with multiple recipients. messageId: {}", message.messageId());
return;
}

final String unsubscribeRequestEmail = message.emailAddresses().get(0);
final List<String> emails = outgoingMessageDao.getEmails(message.messageId());

if (emails.stream().noneMatch(email -> email.equalsIgnoreCase(unsubscribeRequestEmail))){
if (emails.stream().noneMatch(email -> email.equalsIgnoreCase(unsubscribeRequestEmail))) {
LOGGER.warn("Couldn't find outgoing message matching {}", message.messageId());
return;
}
Expand All @@ -81,8 +93,8 @@ public void verifyAndSetAsUnsubscribed(final EmailMessageInfo message){
emailStatusDao.createEmailStatus(unsubscribeRequestEmail, EmailStatus.UNSUBSCRIBE);
}

private boolean isValidMessage(final EmailMessageInfo message, final List<String> outgoingEmail){
if (message.messageId() == null || message.emailAddresses() == null || message.emailAddresses().isEmpty()){
private boolean isValidMessage(final EmailMessageInfo message, final List<String> outgoingEmail) {
if (message.messageId() == null || message.emailAddresses() == null || message.emailAddresses().isEmpty()) {
LOGGER.warn("Incorrect message {}", message.messageId());
return false;
}
Expand All @@ -99,7 +111,7 @@ private boolean isValidMessage(final EmailMessageInfo message, final List<String
return true;
}

private boolean containsAllCaseInsensitive(final List<String> messageRecipients, final List<String> storedEmails){
private boolean containsAllCaseInsensitive(final List<String> messageRecipients, final List<String> storedEmails) {
final List<String> emailsInLowerCase = storedEmails
.stream()
.map(String::toLowerCase)
Expand Down
15 changes: 15 additions & 0 deletions whois-api/src/test/java/net/ripe/db/whois/api/mail/dequeue/BouncedMessageParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,21 @@ public void setup() {
this.subject = new BouncedMessageParser("bounce-handler@ripe.net");
}

@Test
public void parse_permanent_delivery_failure_multipart_mixed_rfc822() throws Exception {
final EmailMessageInfo bouncedMessage = subject.parse(MimeMessageProvider.getUpdateMessage("permanentFailureMessageMultipartMixedRfc822.mail"));

assertThat(bouncedMessage.messageId(), is("XXXXXXXX-64b6-476a-9670-13576e4223c8@ripe.net"));
assertThat(bouncedMessage.emailAddresses(), containsInAnyOrder("lir@example.com"));
}

@Test
public void parse_multipart_mixed_unsigned() throws Exception {
final EmailMessageInfo bouncedMessage = subject.parse(MimeMessageProvider.getUpdateMessage("multipartMixedUnsigned.mail"));

assertThat(bouncedMessage, is(nullValue()));
}

@Test
public void parse_permanent_delivery_failure_message_rfc822() throws Exception {
final EmailMessageInfo bouncedMessage = subject.parse(MimeMessageProvider.getUpdateMessage("permanentFailureMessageRfc822.mail"));
Expand Down
Loading

0 comments on commit 30e26dc

Please sign in to comment.