Add imip processing

Signed-off-by: Anna Larch <anna@nextcloud.com>

appinfo/info.xml

@@ -35,6 +35,7 @@
 	<background-jobs>
 		<job>OCA\Mail\BackgroundJob\CleanupJob</job>
 		<job>OCA\Mail\BackgroundJob\OutboxWorkerJob</job>
+		<job>OCA\Mail\BackgroundJob\IMipMessageJob</job>
 	</background-jobs>
 	<repair-steps>
 		<post-migration>

lib/BackgroundJob/IMipMessageJob.php

@@ -0,0 +1,55 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * @copyright 2022 Anna Larch <anna.larch@gmx.net>
+ *
+ * @author 2022 Anna Larch <anna.larch@gmx.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Mail\BackgroundJob;
+
+use OCA\Mail\Service\IMipService;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\TimedJob;
+use function defined;
+use function method_exists;
+
+class IMipMessageJob extends TimedJob {
+	private IMipService $iMipService;
+
+	public function __construct(ITimeFactory $time,
+								IMipService $iMipService) {
+		parent::__construct($time);
+
+		// Run once per hour
+		$this->setInterval(60 * 60);
+		/**
+		 * @todo remove checks with 24+
+		 */
+		if (defined('\OCP\BackgroundJob\IJob::TIME_SENSITIVE') && method_exists($this, 'setTimeSensitivity')) {
+			$this->setTimeSensitivity(self::TIME_SENSITIVE);
+		}
+		$this->iMipService = $iMipService;
+	}
+
+	protected function run($argument): void {
+		$this->iMipService->process();
+	}
+}

lib/Db/Message.php

@@ -76,6 +76,12 @@
  * @method null|string getPreviewText()
  * @method void setUpdatedAt(int $time)
  * @method int getUpdatedAt()
+ * @method bool isImipMessage()
+ * @method void setImipMessage(bool $imipMessage)
+ * @method bool isImipProcessed()
+ * @method void setImipProcessed(bool $imipProcessed)
+ * @method bool isImipError()
+ * @method void setImipError(bool $imipError)
  */
 class Message extends Entity implements JsonSerializable {
 	private const MUTABLE_FLAGS = [
@@ -114,6 +120,9 @@ class Message extends Entity implements JsonSerializable {
 	protected $flagImportant = false;
 	protected $flagMdnsent;
 	protected $previewText;
+	protected $imipMessage = false;
+	protected $imipProcessed = false;
+	protected $imipError = false;
 
 	/** @var AddressList */
 	private $from;
@@ -152,6 +161,9 @@ public function __construct() {
 		$this->addType('flagImportant', 'boolean');
 		$this->addType('flagMdnsent', 'boolean');
 		$this->addType('updatedAt', 'integer');
+		$this->addType('imipMessage', 'boolean');
+		$this->addType('imipProcessed', 'boolean');
+		$this->addType('imipError', 'boolean');
 	}
 
 	/**
@@ -316,6 +328,7 @@ function (Tag $tag) {
 			'inReplyTo' => $this->getInReplyTo(),
 			'references' => empty($this->getReferences()) ? null: json_decode($this->getReferences(), true),
 			'threadRootId' => $this->getThreadRootId(),
+			'imipMessage' => $this->isImipMessage(),
 			'previewText' => $this->getPreviewText(),
 		];
 	}

lib/Db/MessageMapper.php

@@ -312,7 +312,6 @@ public function insertBulk(Account $account, Message ...$messages): void {
 				$qb1->setParameter('flag_notjunk', $message->getFlagNotjunk(), IQueryBuilder::PARAM_BOOL);
 				$qb1->setParameter('flag_important', $message->getFlagImportant(), IQueryBuilder::PARAM_BOOL);
 				$qb1->setParameter('flag_mdnsent', $message->getFlagMdnsent(), IQueryBuilder::PARAM_BOOL);
-
 				$qb1->execute();
 
 				$messageId = $qb1->getLastInsertId();
@@ -380,6 +379,7 @@ public function updateBulk(Account $account, bool $permflagsEnabled, Message ...
 				->set('flag_mdnsent', $query->createParameter('flag_mdnsent'))
 				->set('flag_important', $query->createParameter('flag_important'))
 				->set('updated_at', $query->createNamedParameter($this->timeFactory->getTime()))
+				->set('imip_message', $query->createParameter('imip_message'))
 				->where($query->expr()->andX(
 					$query->expr()->eq('uid', $query->createParameter('uid')),
 					$query->expr()->eq('mailbox_id', $query->createParameter('mailbox_id'))
@@ -407,6 +407,7 @@ public function updateBulk(Account $account, bool $permflagsEnabled, Message ...
 					$query->setParameter('flag_notjunk', $message->getFlagNotjunk(), IQueryBuilder::PARAM_BOOL);
 					$query->setParameter('flag_mdnsent', $message->getFlagMdnsent(), IQueryBuilder::PARAM_BOOL);
 					$query->setParameter('flag_important', $message->getFlagImportant(), IQueryBuilder::PARAM_BOOL);
+					$query->setParameter('imip_message', $message->isImipMessage(), IQueryBuilder::PARAM_BOOL);
 					$query->execute();
 					$perf->step('Updated message ' . $message->getId());
 				}
@@ -482,6 +483,7 @@ public function updatePreviewDataBulk(Message ...$messages): array {
 				->set('preview_text', $query->createParameter('preview_text'))
 				->set('structure_analyzed', $query->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
 				->set('updated_at', $query->createNamedParameter($this->timeFactory->getTime(), IQueryBuilder::PARAM_INT))
+				->set('imip_message', $query->createParameter('imip_message'))
 				->where($query->expr()->andX(
 					$query->expr()->eq('uid', $query->createParameter('uid')),
 					$query->expr()->eq('mailbox_id', $query->createParameter('mailbox_id'))
@@ -501,7 +503,52 @@ public function updatePreviewDataBulk(Message ...$messages): array {
 					$message->getPreviewText() === null ? null : mb_strcut($message->getPreviewText(), 0, 255),
 					$message->getPreviewText() === null ? IQueryBuilder::PARAM_NULL : IQueryBuilder::PARAM_STR
 				);
+				$query->setParameter('imip_message', $message->isImipMessage(), IQueryBuilder::PARAM_BOOL);
+
+				$query->execute();
+			}
+
+			$this->db->commit();
+		} catch (Throwable $e) {
+			// Make sure to always roll back, otherwise the outer code runs in a failed transaction
+			$this->db->rollBack();
+
+			throw $e;
+		}
+
+		return $messages;
+	}
+
+	/**
+	 * @param Message ...$messages
+	 *
+	 * @return Message[]
+	 */
+	public function updateImipData(Message ...$messages): array {
+		$this->db->beginTransaction();
+
+		try {
+			$query = $this->db->getQueryBuilder();
+			$query->update($this->getTableName())
+				->set('imip_message', $query->createParameter('imip_message'))
+				->set('imip_error', $query->createParameter('imip_error'))
+				->set('imip_processed', $query->createParameter('imip_processed'))
+				->where($query->expr()->andX(
+					$query->expr()->eq('uid', $query->createParameter('uid')),
+					$query->expr()->eq('mailbox_id', $query->createParameter('mailbox_id'))
+				));
 
+			foreach ($messages as $message) {
+				if (empty($message->getUpdatedFields())) {
+					// Micro optimization
+					continue;
+				}
+
+				$query->setParameter('uid', $message->getUid(), IQueryBuilder::PARAM_INT);
+				$query->setParameter('mailbox_id', $message->getMailboxId(), IQueryBuilder::PARAM_INT);
+				$query->setParameter('imip_message', $message->isImipMessage(), IQueryBuilder::PARAM_BOOL);
+				$query->setParameter('imip_error', $message->isImipError(), IQueryBuilder::PARAM_BOOL);
+				$query->setParameter('imip_processed', $message->isImipProcessed(), IQueryBuilder::PARAM_BOOL);
 				$query->execute();
 			}
 
@@ -1228,4 +1275,25 @@ public function resetInReplyTo(): int {
 			);
 		return $update->execute();
 	}
+
+	/**
+	 * Get all iMIP messages from the last two weeks
+	 * that haven't been processed yet
+	 * @return Message[]
+	 */
+	public function findIMipMessagesAscending(): array {
+		$time = $this->timeFactory->getTime() - 60 * 60 * 24 * 14;
+		$qb = $this->db->getQueryBuilder();
+
+		$select = $qb->select('*')
+			->from($this->getTableName())
+			->where(
+				$qb->expr()->eq('imip_message', $qb->createNamedParameter(true, IQueryBuilder::PARAM_BOOL), IQueryBuilder::PARAM_BOOL),
+				$qb->expr()->eq('imip_processed', $qb->createNamedParameter(false, IQueryBuilder::PARAM_BOOL), IQueryBuilder::PARAM_BOOL),
+				$qb->expr()->eq('imip_error', $qb->createNamedParameter(false, IQueryBuilder::PARAM_BOOL), IQueryBuilder::PARAM_BOOL),
+				$qb->expr()->gt('sent_at', $qb->createNamedParameter($time, IQueryBuilder::PARAM_INT)),
+			)->orderBy('sent_at', 'ASC'); // make sure we don't process newer messages first
+
+		return $this->findEntities($select);
+	}
 }

lib/IMAP/MessageMapper.php

@@ -666,6 +666,10 @@ public function getBodyStructureData(Horde_Imap_Client_Socket $client,
 										 array $uids): array {
 		$structureQuery = new Horde_Imap_Client_Fetch_Query();
 		$structureQuery->structure();
+		$structureQuery->headerText([
+			'cache' => true,
+			'peek' => true,
+		]);
 
 		$structures = $client->fetch($mailbox, $structureQuery, [
 			'ids' => new Horde_Imap_Client_Ids($uids),
@@ -674,12 +678,19 @@ public function getBodyStructureData(Horde_Imap_Client_Socket $client,
 		return array_map(function (Horde_Imap_Client_Data_Fetch $fetchData) use ($mailbox, $client) {
 			$hasAttachments = false;
 			$text = '';
+			$isImipMessage = false;
 
 			$structure = $fetchData->getStructure();
-			foreach ($structure as $part) {
-				if ($part instanceof Horde_Mime_Part && $part->isAttachment()) {
+			foreach ($structure->getParts() as $part) {
+				if ($part->isAttachment()) {
 					$hasAttachments = true;
-					break;
+				}
+				$bodyParts = $part->getParts();
+				foreach ($bodyParts as $bodyPart) {
+					$contentParameters = $bodyPart->getAllContentTypeParameters();
+					if ($bodyPart->getType() === 'text/calendar' && isset($contentParameters['method'])) {
+						$isImipMessage = true;
+					}
 				}
 			}
 
@@ -734,7 +745,7 @@ public function getBodyStructureData(Horde_Imap_Client_Socket $client,
 				return new MessageStructureData($hasAttachments, $textBody);
 			}
 
-			return new MessageStructureData($hasAttachments, $text);
+			return new MessageStructureData($hasAttachments, $text, $isImipMessage);
 		}, iterator_to_array($structures->getIterator()));
 	}
 }

lib/IMAP/MessageStructureData.php

@@ -33,10 +33,15 @@ class MessageStructureData {
 	/** @var string */
 	private $previewText;
 
+	/** @var bool */
+	private $isImipMessage;
+
 	public function __construct(bool $hasAttachments,
-								string $previewText) {
+								string $previewText,
+								bool $isImipMessage) {
 		$this->hasAttachments = $hasAttachments;
 		$this->previewText = $previewText;
+		$this->isImipMessage = $isImipMessage;
 	}
 
 	public function hasAttachments(): bool {
@@ -46,4 +51,8 @@ public function hasAttachments(): bool {
 	public function getPreviewText(): string {
 		return $this->previewText;
 	}
+
+	public function isImipMessage(): bool {
+		return $this->isImipMessage;
+	}
 }

lib/IMAP/PreviewEnhancer.php

@@ -110,6 +110,7 @@ public function process(Account $account, Mailbox $mailbox, array $messages): ar
 			$message->setFlagAttachments($structureData->hasAttachments());
 			$message->setPreviewText($structureData->getPreviewText());
 			$message->setStructureAnalyzed(true);
+			$message->setImipMessage($structureData->isImipMessage());
 
 			return $message;
 		}, $messages));

lib/Migration/Version1140Date20220630113356.php

@@ -0,0 +1,67 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2022 Anna Larch <anna.larch@gmx.net>
+ *
+ * @author Anna Larch <anna.larch@gmx.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Mail\Migration;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version1140Date20220630113356 extends SimpleMigrationStep {
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 * @param array $options
+	 * @return null|ISchemaWrapper
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		$messagesTable = $schema->getTable('mail_messages');
+		if (!$messagesTable->hasColumn('imip_message')) {
+			$messagesTable->addColumn('imip_message', 'boolean', [
+				'notnull' => false,
+				'default' => false,
+			]);
+		}
+		if (!$messagesTable->hasColumn('imip_processed')) {
+			$messagesTable->addColumn('imip_processed', 'boolean', [
+				'notnull' => false,
+				'default' => false,
+			]);
+		}
+		if (!$messagesTable->hasColumn('imip_error')) {
+			$messagesTable->addColumn('imip_error', 'boolean', [
+				'notnull' => false,
+				'default' => false,
+			]);
+		}
+		return $schema;
+	}
+}

lib/Model/IMAPMessage.php

@@ -732,6 +732,22 @@ public function setInReplyTo(string $id) {
 		throw new Exception('not implemented');
 	}
 
+	/**
+	 * @return AddressList
+	 */
+	public function getReplyTo() {
+		return AddressList::fromHorde($this->getEnvelope()->reply_to);
+	}
+
+	/**
+	 * @param string $id
+	 *
+	 * @return void
+	 */
+	public function setReplyTo(string $id) {
+		throw new Exception('not implemented');
+	}
+
 	/**
 	 * Cast all values from an IMAP message into the correct DB format
 	 *
@@ -779,6 +795,9 @@ public function toDbMessage(int $mailboxId, MailAccount $account): Message {
 		$msg->setFlagImportant(in_array('$important', $flags, true) || in_array('$labelimportant', $flags, true) || in_array(Tag::LABEL_IMPORTANT, $flags, true));
 		$msg->setFlagAttachments(false);
 		$msg->setFlagMdnsent(in_array(Horde_Imap_Client::FLAG_MDNSENT, $flags, true));
+		if(!empty($this->scheduling)) {
+			$msg->setImipMessage(true);
+		}
 
 		$allowed = [
 			Horde_Imap_Client::FLAG_ANSWERED,