Add a cache for IMAP message in the database

Co-authored-by: Roeland Jago Douma <roeland@famdouma.nl>
Signed-off-by: Christoph Wurst <christoph@winzerhof-wurst.at>
Signed-off-by: Roeland Jago Douma <roeland@famdouma.nl>

appinfo/info.xml

@@ -12,7 +12,7 @@
 - **🙈 We’re not reinventing the wheel!** Based on the great [Horde](http://horde.org) libraries.
 - **📬 Want to host your own mail server?** We don’t have to reimplement this as you could set up [Mail-in-a-Box](https://mailinabox.email)!
 	]]></description>
-	<version>1.1.2</version>
+	<version>1.2.0</version>
 	<licence>agpl</licence>
 	<author>Christoph Wurst</author>
 	<author>Jan-Christoph Borchardt</author>
@@ -34,6 +34,7 @@
 	<repair-steps>
 		<post-migration>
 			<step>OCA\Mail\Migration\FixCollectedAddresses</step>
+			<step>OCA\Mail\Migration\FixAccountSyncs</step>
 			<step>OCA\Mail\Migration\MakeItineraryExtractorExecutable</step>
 			<step>OCA\Mail\Migration\MigrateProvisioningConfig</step>
 			<step>OCA\Mail\Migration\ProvisionAccounts</step>
@@ -43,6 +44,7 @@
 		<command>OCA\Mail\Command\CreateAccount</command>
 		<command>OCA\Mail\Command\DiagnoseAccount</command>
 		<command>OCA\Mail\Command\ExportAccount</command>
+		<command>OCA\Mail\Command\SyncAccount</command>
 	</commands>
 	<settings>
 		<admin>OCA\Mail\Settings\AdminSettings</admin>

lib/Address.php

@@ -1,4 +1,5 @@
 <?php
+
 declare(strict_types=1);
 
 /**
@@ -30,6 +31,11 @@
 
 class Address implements JsonSerializable {
 
+	public const TYPE_FROM = 0;
+	public const TYPE_TO = 1;
+	public const TYPE_CC = 2;
+	public const TYPE_BCC = 3;
+
 	/** @var Horde_Mail_Rfc822_Address */
 	private $wrapped;
 
@@ -40,27 +46,27 @@ class Address implements JsonSerializable {
 	public function __construct($label, $email) {
 		$this->wrapped = new Horde_Mail_Rfc822_Address($email);
 		// If no label is set we use the email
-		if ($label !== $email && !is_null($label)) {
+		if ($label !== $email && $label !== null) {
 			$this->wrapped->personal = $label;
 		}
 	}
 
 	/**
-	 * @return string
+	 * @return string|null
 	 */
-	public function getLabel(): string {
+	public function getLabel(): ?string {
 		$personal = $this->wrapped->personal;
-		if (is_null($personal)) {
+		if ($personal === null) {
 			// Fallback
 			return $this->getEmail();
 		}
 		return $personal;
 	}
 
 	/**
-	 * @return string
+	 * @return string|null
 	 */
-	public function getEmail(): string {
+	public function getEmail(): ?string {
 		return $this->wrapped->bare_address;
 	}
 

lib/AddressList.php

@@ -1,4 +1,4 @@
-<?php
+<?php declare(strict_types=1);
 
 /**
  * @copyright 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
@@ -69,6 +69,12 @@ public static function fromHorde(Horde_Mail_Rfc822_List $hordeList) {
 		return new AddressList($addresses);
 	}
 
+	public static function fromRow(array $recipient): self {
+		return new self([
+			new Address($recipient['label'], $recipient['email'])
+		]);
+	}
+
 	/**
 	 * Get first element
 	 *

lib/AppInfo/BootstrapSingleton.php

@@ -48,7 +48,7 @@
 use OCA\Mail\Service\Group\NextcloudGroupService;
 use OCA\Mail\Service\Group\ContactsGroupService;
 use OCA\Mail\Service\MailManager;
-use OCA\Mail\Service\MailSearch;
+use OCA\Mail\Service\Search\MailSearch;
 use OCA\Mail\Service\MailTransmission;
 use OCA\Mail\Service\UserPreferenceSevice;
 use OCP\AppFramework\IAppContainer;

lib/BackgroundJob/SyncJob.php

@@ -0,0 +1,78 @@
+<?php declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2019, Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @author Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Mail\BackgroundJob;
+
+use OCA\Mail\Service\AccountService;
+use OCA\Mail\Service\SyncService;
+use OCP\AppFramework\Db\DoesNotExistException;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\IJobList;
+use OCP\BackgroundJob\TimedJob;
+use OCP\ILogger;
+
+class SyncJob extends TimedJob {
+
+	/** @var AccountService */
+	private $accountService;
+	/** @var SyncService */
+	private $syncService;
+	/** @var ILogger */
+	private $logger;
+	/** @var IJobList */
+	private $jobList;
+
+	public function __construct(ITimeFactory $time,
+								AccountService $accountService,
+								SyncService $syncService,
+								ILogger $logger,
+								IJobList $jobList) {
+		parent::__construct($time);
+
+		$this->accountService = $accountService;
+		$this->syncService = $syncService;
+		$this->logger = $logger;
+
+		$this->setInterval(3600);
+		$this->jobList = $jobList;
+	}
+
+	protected function run($argument) {
+		$accountId = (int)$argument['accountId'];
+
+		try {
+			$account = $this->accountService->findById($accountId);
+		} catch (DoesNotExistException $e) {
+			$this->logger->debug('Could not find account <' . $accountId . '> removing from jobs');
+			$this->jobList->remove(self::class, $argument);
+			return;
+		}
+
+		try {
+			$this->syncService->syncAccount($account);
+		} catch (\Exception $e) {
+			$this->logger->logException($e);
+		}
+	}
+
+}

lib/Command/SyncAccount.php

@@ -0,0 +1,73 @@
+<?php declare(strict_types=1);
+
+/**
+ * @copyright 2019 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @author 2019 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Mail\Command;
+
+use OCA\Mail\Db\MailboxMapper;
+use OCA\Mail\Service\AccountService;
+use OCA\Mail\Service\SyncService;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class SyncAccount extends Command {
+
+	const ARGUMENT_ACCOUNT_ID = 'account-id';
+	const OPTION_FORCE = 'force';
+
+	/** @var AccountService */
+	private $accountService;
+
+	/** @var MailboxMapper */
+	private $mailboxMapper;
+
+	/** @var SyncService */
+	private $syncService;
+
+	public function __construct(AccountService $service,
+								MailboxMapper $mailboxMapper,
+								SyncService $syncService) {
+		parent::__construct();
+
+		$this->accountService = $service;
+		$this->mailboxMapper = $mailboxMapper;
+		$this->syncService = $syncService;
+	}
+
+	protected function configure() {
+		$this->setName('mail:account:sync');
+		$this->setDescription('Synchronize an IMAP account');
+		$this->addArgument(self::ARGUMENT_ACCOUNT_ID, InputArgument::REQUIRED);
+		$this->addOption(self::OPTION_FORCE, 'f', InputOption::VALUE_NONE);
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output) {
+		$accountId = (int)$input->getArgument(self::ARGUMENT_ACCOUNT_ID);
+		$force = $input->getOption(self::OPTION_FORCE);
+
+		$account = $this->accountService->findById($accountId);
+		$this->syncService->syncAccount($account, $force);
+	}
+}

lib/Contracts/IMailManager.php

@@ -68,17 +68,6 @@ public function getFolderStats(Account $account, string $folderId): FolderStats;
 	 */
 	public function getMessage(Account $account, string $mailbox, int $id, bool $loadBody = false): IMAPMessage;
 
-	/**
-	 * @param Account
-	 * @param SyncRequest $syncRequest
-	 *
-	 * @return SyncResponse
-	 *
-	 * @throws ClientException
-	 * @throws ServiceException
-	 */
-	public function syncMessages(Account $account, SyncRequest $syncRequest): SyncResponse;
-
 	/**
 	 * @param Account $sourceAccount
 	 * @param string $sourceFolderId

lib/Controller/FoldersController.php

@@ -25,6 +25,10 @@
 
 namespace OCA\Mail\Controller;
 
+use Horde_Imap_Client;
+use OCA\Mail\Exception\MailboxNotCachedException;
+use OCA\Mail\Exception\ServiceException;
+use OCA\Mail\Service\SyncService;
 use function base64_decode;
 use function is_array;
 use OCA\Mail\Contracts\IMailManager;
@@ -47,20 +51,28 @@ class FoldersController extends Controller {
 	/**  @var IMailManager */
 	private $mailManager;
 
+	/** @var SyncService */
+	private $syncService;
+
 	/**
 	 * @param string $appName
 	 * @param IRequest $request
 	 * @param AccountService $accountService
 	 * @param string $UserId
 	 * @param IMailManager $mailManager
 	 */
-	public function __construct(string $appName, IRequest $request,
-								AccountService $accountService, $UserId, IMailManager $mailManager) {
+	public function __construct(string $appName,
+								IRequest $request,
+								AccountService $accountService,
+								$UserId,
+								IMailManager $mailManager,
+								SyncService $syncService) {
 		parent::__construct($appName, $request);
 
 		$this->accountService = $accountService;
 		$this->currentUserId = $UserId;
 		$this->mailManager = $mailManager;
+		$this->syncService = $syncService;
 	}
 
 	/**
@@ -91,15 +103,28 @@ public function index(int $accountId): JSONResponse {
 	 * @param string $syncToken
 	 * @param int[] $uids
 	 * @return JSONResponse
+	 * @throws ServiceException
 	 */
-	public function sync(int $accountId, string $folderId, string $syncToken, array $uids = []): JSONResponse {
+	public function sync(int $accountId, string $folderId, array $uids): JSONResponse {
 		$account = $this->accountService->find($this->currentUserId, $accountId);
 
-		if (empty($accountId) || empty($folderId) || empty($syncToken) || !is_array($uids)) {
+		if (empty($accountId) || empty($folderId) || !is_array($uids)) {
 			return new JSONResponse(null, Http::STATUS_BAD_REQUEST);
 		}
 
-		$syncResponse = $this->mailManager->syncMessages($account, new SyncRequest(base64_decode($folderId), $syncToken, $uids));
+		try {
+			$syncResponse = $this->syncService->syncMailbox(
+				$account,
+				base64_decode($folderId),
+				Horde_Imap_Client::SYNC_NEWMSGSUIDS | Horde_Imap_Client::SYNC_FLAGSUIDS | Horde_Imap_Client::SYNC_VANISHEDUIDS,
+				array_map(function($uid) {
+					return (int) $uid;
+				}, $uids),
+				true
+			);
+		} catch (MailboxNotCachedException $e) {
+			return new JSONResponse(null, Http::STATUS_PRECONDITION_REQUIRED);
+		}
 
 		return new JSONResponse($syncResponse);
 	}

lib/Controller/MessagesController.php

@@ -40,6 +40,7 @@
 use OCA\Mail\Service\AccountService;
 use OCA\Mail\Service\IMailBox;
 use OCA\Mail\Service\ItineraryService;
+use OCA\Mail\Service\SyncService;
 use OCP\AppFramework\Controller;
 use OCP\AppFramework\Db\DoesNotExistException;
 use OCP\AppFramework\Http;
@@ -69,6 +70,9 @@ class MessagesController extends Controller {
 	/** @var ItineraryService */
 	private $itineraryService;
 
+	/** @var SyncService */
+	private $syncService;
+
 	/** @var string */
 	private $currentUserId;
 
@@ -104,6 +108,7 @@ public function __construct(string $appName,
 								IMailManager $mailManager,
 								IMailSearch $mailSearch,
 								ItineraryService $itineraryService,
+								SyncService $syncService,
 								string $UserId,
 								$userFolder,
 								ILogger $logger,
@@ -116,6 +121,7 @@ public function __construct(string $appName,
 		$this->mailManager = $mailManager;
 		$this->mailSearch = $mailSearch;
 		$this->itineraryService = $itineraryService;
+		$this->syncService = $syncService;
 		$this->currentUserId = $UserId;
 		$this->userFolder = $userFolder;
 		$this->logger = $logger;
@@ -144,6 +150,11 @@ public function index(int $accountId, string $folderId, int $cursor = null, stri
 			return new JSONResponse(null, Http::STATUS_FORBIDDEN);
 		}
 
+		$this->syncService->ensurePopulated(
+			$account,
+			base64_decode($folderId)
+		);
+
 		$this->logger->debug("loading messages of folder <$folderId>");
 
 		return new JSONResponse(