Email Issues (#4827)

* New functionality and a big bug fix

* Added some missing config for tests

* Load configrepo in constructor

* Small changes to accommodate tests

* testing data

* Small fix to help existing installs

* Fixed tests

config/data-provider.php

@@ -30,6 +30,8 @@
         'incoming_security' => '',
         'incoming_username' => '',
         'incoming_password' => '',
+        'incoming_unread_only' => 'Unread',
+        'incoming_last_uid' => '0'
     ],
     'twilio' => [],
     'smssync' => [],

src/Ushahidi/DataSource/DataSourceManager.php

@@ -287,7 +287,8 @@ protected function createEmailSource(array $config)
         return new Email\Email(
             $config,
             app('mailer'),
-            app(MessageRepository::class)
+            app(MessageRepository::class),
+            $this->configRepo
         );
     }
 

src/Ushahidi/DataSource/Email/Email.php

@@ -17,6 +17,7 @@
 use Ushahidi\DataSource\Contracts\IncomingDataSource;
 use Ushahidi\DataSource\Concerns\MapsInboundFields;
 use Ushahidi\Contracts\Repository\Entity\MessageRepository;
+use Ushahidi\Contracts\Repository\Entity\ConfigRepository;
 use Ushahidi\DataSource\Contracts\MessageType;
 
 class Email extends OutgoingEmail implements IncomingDataSource
@@ -26,21 +27,25 @@ class Email extends OutgoingEmail implements IncomingDataSource
     protected $config;
     protected $mailer;
     protected $messageRepo;
+    protected $configRepo;
 
     /**
      * Constructor function for DataSource
      * @param array $config
      * @param Mailer|null $mailer
      * @param MessageRepository|null $messageRepo
+     * @param ConfigRepository|null $configRepo
      */
     public function __construct(
         array $config,
         Mailer $mailer = null,
-        MessageRepository $messageRepo = null
+        MessageRepository $messageRepo = null,
+        ConfigRepository $configRepo = null
     ) {
         $this->config = $config;
         $this->mailer = $mailer;
         $this->messageRepo = $messageRepo;
+        $this->configRepo = $configRepo;
     }
 
     public function getName()
@@ -104,6 +109,13 @@ public function getOptions()
                 'description' => '',
                 'placeholder' => 'Email account password',
                 'rules' => ['required']
+            ],
+            'incoming_all_unread' => [
+                'label' => 'Fetch Emails',
+                'input' => 'radio',
+                'description' => 'Fetch every email from the inbox, or only unread.',
+                'options' => ['All', 'Unread'],
+                'rules' => ['required']
             ]
         ];
     }
@@ -154,14 +166,17 @@ public function fetch($limit = false)
         $encryption = $this->config['incoming_security'] ?? '';
         $username = $this->config['incoming_username'] ?? '';
         $password = $this->config['incoming_password'] ?? '';
+        $unread_only = $this->config['incoming_all_unread'] ?? 'Unread';
+        $last_uid = $this->config['incoming_last_uid'] ?? '';
+        $new_last_uid = 0;
 
         // Encryption type
         $encryption = (strcasecmp($encryption, 'none') != 0) ? '/'.$encryption : '';
 
         // To connect to an SSL IMAP or POP3 server with a self-signed certificate,
         // add /novalidate-cert after the encryption protocol specification:
         $no_cert_validation = !empty($encryption) ? '/novalidate-cert' : '';
-        
+
         try {
             // Try to connect
             $inbox = '{'.$server.':'.$port.'/'.$type.$encryption.$no_cert_validation.'}INBOX';
@@ -182,7 +197,11 @@ public function fetch($limit = false)
 
             Log::info("Connected to $inbox", [$mailboxinfo]);
 
-            $last_uid = $this->messageRepo->getLastUID('email');
+            // Allow an existing installation to transition to config based without forcing the platform to download everything again.
+            if ($last_uid == '') {
+                $last_uid = $this->messageRepo->getLastUID('email');
+            }
+
             if ($last_uid > 0) {
                 $max_range = $last_uid + $limit;
                 $search_string = $last_uid ? $last_uid + 1 . ':' . $max_range : '1:' . $max_range;
@@ -208,6 +227,10 @@ public function fetch($limit = false)
                         break;
                     }
 
+                    if ($unread_only == 'Unread' and $email->seen == 1) {
+                        continue;
+                    }
+
                     $message = $html_message = "";
                     $structure = imap_fetchstructure($connection, $email->uid, FT_UID);
 
@@ -235,9 +258,16 @@ public function fetch($limit = false)
                         $message = imap_qprint($message);
                         $messages[] = $this->processIncoming($email, $message);
                     }
+                    if (isset($email->uid)) {
+                        $new_last_uid = isset($email->uid) ? $email->uid : null;
+                    }
                 }
             }
 
+            if ($new_last_uid && $new_last_uid != $last_uid) {
+                $this->updateLastUid($this->config, $new_last_uid);
+            }
+
             imap_errors();
             imap_alerts();
 
@@ -253,6 +283,16 @@ public function fetch($limit = false)
         return $messages;
     }
 
+    private function updateLastUid($config, $last_uid)
+    {
+        $providerConfig = $this->configRepo->get('data-provider');
+        $config['incoming_last_uid'] = $last_uid;
+        $providerConfigArray = $providerConfig->asArray();
+        $providerConfigArray[$this->getId()] = $config;
+        $providerConfig->setState($providerConfigArray);
+        $this->configRepo->update($providerConfig);
+    }
+
     /**
      * Process individual incoming email
      *

tests/Unit/DataSource/EmailDataSourceTest.php

@@ -16,6 +16,8 @@
 use phpmock\mockery\PHPMockery;
 use Ushahidi\Tests\TestCase;
 use Ushahidi\Contracts\Repository\Entity\MessageRepository;
+use Ushahidi\Contracts\Repository\Entity\ConfigRepository;
+use Ushahidi\Core\Entity\Config;
 use Ushahidi\DataSource\Email\Email;
 use Ushahidi\Multisite\Site;
 
@@ -51,10 +53,13 @@ public function testSend()
         $this->app->make('multisite')->setSite($site);
 
         $mockMailer = M::mock(\Illuminate\Contracts\Mail\Mailer::class);
+        $mockConfigRepo = M::mock(ConfigRepository::class);
 
         $email = new Email(
             [],
-            $mockMailer
+            $mockMailer,
+            null,
+            $mockConfigRepo
         );
 
         $mockMailer->shouldReceive('send')->once()->with(
@@ -92,6 +97,19 @@ public function testFetch()
 
         $mockMessageRepo->shouldReceive('getLastUID')->once()->andReturn(712);
 
+        $mockConfigRepo = M::mock(ConfigRepository::class);
+        $mockConfigRepo->shouldReceive('get')->once()->andReturn(new Config([
+                'incoming_type' => 'imap',
+                'incoming_server' => 'imap.somewhere.com',
+                'incoming_port' => '993',
+                'incoming_security' => 'ssl',
+                'incoming_username' => 'someuser',
+                'incoming_password' => 'mypassword',
+                'incoming_all_unread' => 'All',
+                'incoming_last_uid' => '712'
+        ]));
+        $mockConfigRepo->shouldReceive('update')->once()->andReturnUndefined();
+
         $email = new Email(
             [
                 'incoming_type' => 'imap',
@@ -100,9 +118,11 @@ public function testFetch()
                 'incoming_security' => 'ssl',
                 'incoming_username' => 'someuser',
                 'incoming_password' => 'mypassword',
+                'incoming_all_unread' => 'All',
             ],
             $mockMailer,
-            $mockMessageRepo
+            $mockMessageRepo,
+            $mockConfigRepo
         );
 
         $mockImapOpen = PHPMockery::mock("Ushahidi\DataSource\Email", 'imap_open');

tests/datasets/ushahidi/Base.yml

@@ -2198,7 +2198,7 @@ config:
   -
     group_name: "data-provider"
     config_key: email
-    config_value: '{"incoming_type":"","incoming_server":"","incoming_port":"","incoming_security":"","incoming_username":"","incoming_password":""}'
+    config_value: '{"incoming_type":"","incoming_server":"","incoming_port":"","incoming_security":"","incoming_username":"","incoming_password":"","incoming_all_unread":"","incoming_last_uid":""}'
   -
     group_name: "data-provider"
     config_key: frontlinesms