[Analytics] Notice/warning about old events left in the events queue …

src/module-elasticsuite-analytics/Controller/Adminhtml/Search/Usage.php

@@ -15,10 +15,13 @@
 
 use Magento\Backend\App\Action;
 use Magento\Backend\App\Action\Context;
+use Magento\Framework\Exception\LocalizedException;
 use Magento\Framework\View\Result\PageFactory;
 use Smile\ElasticsuiteCore\Api\Index\IndexOperationInterface;
+use Smile\ElasticsuiteAnalytics\Helper\Data as AnalyticsHelper;
 use Smile\ElasticsuiteAnalytics\Model\Report\Context as ReportContext;
 use Smile\ElasticsuiteTracker\Api\EventIndexInterface;
+use Smile\ElasticsuiteTracker\Api\EventQueueInterface;
 use Smile\ElasticsuiteTracker\Api\SessionIndexInterface;
 
 /**
@@ -35,6 +38,16 @@ class Usage extends Action
      */
     private $indexOperation;
 
+    /**
+     * @var EventQueueInterface
+     */
+    private $eventQueue;
+
+    /**
+     * @var AnalyticsHelper
+     */
+    private $config;
+
     /**
      * @var ReportContext
      */
@@ -50,17 +63,23 @@ class Usage extends Action
      *
      * @param Context                 $context           Context.
      * @param IndexOperationInterface $indexOperation    Index operation.
+     * @param EventQueueInterface     $eventQueue        Events queue.
+     * @param AnalyticsHelper         $config            Analytics config helper.
      * @param ReportContext           $reportContext     Report context.
      * @param PageFactory             $resultPageFactory Result page factory.
      */
     public function __construct(
         Context $context,
         IndexOperationInterface $indexOperation,
+        EventQueueInterface $eventQueue,
+        AnalyticsHelper $config,
         ReportContext $reportContext,
         PageFactory $resultPageFactory
     ) {
         parent::__construct($context);
         $this->indexOperation           = $indexOperation;
+        $this->eventQueue               = $eventQueue;
+        $this->config                   = $config;
         $this->reportContext            = $reportContext;
         $this->resultPageFactory        = $resultPageFactory;
     }
@@ -81,6 +100,8 @@ public function execute()
             }
         }
 
+        $this->addEventsWarningIfRequired();
+
         /** @var \Magento\Backend\Model\View\Result\Page $resultPage */
         $resultPage = $this->resultPageFactory->create();
         $resultPage->setActiveMenu('Smile_ElasticsuiteAnalytics::search_usage');
@@ -101,4 +122,30 @@ private function checkIndexPresence($indexIdentifier)
     {
         return $this->indexOperation->indexExists($indexIdentifier, $this->reportContext->getStoreId());
     }
+
+    /**
+     * Check if some events are waiting being indexed for too long and if so, display a notice message.
+     *
+     * @return void
+     */
+    private function addEventsWarningIfRequired()
+    {
+        $eventsWarningAfter = $this->config->getMaxHoursBeforeEventsWarning();
+        try {
+            $pendingEventsCount = $this->eventQueue->getPendingEventsCount($eventsWarningAfter);
+        } catch (LocalizedException $e) {
+            $pendingEventsCount = 0;
+        }
+        if ($pendingEventsCount > 0) {
+            $this->messageManager->addNoticeMessage(
+                __(
+                    'There are currently %1 events created more than %2 hours ago in the events queue table.'
+                    // phpcs:ignore Generic.Files.LineLength
+                    . ' If you think this screen is lacking some behavioral data, make sure the "elasticsuite_index_log_event" cronjob is running frequently enough.',
+                    $pendingEventsCount,
+                    $eventsWarningAfter
+                )
+            );
+        }
+    }
 }

src/module-elasticsuite-analytics/Helper/Data.php

@@ -28,6 +28,12 @@ class Data extends \Magento\Framework\App\Helper\AbstractHelper
      */
     const CONFIG_MAX_SEARCH_TERMS_XPATH = 'smile_elasticsuite_analytics/search_terms/max_size';
 
+    /**
+     * Maximum number of hours a valid pending event can stay in the events queue before a warning is displayed.
+     * @var string
+     */
+    const CONFIG_MAX_HOURS_BEFORE_EVENTS_WARNING = 'smile_elasticsuite_analytics/pending_events/warn_after_hours';
+
     /**
      * Returns the maximum number of search terms to display in reports
      *
@@ -37,4 +43,14 @@ public function getMaxSearchTerms()
     {
         return (int) $this->scopeConfig->getValue(self::CONFIG_MAX_SEARCH_TERMS_XPATH);
     }
+
+    /**
+     * Returns the maximum number of hours an event can stay in the events queue before a warning is displayed.
+     *
+     * @return int
+     */
+    public function getMaxHoursBeforeEventsWarning()
+    {
+        return (int) $this->scopeConfig->getValue(self::CONFIG_MAX_HOURS_BEFORE_EVENTS_WARNING);
+    }
 }

src/module-elasticsuite-analytics/etc/adminhtml/system.xml

@@ -29,6 +29,14 @@
                     <comment><![CDATA[Maximum number of search terms to display in the search usage report blocks.]]></comment>
                 </field>
             </group>
+
+            <group id="pending_events" translate="label" type="text" sortOrder="20" showInDefault="1" showInWebsite="0" showInStore="0">
+                <label>Pending events configuration</label>
+                <field id="warn_after_hours" translate="label comment" type="text" sortOrder="10" showInDefault="1" showInWebsite="0" showInStore="0" canRestore="1">
+                    <label>Hours before warning</label>
+                    <comment><![CDATA[A warning will be displayed on the search usage dashboard if there are pending events older than this number of hours still in the events queue table.]]></comment>
+                </field>
+            </group>
         </section>
     </system>
 </config>

src/module-elasticsuite-analytics/etc/config.xml

@@ -20,6 +20,9 @@
             <search_terms>
                 <max_size>100</max_size>
             </search_terms>
+            <pending_events>
+                <warn_after_hours>6</warn_after_hours>
+            </pending_events>
         </smile_elasticsuite_analytics>
     </default>
 </config>

src/module-elasticsuite-analytics/i18n/en_US.csv

@@ -34,3 +34,7 @@
 "The terms listed below are the most popular search terms, whether they lead to exact or close matches.<br />The link on a term leads to its edit screen where you can either <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ProductSorting#using-the-merchandiser"" target=""_blank"">merchandise</a> the search results page or configure a redirection - for instance to redirect the user to a dedicated category.<br />The ""conversion rate"" column value is computed as the ratio of the number of sessions where an order was placed and the term(s) searched for on the total number of sessions where the term(s) were searched for.","The terms listed below are the most popular search terms, whether they lead to exact or close matches.<br />The link on a term leads to its edit screen where you can either <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ProductSorting#using-the-merchandiser"" target=""_blank"">merchandise</a> the search results page or configure a redirection - for instance to redirect the user to a dedicated category.<br />The ""conversion rate"" column value is computed as the ratio of the number of sessions where an order was placed and the term(s) searched for on the total number of sessions where the term(s) were searched for."
 "The terms listed below were those for which they were no exact results and only close matches were displayed.<br />If a term is a ""popular mispelling"" with not many results while you know the correct search term would return more products, you might want to add a rule in the <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration"" target=""blank"">Thesaurus</a>.","The terms listed below were those for which they were no exact results and only close matches were displayed.<br />If a term is a ""popular mispelling"" with not many results while you know the correct search term would return more products, you might want to add a rule in the <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration"" target=""blank"">Thesaurus</a>."
 "The terms listed below were those for which they were no exact results and only close matches were displayed.<br />If a term is a ""popular mispelling"" with not many results while you know the correct search term would return more products, you might want to add a rule in the <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration"" target=""blank"">Thesaurus</a>.","The terms listed below were those for which they were no exact results and only close matches were displayed.<br />If a term is a ""popular mispelling"" with not many results while you know the correct search term would return more products, you might want to add a rule in the <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration"" target=""blank"">Thesaurus</a>."
+"There are currently %1 events created more than %2 hours ago in the events queue table. If you think this screen is lacking some behavioral data, make sure the ""elasticsuite_index_log_event"" cronjob is running frequently enough.","There are currently %1 events created more than %2 hours ago in the events queue table. If you think this screen is lacking some behavioral data, make sure the ""elasticsuite_index_log_event"" cronjob is running frequently enough."
+"Pending events configuration","Pending events configuration"
+"Hours before warning","Hours before warning"
+"A warning will be displayed on the search usage dashboard if there are pending events older than this number of hours still in the events queue table.","A warning will be displayed on the search usage dashboard if there are pending events older than this number of hours still in the events queue table."

src/module-elasticsuite-analytics/i18n/fr_FR.csv

@@ -34,3 +34,7 @@
 "The terms listed below are the most popular search terms, whether they lead to exact or close matches.<br />The link on a term leads to its edit screen where you can either <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ProductSorting#using-the-merchandiser"" target=""_blank"">merchandise</a> the search results page or configure a redirection - for instance to redirect the user to a dedicated category.<br />The ""conversion rate"" column value is computed as the ratio of the number of sessions where an order was placed and the term(s) searched for on the total number of sessions where the term(s) were searched for.","Les termes listés ci-dessous sont les termes les plus populaires, qu'ils amènent à des résultats exacts ou approchants.<br />Le lien sous un terme permet d'atteindre son écran d'édition où vous pouvez soit <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ProductSorting#using-the-merchandiser"" target=""_blank"">merchandiser</a> la page de résultats soit bien configurer une redirection - par exemple en redirigeant l'utilisateur vers une catégorie dédiée.<br />La valeur de la colonne ""taux de conversion"" est calculée comme le rapport du nombre de sessions avec un passage de commande et la recherche du/de ce(s) terme(s) sur le nombre total de sessions avec ce(s) terme(s) de recherche."
 "The terms listed below were those for which they were no exact results and only close matches were displayed.<br />If a term is a ""popular mispelling"" with not many results while you know the correct search term would return more products, you might want to add a rule in the <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration"" target=""blank"">Thesaurus</a>.","Les termes listés ci-dessous sont ceux pour lesquels seuls des résultats approchants ont été affichés.<br />Si un terme est une ""mauvaise orthographe populaire"" avec peu de résultats alors que vous savez que la recherche correcte est censée retourner plus de produits, vous pouvez souhaiter ajouter une règle dans le <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration"" target=""blank"">Thesaurus</a>."
 "The terms listed below were those for which they were neither exact results nor close matches.<br />If you have nonetheless some related products, you probably should add a rule in the <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration"" target=""blank"">Thesaurus</a>.<br />For instance, if you have a lot of ""running"" related products and visitors are searching repeatly for ""jogging"" without any results, you should definitely add a <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration#synonym"" target=""_blank"">synonym</a> or <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration#expansion"" target=""_blank"">expansion</a> rule in the Thesaurus to associate the terms ""running"" and ""jogging"".","Les termes listés ci-dessous sont ceux pour lesquels il n'y avait aucun résultat, exact ou approchant.<br />Si vous avez néanmoins des produits liés à la recherche effectuée, vous devriez probablement ajouter une règle dans le <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration"" target=""blank"">Thesaurus</a>.<br />Par exemple, si vous avez beaucoup de produits ""running"" et que les visiteurs recherchent fréquemment ""jogging"" sans succès, vous devriez très certainement ajouter une règle <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration#synonym"" target=""_blank"">de synonyme</a> ou <a href=""https://github.com/Smile-SA/elasticsuite/wiki/ThesaurusAdministration#expansion"" target=""_blank"">d'expansion</a> dans le Thesaurus pour associer les termes ""running"" and ""jogging""."
+"There are currently %1 events created more than %2 hours ago in the events queue table. If you think this screen is lacking some behavioral data, make sure the ""elasticsuite_index_log_event"" cronjob is running frequently enough.","Il y a actuellement %1 évènements créés il y a plus de %2 heures dans la table des évènements. S'il vous semble que des données comportementales sont absentes de cet écran, assurez-vous que la tâche automatisée ""elasticsuite_index_log_event"" tourne fréquemment."
+"Pending events configuration","Configuration des évènements en attente"
+"Hours before warning","Heures avant avertissement"
+"A warning will be displayed on the search usage dashboard if there are pending events older than this number of hours still in the events queue table.","Un avertissement sera affiché sur le tableau de bord d'utilisation de la recherche s'il y a des évènements plus vieux que ce nombre d'heures dans la table d'attente des évènements."

src/module-elasticsuite-tracker/Api/EventQueueInterface.php

@@ -78,4 +78,14 @@ public function purgeInvalidEvents($delay = 3, $limit = null);
      * @return array
      */
     public function getEvents($limit = null);
+
+    /**
+     * Get the number of supposedly valid events not yet indexed after the specified amount of hours.
+     *
+     * @param int $hours Only events whose creation date is older than this amount of hours will be counted.
+     *
+     * @return int
+     * @throws LocalizedException
+     */
+    public function getPendingEventsCount($hours = 24);
 }

src/module-elasticsuite-tracker/Model/EventQueue.php

@@ -100,4 +100,12 @@ public function purgeInvalidEvents($delay = 3, $limit = null)
     {
         $this->resourceModel->purgeInvalidEvents($delay, $limit);
     }
+
+    /**
+     * {@inheritDoc}
+     */
+    public function getPendingEventsCount($hours = 24)
+    {
+        return $this->resourceModel->getPendingEventsCount($hours);
+    }
 }

src/module-elasticsuite-tracker/Model/ResourceModel/EventQueue.php

@@ -193,6 +193,35 @@ public function purgeInvalidEvents($delay = 3, $limit = null)
         $connection->query($query);
     }
 
+    /**
+     * Get the number of supposedly valid events not yet indexed after the specified amount of hours.
+     *
+     * @param int $hours Only events whose creation date is older than this amount of hours will be counted.
+     *
+     * @return int
+     * @throws \Magento\Framework\Exception\LocalizedException
+     */
+    public function getPendingEventsCount($hours = 24)
+    {
+        $connection = $this->getConnection();
+        $select = $connection->select()
+            ->from($this->getMainTable(), ['count' => 'COUNT(*)'])
+            ->where('is_invalid = ?', 0);
+
+        if ($hours > 0) {
+            $select->where(
+                'created_at <= ?',
+                $connection->getDateSubSql(
+                    $connection->quote($this->dateTime->formatDate(true)),
+                    $hours,
+                    AdapterInterface::INTERVAL_HOUR
+                )
+            );
+        }
+
+        return (int) $this->getConnection()->fetchOne($select);
+    }
+
     /**
      * {@inheritDoc}
      *