!!![TASK] Introduce queue and queue item interfaces

As an initial steps to allow custom index queue types interfaces for queue and queue items are introduced. Additionally custom queue classes can be configured via: plugin.tx_solr.index.queue.<configurationName>.indexQueue This is a breaking change for all instances that implemented own queues and queue items! Resolves: #3763
TYPO3-Solr · Sep 15, 2023 · 6867b73 · 6867b73
1 parent 5b865ea
commit 6867b73
Show file tree

Hide file tree

Showing 20 changed files with 691 additions and 75 deletions.
diff --git a/Classes/Controller/Backend/Search/AbstractModuleController.php b/Classes/Controller/Backend/Search/AbstractModuleController.php
@@ -20,6 +20,7 @@
 use ApacheSolrForTypo3\Solr\Domain\Site\Site;
 use ApacheSolrForTypo3\Solr\Domain\Site\SiteRepository;
 use ApacheSolrForTypo3\Solr\IndexQueue\Queue;
+use ApacheSolrForTypo3\Solr\IndexQueue\QueueInterface;
 use ApacheSolrForTypo3\Solr\System\Mvc\Backend\Service\ModuleDataStorageService;
 use ApacheSolrForTypo3\Solr\System\Solr\SolrConnection as SolrCoreConnection;
 use Doctrine\DBAL\Exception as DBALException;
@@ -65,7 +66,7 @@ public function __construct(
         protected readonly SiteRepository $siteRepository,
         protected readonly SiteFinder $siteFinder,
         protected readonly ConnectionManager $solrConnectionManager,
-        protected Queue $indexQueue,
+        protected QueueInterface $indexQueue,
         protected ?int $selectedPageUID = null,
     ) {
         $this->selectedPageUID = $selectedPageUID ?? 0;

diff --git a/Classes/Controller/Backend/Search/IndexQueueModuleController.php b/Classes/Controller/Backend/Search/IndexQueueModuleController.php
@@ -17,8 +17,11 @@
 
 use ApacheSolrForTypo3\Solr\Backend\IndexingConfigurationSelectorField;
 use ApacheSolrForTypo3\Solr\Domain\Index\IndexService;
+use ApacheSolrForTypo3\Solr\Domain\Index\Queue\QueueInitializationService;
 use ApacheSolrForTypo3\Solr\Domain\Site\Exception\UnexpectedTYPO3SiteInitializationException;
 use ApacheSolrForTypo3\Solr\IndexQueue\Queue;
+use ApacheSolrForTypo3\Solr\IndexQueue\QueueInterface;
+use ApacheSolrForTypo3\Solr\IndexQueue\QueueInitializationServiceAwareInterface;
 use Doctrine\DBAL\ConnectionException;
 use Doctrine\DBAL\Exception as DBALException;
 use Psr\Http\Message\ResponseInterface;
@@ -36,7 +39,7 @@
  */
 class IndexQueueModuleController extends AbstractModuleController
 {
-    public function setIndexQueue(Queue $indexQueue): void
+    public function setIndexQueue(QueueInterface $indexQueue): void
     {
         $this->indexQueue = $indexQueue;
     }
@@ -107,7 +110,13 @@ public function initializeIndexQueueAction(): ResponseInterface
         if ((!empty($indexingConfigurationsToInitialize)) && (is_array($indexingConfigurationsToInitialize))) {
             // initialize selected indexing configuration
             try {
-                $initializedIndexingConfigurations = $this->indexQueue->getInitializationService()->initializeBySiteAndIndexConfigurations($this->selectedSite, $indexingConfigurationsToInitialize);
+                if ($this->indexQueue instanceof QueueInitializationServiceAwareInterface) {
+                    $initializationService = $this->indexQueue->getQueueInitializationService();
+                } else {
+                    $initializationService = GeneralUtility::makeInstance(QueueInitializationService::class);
+                }
+
+                $initializedIndexingConfigurations = $initializationService->initializeBySiteAndIndexConfigurations($this->selectedSite, $indexingConfigurationsToInitialize);
             } catch (Throwable $e) {
                 $this->addFlashMessage(
                     sprintf(

diff --git a/Classes/Domain/Index/IndexService.php b/Classes/Domain/Index/IndexService.php
@@ -24,6 +24,7 @@
 use ApacheSolrForTypo3\Solr\IndexQueue\Indexer;
 use ApacheSolrForTypo3\Solr\IndexQueue\Item;
 use ApacheSolrForTypo3\Solr\IndexQueue\Queue;
+use ApacheSolrForTypo3\Solr\IndexQueue\QueueInterface;
 use ApacheSolrForTypo3\Solr\System\Configuration\TypoScriptConfiguration;
 use ApacheSolrForTypo3\Solr\System\Logging\SolrLogManager;
 use ApacheSolrForTypo3\Solr\Task\IndexQueueWorkerTask;
@@ -45,15 +46,15 @@ class IndexService
 
     protected ?IndexQueueWorkerTask $contextTask = null;
 
-    protected Queue $indexQueue;
+    protected QueueInterface $indexQueue;
 
     protected EventDispatcherInterface $eventDispatcher;
 
     protected SolrLogManager $logger;
 
     public function __construct(
         Site $site,
-        Queue $queue = null,
+        QueueInterface $queue = null,
         EventDispatcherInterface $eventDispatcher = null,
         SolrLogManager $solrLogManager = null,
     ) {

diff --git a/Classes/Domain/Index/Queue/GarbageRemover/AbstractStrategy.php b/Classes/Domain/Index/Queue/GarbageRemover/AbstractStrategy.php
@@ -19,6 +19,7 @@
 use ApacheSolrForTypo3\Solr\Domain\Site\Exception\UnexpectedTYPO3SiteInitializationException;
 use ApacheSolrForTypo3\Solr\GarbageCollectorPostProcessor;
 use ApacheSolrForTypo3\Solr\IndexQueue\Queue;
+use ApacheSolrForTypo3\Solr\IndexQueue\QueueInterface;
 use ApacheSolrForTypo3\Solr\System\Logging\SolrLogManager;
 use Doctrine\DBAL\Exception as DBALException;
 use InvalidArgumentException;
@@ -31,12 +32,12 @@
  */
 abstract class AbstractStrategy
 {
-    protected Queue $queue;
+    protected QueueInterface $queue;
 
     protected ConnectionManager $connectionManager;
 
     public function __construct(
-        Queue $queue = null,
+        QueueInterface $queue = null,
         ConnectionManager $connectionManager = null,
     ) {
         $this->queue = $queue ?? GeneralUtility::makeInstance(Queue::class);

diff --git a/Classes/Domain/Index/Queue/QueueInitializationService.php b/Classes/Domain/Index/Queue/QueueInitializationService.php
@@ -20,7 +20,7 @@
 use ApacheSolrForTypo3\Solr\Domain\Site\Site;
 use ApacheSolrForTypo3\Solr\Event\IndexQueue\AfterIndexQueueHasBeenInitializedEvent;
 use ApacheSolrForTypo3\Solr\IndexQueue\Initializer\AbstractInitializer;
-use ApacheSolrForTypo3\Solr\IndexQueue\Queue;
+use ApacheSolrForTypo3\Solr\IndexQueue\QueueInterface;
 use Doctrine\DBAL\ConnectionException;
 use Doctrine\DBAL\Exception as DBALException;
 use Psr\EventDispatcher\EventDispatcherInterface;
@@ -35,14 +35,18 @@
  */
 class QueueInitializationService
 {
-    protected Queue $queue;
+    protected bool $clearQueueOnInitialization = true;
     protected EventDispatcherInterface $eventDispatcher;
 
-    public function __construct(Queue $queue, EventDispatcherInterface $eventDispatcher = null)
+    public function __construct(EventDispatcherInterface $eventDispatcher = null)
     {
-        $this->queue = $queue;
         $this->eventDispatcher = $eventDispatcher ?? GeneralUtility::makeInstance(EventDispatcherInterface::class);
     }
+
+    public function setClearQueueOnInitialization(bool $clearQueueOnInitialization): void
+    {
+        $this->clearQueueOnInitialization = $clearQueueOnInitialization;
+    }
 
     /**
      * Truncate and rebuild the tx_solr_indexqueue_item table. This is the most
@@ -113,10 +117,21 @@ public function initializeBySiteAndIndexConfigurations(Site $site, array $indexi
      */
     protected function applyInitialization(Site $site, string $indexingConfigurationName): bool
     {
+        $solrConfiguration = $site->getSolrConfiguration();
+
+        /** @var QueueInterface $queue */
+        $queue = GeneralUtility::makeInstance(
+            $solrConfiguration->getIndexQueueClassByConfigurationName($indexingConfigurationName)
+        );
+        if ($queue instanceof QueueInitializationServiceAwareInterface) {
+            $queue->setQueueInitializationService($this);
+        }
+
         // clear queue
-        $this->queue->deleteItemsBySite($site, $indexingConfigurationName);
+        if ($this->clearQueueOnInitialization) {
+            $queue->deleteItemsBySite($site, $indexingConfigurationName);
+        }
 
-        $solrConfiguration = $site->getSolrConfiguration();
         $type = $solrConfiguration->getIndexQueueTypeOrFallbackToConfigurationName($indexingConfigurationName);
         $initializerClass = $solrConfiguration->getIndexQueueInitializerClassByConfigurationName($indexingConfigurationName);
         $indexConfiguration = $solrConfiguration->getIndexQueueConfigurationByName($indexingConfigurationName);

diff --git a/Classes/Domain/Index/Queue/QueueItemRepository.php b/Classes/Domain/Index/Queue/QueueItemRepository.php
@@ -20,6 +20,7 @@
 use ApacheSolrForTypo3\Solr\Domain\Site\Site;
 use ApacheSolrForTypo3\Solr\Event\IndexQueue\AfterRecordsForIndexQueueItemsHaveBeenRetrievedEvent;
 use ApacheSolrForTypo3\Solr\IndexQueue\Item;
+use ApacheSolrForTypo3\Solr\IndexQueue\ItemInterface;
 use ApacheSolrForTypo3\Solr\System\Logging\SolrLogManager;
 use ApacheSolrForTypo3\Solr\System\Records\AbstractRepository;
 use ApacheSolrForTypo3\Solr\System\Util\SiteUtility;
@@ -118,7 +119,7 @@ public function flushErrorsBySite(Site $site): int
     /**
      * Flushes the error for a single item.
      */
-    public function flushErrorByItem(Item $item): int
+    public function flushErrorByItem(ItemInterface $item): int
     {
         $queryBuilder = $this->getQueryBuilder();
         return $this->getPreparedFlushErrorQuery($queryBuilder)
@@ -751,7 +752,7 @@ protected function getQueueItemObjectsByRecords(array $indexQueueItemRecords, ar
      * Marks an item as failed and causes the indexer to skip the item in the
      * next run.
      */
-    public function markItemAsFailed(Item|int|null $item, string $errorMessage = ''): int
+    public function markItemAsFailed(ItemInterface|int|null $item, string $errorMessage = ''): int
     {
         $itemUid = ($item instanceof Item) ? $item->getIndexQueueUid() : (int)$item;
         $errorMessage = empty($errorMessage) ? '1' : $errorMessage;
@@ -767,7 +768,7 @@ public function markItemAsFailed(Item|int|null $item, string $errorMessage = '')
     /**
      * Sets the timestamp of when an item last has been indexed.
      */
-    public function updateIndexTimeByItem(Item $item): int
+    public function updateIndexTimeByItem(ItemInterface $item): int
     {
         $queryBuilder = $this->getQueryBuilder();
         return $queryBuilder
@@ -780,7 +781,7 @@ public function updateIndexTimeByItem(Item $item): int
     /**
      * Sets the change timestamp of an item.
      */
-    public function updateChangedTimeByItem(Item $item, int $changedTime = 0): int
+    public function updateChangedTimeByItem(ItemInterface $item, int $changedTime = 0): int
     {
         $queryBuilder = $this->getQueryBuilder();
         return $queryBuilder

diff --git a/Classes/Event/IndexQueue/AfterIndexQueueItemHasBeenMarkedForReindexingEvent.php b/Classes/Event/IndexQueue/AfterIndexQueueItemHasBeenMarkedForReindexingEvent.php
@@ -27,9 +27,10 @@ final class AfterIndexQueueItemHasBeenMarkedForReindexingEvent
 {
     public function __construct(
         private readonly string $itemType,
-        private readonly int $itemUid,
+        private readonly int|string $itemUid,
         private readonly int $forcedChangeTime,
-        private int $updateCount
+        private int $updateCount,
+        private ?array $validLanguageUids
     ) {
     }
 
@@ -38,7 +39,7 @@ public function getItemType(): string
         return $this->itemType;
     }
 
-    public function getItemUid(): int
+    public function getItemUid(): int|string
     {
         return $this->itemUid;
     }
@@ -57,4 +58,9 @@ public function setUpdateCount(int $updateCount): void
     {
         $this->updateCount = $updateCount;
     }
+
+    public function getValidLanguageUids(): ?array
+    {
+        return $this->validLanguageUids;
+    }
 }
diff --git a/Classes/IndexQueue/Item.php b/Classes/IndexQueue/Item.php
@@ -34,14 +34,8 @@
  *
  * @author Ingo Renner <ingo@typo3.org>
  */
-class Item
+class Item implements ItemInterface, MountPointAwareItemInterface
 {
-    public const STATE_BLOCKED = -1;
-
-    public const STATE_PENDING = 0;
-
-    public const STATE_INDEXED = 1;
-
     /**
      * The item's uid in the index queue (tx_solr_indexqueue_item.uid)
      */
@@ -94,6 +88,13 @@ class Item
      */
     protected ?int $recordUid = null;
 
+    /**
+     * The indexing priority
+     *
+     * @var int
+     */
+    protected int $indexingPriority = 0;
+
     /**
      * The record itself
      */
@@ -136,6 +137,7 @@ public function __construct(
 
         $this->indexingConfigurationName = $itemMetaData['indexing_configuration'] ?? '';
         $this->hasIndexingProperties = (bool)($itemMetaData['has_indexing_properties'] ?? false);
+        $this->indexingPriority = (int)($itemMetaData['indexing_priority'] ?? 0);
 
         if (!empty($fullRecord)) {
             $this->record = $fullRecord;
@@ -217,7 +219,7 @@ public function getType(): ?string
         return $this->type;
     }
 
-    public function setType($type): void
+    public function setType(string $type): void
     {
         $this->type = $type;
     }
@@ -266,7 +268,7 @@ public function getRecordUid(): int
     {
         $this->getRecord();
 
-        return $this->record['uid'];
+        return (int)$this->record['uid'];
     }
 
     /**
@@ -439,4 +441,12 @@ public function getIndexingPropertyNames(): array
         $this->loadIndexingProperties();
         return array_keys($this->indexingProperties);
     }
+
+    /**
+     * Returns the index priority.
+     */
+    public function getIndexPriority(): int
+    {
+        return $this->indexingPriority;
+    }
 }