diff --git a/app/code/Magento/Eav/Model/Mview/ChangeLogBatchWalker.php b/app/code/Magento/Eav/Model/Mview/ChangeLogBatchWalker.php new file mode 100644 index 0000000000000..464d145c43024 --- /dev/null +++ b/app/code/Magento/Eav/Model/Mview/ChangeLogBatchWalker.php @@ -0,0 +1,167 @@ +resourceConnection = $resourceConnection; + $this->entityTypeCodes = $entityTypeCodes; + $this->changelog = $changelog; + $this->fromVersionId = $fromVersionId; + $this->toVersionId = $toVersionId; + $this->batchSize = $batchSize; + } + + /** + * @return \Generator|\Traversable + * @throws \Exception + */ + public function getIterator(): \Generator + { + while ($this->fromVersionId < $this->toVersionId) { + $ids = $this->walk(); + + if (empty($ids)) { + break; + } + $this->fromVersionId += $this->batchSize; + yield $ids; + } + } + + /** + * Calculate EAV attributes size + * + * @param ChangelogInterface $changelog + * @return int + * @throws LocalizedException + */ + private function calculateEavAttributeSize(): int + { + $connection = $this->resourceConnection->getConnection(); + + if (!isset($this->entityTypeCodes[$this->changelog->getViewId()])) { + throw new LocalizedException(__('Entity type for view was not defined')); + } + + $select = $connection->select(); + $select->from( + $this->resourceConnection->getTableName('eav_attribute'), + new Expression('COUNT(*)') + )->joinInner( + ['type' => $connection->getTableName('eav_entity_type')], + 'type.entity_type_id=eav_attribute.entity_type_id' + )->where('type.entity_type_code = ?', $this->entityTypeCodes[$this->changelog->getViewId()]); + + return (int) $connection->fetchOne($select); + } + + /** + * Prepare group max concat + * + * @param int $numberOfAttributes + * @return void + * @throws \Exception + */ + private function setGroupConcatMax(int $numberOfAttributes): void + { + $connection = $this->resourceConnection->getConnection(); + $connection->query(sprintf( + 'SET SESSION %s=%s', + self::GROUP_CONCAT_MAX_VARIABLE, + $numberOfAttributes * (self::DEFAULT_ID_SIZE + 1) + )); + } + + /** + * @inheritdoc + * @throws \Exception + */ + private function walk() + { + $connection = $this->resourceConnection->getConnection(); + $numberOfAttributes = $this->calculateEavAttributeSize(); + $this->setGroupConcatMax($numberOfAttributes); + $select = $connection->select()->distinct(true) + ->where( + 'version_id > ?', + (int) $this->fromVersionId + ) + ->where( + 'version_id <= ?', + $this->toVersionId + ) + ->group([$this->changelog->getColumnName(), 'store_id']) + ->limit($this->batchSize); + + $columns = [ + $this->changelog->getColumnName(), + 'attribute_ids' => new Expression('GROUP_CONCAT(attribute_id)'), + 'store_id' + ]; + $select->from($this->changelog->getName(), $columns); + return $connection->fetchAll($select); + } +} diff --git a/dev/tests/integration/_files/Magento/TestModuleMview/etc/module.xml b/dev/tests/integration/_files/Magento/TestModuleMview/etc/module.xml new file mode 100644 index 0000000000000..64d857a0132a0 --- /dev/null +++ b/dev/tests/integration/_files/Magento/TestModuleMview/etc/module.xml @@ -0,0 +1,11 @@ + + + + + + diff --git a/dev/tests/integration/_files/Magento/TestModuleMview/etc/mview.xml b/dev/tests/integration/_files/Magento/TestModuleMview/etc/mview.xml new file mode 100644 index 0000000000000..c7ccd8512fbf6 --- /dev/null +++ b/dev/tests/integration/_files/Magento/TestModuleMview/etc/mview.xml @@ -0,0 +1,24 @@ + + + + + + +
+ + + + +
+ + + diff --git a/dev/tests/integration/_files/Magento/TestModuleMview/registration.php b/dev/tests/integration/_files/Magento/TestModuleMview/registration.php new file mode 100644 index 0000000000000..5c5453c1bd413 --- /dev/null +++ b/dev/tests/integration/_files/Magento/TestModuleMview/registration.php @@ -0,0 +1,12 @@ +getPath(ComponentRegistrar::MODULE, 'Magento_TestModuleMview') === null) { + ComponentRegistrar::register(ComponentRegistrar::MODULE, 'Magento_TestModuleMview', __DIR__); +} diff --git a/dev/tests/integration/testsuite/Magento/Eav/Model/EnhancedMviewTest.php b/dev/tests/integration/testsuite/Magento/Eav/Model/EnhancedMviewTest.php new file mode 100644 index 0000000000000..6dfc2ba123464 --- /dev/null +++ b/dev/tests/integration/testsuite/Magento/Eav/Model/EnhancedMviewTest.php @@ -0,0 +1,102 @@ +objectManager = \Magento\TestFramework\Helper\Bootstrap::getObjectManager(); + $this->resourceConnection = $this->objectManager->get(ResourceConnection::class); + parent::setUp(); + } + + /** + * @dataProvider attributesDataProvider + * @magentoDataFixture Magento/Eav/_files/enable_mview_for_test_view.php + * @magentoDataFixture Magento/Catalog/_files/product_simple.php + * @param array $expectedAttributes + * @throws \Magento\Framework\Exception\NoSuchEntityException + */ + public function testCreateProduct(array $expectedAttributes) + { + $changelogData = $this->getChangelogData(); + $attributesMapping = $this->getAttributeCodes(); + $attributes = []; + foreach ($changelogData as $row) { + $this->assertArrayHasKey('store_id', $row); + $this->assertArrayHasKey('attribute_id', $row); + + if ($row['store_id'] == 0 && $row['attribute_id'] !== null) { + $attributes[$attributesMapping[$row['attribute_id']]] = $attributesMapping[$row['attribute_id']]; + } + } + sort($expectedAttributes); + sort($attributes); + $this->assertEquals($attributes, $expectedAttributes); + + $this->assertTrue(true); + } + + /** + * @return array + */ + private function getAttributeCodes(): array + { + $connection = $this->resourceConnection->getConnection(); + $select = $connection->select() + ->from($connection->getTableName('eav_attribute'), ['attribute_id', 'attribute_code']); + return $connection->fetchPairs($select); + } + + /** + * @return array|\string[][] + */ + public function attributesDataProvider(): array + { + return [ + [ + 'default' => [ + 'name', + 'meta_title', + 'meta_description', + 'is_returnable', + 'options_container' + ] + ] + ]; + } + + /** + * @return array + */ + private function getChangelogData() + { + $connection = $this->resourceConnection->getConnection(); + $select = $connection->select() + ->from($connection->getTableName('test_view_cl')); + return $connection->fetchAll($select); + } +} diff --git a/dev/tests/integration/testsuite/Magento/Eav/_files/enable_mview_for_test_view.php b/dev/tests/integration/testsuite/Magento/Eav/_files/enable_mview_for_test_view.php new file mode 100644 index 0000000000000..bb22de22d0b54 --- /dev/null +++ b/dev/tests/integration/testsuite/Magento/Eav/_files/enable_mview_for_test_view.php @@ -0,0 +1,14 @@ +create(\Magento\Framework\Mview\View::class); +$view->load('test_view'); +$view->subscribe(); diff --git a/lib/internal/Magento/Framework/Mview/Config/Converter.php b/lib/internal/Magento/Framework/Mview/Config/Converter.php index 5c33ac150d00a..988cffa8b7ce2 100644 --- a/lib/internal/Magento/Framework/Mview/Config/Converter.php +++ b/lib/internal/Magento/Framework/Mview/Config/Converter.php @@ -5,10 +5,34 @@ */ namespace Magento\Framework\Mview\Config; +use Magento\Framework\Mview\View\AdditionalColumnsProcessor\DefaultProcessor; +use Magento\Framework\Mview\View\ChangeLogBatchWalker; use Magento\Framework\Mview\View\SubscriptionInterface; class Converter implements \Magento\Framework\Config\ConverterInterface { + /** + * @var string + */ + private $defaultProcessor; + + /** + * @var string + */ + private $defaultIterator; + + /** + * @param string $defaultProcessor + * @param string $defaultIterator + */ + public function __construct( + string $defaultProcessor = DefaultProcessor::class, + string $defaultIterator = ChangeLogBatchWalker::class + ) { + $this->defaultProcessor = $defaultProcessor; + $this->defaultIterator = $defaultIterator; + } + /** * Convert dom node tree to array * @@ -28,6 +52,7 @@ public function convert($source) $data['view_id'] = $viewId; $data['action_class'] = $this->getAttributeValue($viewNode, 'class'); $data['group'] = $this->getAttributeValue($viewNode, 'group'); + $data['walker'] = $this->getAttributeValue($viewNode, 'walker') ?: $this->defaultIterator; $data['subscriptions'] = []; /** @var $childNode \DOMNode */ @@ -76,6 +101,7 @@ protected function convertChild(\DOMNode $childNode, $data) $name = $this->getAttributeValue($subscription, 'name'); $column = $this->getAttributeValue($subscription, 'entity_column'); $subscriptionModel = $this->getAttributeValue($subscription, 'subscription_model'); + if (!empty($subscriptionModel) && !in_array( SubscriptionInterface::class, @@ -89,11 +115,44 @@ class_implements(ltrim($subscriptionModel, '\\')) $data['subscriptions'][$name] = [ 'name' => $name, 'column' => $column, - 'subscription_model' => $subscriptionModel + 'subscription_model' => $subscriptionModel, + 'additional_columns' => $this->getAdditionalColumns($subscription), + 'processor' => $this->getAttributeValue($subscription, 'processor') + ?: $this->defaultProcessor ]; } break; } return $data; } + + /** + * Retrieve additional columns of subscription table + * + * @param \DOMNode $subscription + * @return array + */ + private function getAdditionalColumns(\DOMNode $subscription): array + { + $additionalColumns = []; + foreach ($subscription->childNodes as $childNode) { + if ($childNode->nodeType != XML_ELEMENT_NODE || $childNode->nodeName != 'additionalColumns') { + continue; + } + + foreach ($childNode->childNodes as $columnNode) { + if ($columnNode->nodeName !== 'column') { + continue; + } + + $additionalColumns[$this->getAttributeValue($columnNode, 'name')] = [ + 'name' => $this->getAttributeValue($columnNode, 'name'), + 'cl_name' => $this->getAttributeValue($columnNode, 'cl_name'), + 'constant' => $this->getAttributeValue($columnNode, 'constant'), + ]; + } + } + + return $additionalColumns; + } } diff --git a/lib/internal/Magento/Framework/Mview/Test/Unit/View/ChangelogTest.php b/lib/internal/Magento/Framework/Mview/Test/Unit/View/ChangelogTest.php index 0a4e8a3840749..4af3f65be6883 100644 --- a/lib/internal/Magento/Framework/Mview/Test/Unit/View/ChangelogTest.php +++ b/lib/internal/Magento/Framework/Mview/Test/Unit/View/ChangelogTest.php @@ -11,6 +11,7 @@ use Magento\Framework\DB\Adapter\Pdo\Mysql; use Magento\Framework\DB\Ddl\Table; use Magento\Framework\DB\Select; +use Magento\Framework\Mview\Config; use Magento\Framework\Mview\View\Changelog; use Magento\Framework\Mview\View\ChangelogInterface; use PHPUnit\Framework\MockObject\MockObject; @@ -46,7 +47,21 @@ protected function setUp(): void $this->resourceMock = $this->createMock(ResourceConnection::class); $this->mockGetConnection($this->connectionMock); - $this->model = new Changelog($this->resourceMock); + $this->model = new Changelog($this->resourceMock, $this->getMviewConfigMock()); + } + + /** + * @return Config|MockObject + */ + private function getMviewConfigMock() + { + $mviewConfigMock = $this->createMock(Config::class); + $mviewConfigMock->expects($this->any()) + ->method('getView') + ->willReturn([ + 'subscriptions' => [] + ]); + return $mviewConfigMock; } public function testInstanceOf() @@ -54,7 +69,7 @@ public function testInstanceOf() $resourceMock = $this->createMock(ResourceConnection::class); $resourceMock->expects($this->once())->method('getConnection')->willReturn(true); - $model = new Changelog($resourceMock); + $model = new Changelog($resourceMock, $this->getMviewConfigMock()); $this->assertInstanceOf(ChangelogInterface::class, $model); } @@ -65,7 +80,7 @@ public function testCheckConnectionException() $resourceMock = $this->createMock(ResourceConnection::class); $resourceMock->expects($this->once())->method('getConnection')->willReturn(null); - $model = new Changelog($resourceMock); + $model = new Changelog($resourceMock, $this->getMviewConfigMock()); $model->setViewId('ViewIdTest'); $this->assertNull($model); } diff --git a/lib/internal/Magento/Framework/Mview/Test/Unit/View/SubscriptionTest.php b/lib/internal/Magento/Framework/Mview/Test/Unit/View/SubscriptionTest.php index b91c0b525390f..2178009eda436 100644 --- a/lib/internal/Magento/Framework/Mview/Test/Unit/View/SubscriptionTest.php +++ b/lib/internal/Magento/Framework/Mview/Test/Unit/View/SubscriptionTest.php @@ -7,10 +7,13 @@ namespace Magento\Framework\Mview\Test\Unit\View; +use Magento\Framework\App\ObjectManager; use Magento\Framework\App\ResourceConnection; use Magento\Framework\DB\Adapter\Pdo\Mysql; use Magento\Framework\DB\Ddl\Trigger; use Magento\Framework\DB\Ddl\TriggerFactory; +use Magento\Framework\Mview\Config; +use Magento\Framework\Mview\View\AdditionalColumnsProcessor\DefaultProcessor; use Magento\Framework\Mview\View\ChangelogInterface; use Magento\Framework\Mview\View\CollectionInterface; use Magento\Framework\Mview\View\StateInterface; @@ -48,6 +51,7 @@ class SubscriptionTest extends TestCase protected function setUp(): void { + $this->tableName = 'test_table'; $this->connectionMock = $this->createMock(Mysql::class); $this->resourceMock = $this->createMock(ResourceConnection::class); @@ -55,10 +59,17 @@ protected function setUp(): void ->method('quoteIdentifier') ->willReturnArgument(0); + $this->connectionMock->expects($this->any()) + ->method('describeTable') + ->willReturn([]); + $this->resourceMock->expects($this->atLeastOnce()) ->method('getConnection') ->willReturn($this->connectionMock); - + ObjectManager::getInstance()->expects($this->any()) + ->method('get') + ->with(DefaultProcessor::class) + ->willReturn(2); $this->triggerFactoryMock = $this->createMock(TriggerFactory::class); $this->viewCollectionMock = $this->getMockForAbstractClass( CollectionInterface::class, @@ -78,18 +89,32 @@ protected function setUp(): void true, [] ); - + $this->viewMock->expects($this->any()) + ->method('getId') + ->willReturn(1); $this->resourceMock->expects($this->any()) ->method('getTableName') ->willReturnArgument(0); - + $mviewConfigMock = $this->createMock(Config::class); + $mviewConfigMock->expects($this->any()) + ->method('getView') + ->willReturn([ + 'subscriptions' => [ + $this->tableName => [ + 'processor' => DefaultProcessor::class + ] + ] + ]); + $this->mviewConfig = $mviewConfigMock; $this->model = new Subscription( $this->resourceMock, $this->triggerFactoryMock, $this->viewCollectionMock, $this->viewMock, $this->tableName, - 'columnName' + 'columnName', + [], + $mviewConfigMock ); } @@ -122,7 +147,7 @@ public function testCreate() $triggerMock->expects($this->exactly(3)) ->method('setName') ->with($triggerName)->willReturnSelf(); - $triggerMock->expects($this->exactly(3)) + $triggerMock->expects($this->any()) ->method('getName') ->willReturn('triggerName'); $triggerMock->expects($this->exactly(3)) @@ -167,7 +192,7 @@ public function testCreate() true, [] ); - $changelogMock->expects($this->exactly(3)) + $changelogMock->expects($this->any()) ->method('getName') ->willReturn('test_view_cl'); $changelogMock->expects($this->exactly(3)) @@ -191,7 +216,7 @@ public function testCreate() true, [] ); - $otherChangelogMock->expects($this->exactly(3)) + $otherChangelogMock->expects($this->any()) ->method('getName') ->willReturn('other_test_view_cl'); $otherChangelogMock->expects($this->exactly(3)) @@ -217,7 +242,7 @@ public function testCreate() ->method('getChangelog') ->willReturn($otherChangelogMock); - $this->viewMock->expects($this->exactly(3)) + $this->viewMock->expects($this->any()) ->method('getId') ->willReturn('this_id'); $this->viewMock->expects($this->never()) @@ -235,7 +260,6 @@ public function testCreate() $this->connectionMock->expects($this->exactly(3)) ->method('createTrigger') ->with($triggerMock); - $this->model->create(); } @@ -244,7 +268,7 @@ public function testRemove() $triggerMock = $this->createMock(Trigger::class); $triggerMock->expects($this->exactly(3)) ->method('setName')->willReturnSelf(); - $triggerMock->expects($this->exactly(3)) + $triggerMock->expects($this->any()) ->method('getName') ->willReturn('triggerName'); $triggerMock->expects($this->exactly(3)) @@ -271,7 +295,7 @@ public function testRemove() true, [] ); - $otherChangelogMock->expects($this->exactly(3)) + $otherChangelogMock->expects($this->any()) ->method('getName') ->willReturn('other_test_view_cl'); $otherChangelogMock->expects($this->exactly(3)) @@ -297,7 +321,7 @@ public function testRemove() ->method('getChangelog') ->willReturn($otherChangelogMock); - $this->viewMock->expects($this->exactly(3)) + $this->viewMock->expects($this->any()) ->method('getId') ->willReturn('this_id'); $this->viewMock->expects($this->never()) diff --git a/lib/internal/Magento/Framework/Mview/Test/Unit/ViewTest.php b/lib/internal/Magento/Framework/Mview/Test/Unit/ViewTest.php index 7d69ff43f158b..6fff95a0cb089 100644 --- a/lib/internal/Magento/Framework/Mview/Test/Unit/ViewTest.php +++ b/lib/internal/Magento/Framework/Mview/Test/Unit/ViewTest.php @@ -7,6 +7,7 @@ namespace Magento\Framework\Mview\Test\Unit; +use Laminas\Log\Filter\Mock; use Magento\Framework\Mview\ActionFactory; use Magento\Framework\Mview\ActionInterface; use Magento\Framework\Mview\ConfigInterface; @@ -53,6 +54,11 @@ class ViewTest extends TestCase */ protected $subscriptionFactoryMock; + /** + * @var MockObject|View\ChangeLogBatchWalkerInterface + */ + private $iteratorMock; + /** * @inheritdoc */ @@ -67,6 +73,7 @@ protected function setUp(): void true, ['getView'] ); + $this->iteratorMock = $this->createMock(View\ChangeLogBatchWalker::class); $this->actionFactoryMock = $this->createPartialMock(ActionFactory::class, ['get']); $this->stateMock = $this->createPartialMock( State::class, @@ -97,7 +104,10 @@ protected function setUp(): void $this->actionFactoryMock, $this->stateMock, $this->changelogMock, - $this->subscriptionFactoryMock + $this->subscriptionFactoryMock, + [], + [], + $this->iteratorMock ); } @@ -334,7 +344,7 @@ public function testUpdate() $currentVersionId ); $this->changelogMock->expects( - $this->once() + $this->any() )->method( 'getList' )->with( @@ -345,6 +355,7 @@ public function testUpdate() ); $actionMock = $this->getMockForAbstractClass(ActionInterface::class); + $this->iteratorMock->expects($this->once())->method('walk')->willReturn($listId); $actionMock->expects($this->once())->method('execute')->with($listId)->willReturnSelf(); $this->actionFactoryMock->expects( $this->once() @@ -390,7 +401,7 @@ public function testUpdateEx(): void ->expects($this->once()) ->method('getVersion') ->willReturn($currentVersionId); - + $this->iteratorMock->expects($this->any())->method('walk')->willReturn($this->generateChangeLog(150, 1, 150)); $this->changelogMock->method('getList') ->willReturnMap( [ @@ -401,7 +412,7 @@ public function testUpdateEx(): void ); $actionMock = $this->getMockForAbstractClass(ActionInterface::class); - $actionMock->expects($this->once()) + $actionMock->expects($this->any()) ->method('execute') ->with($this->generateChangeLog(150, 1, 150)) ->willReturnSelf(); @@ -457,7 +468,7 @@ public function testUpdateWithException() $this->stateMock->expects($this->atLeastOnce()) ->method('getMode') ->willReturn(StateInterface::MODE_ENABLED); - $this->stateMock->expects($this->exactly(2)) + $this->stateMock->expects($this->any()) ->method('getStatus') ->willReturn(StateInterface::STATUS_IDLE); $this->stateMock->expects($this->exactly(2)) @@ -472,16 +483,9 @@ public function testUpdateWithException() )->willReturn( $currentVersionId ); - $this->changelogMock->expects( - $this->once() - )->method( - 'getList' - )->with( - $lastVersionId, - $currentVersionId - )->willReturn( - $listId - ); + $this->iteratorMock->expects($this->any()) + ->method('walk') + ->willReturn([2,3]); $actionMock = $this->createPartialMock(ActionInterface::class, ['execute']); $actionMock->expects($this->once())->method('execute')->with($listId)->willReturnCallback( @@ -767,8 +771,11 @@ public function testGetUpdated() protected function loadView() { $viewId = 'view_test'; + $this->changelogMock->expects($this->any()) + ->method('getViewId') + ->willReturn($viewId); $this->configMock->expects( - $this->once() + $this->any() )->method( 'getView' )->with( @@ -788,7 +795,7 @@ protected function getViewData() 'view_id' => 'view_test', 'action_class' => 'Some\Class\Name', 'group' => 'some_group', - 'subscriptions' => ['some_entity' => ['name' => 'some_entity', 'column' => 'entity_id']] + 'subscriptions' => ['some_entity' => ['name' => 'some_entity', 'column' => 'entity_id']], ]; } } diff --git a/lib/internal/Magento/Framework/Mview/Test/Unit/_files/mview_config.php b/lib/internal/Magento/Framework/Mview/Test/Unit/_files/mview_config.php index a19f90546bac3..af488ecf5589e 100644 --- a/lib/internal/Magento/Framework/Mview/Test/Unit/_files/mview_config.php +++ b/lib/internal/Magento/Framework/Mview/Test/Unit/_files/mview_config.php @@ -20,14 +20,19 @@ 'some_entity' => [ 'name' => 'some_entity', 'column' => 'entity_id', - 'subscription_model' => null + 'subscription_model' => null, + 'additional_columns' => [], + 'processor' => \Magento\Framework\Mview\View\AdditionalColumnsProcessor\DefaultProcessor::class ], 'some_product_relation' => [ 'name' => 'some_product_relation', 'column' => 'product_id', - 'subscription_model' => null + 'subscription_model' => null, + 'additional_columns' => [], + 'processor' => \Magento\Framework\Mview\View\AdditionalColumnsProcessor\DefaultProcessor::class ], ], + 'walker' => \Magento\Framework\Mview\View\ChangeLogBatchWalker::class ], ] ]; diff --git a/lib/internal/Magento/Framework/Mview/View.php b/lib/internal/Magento/Framework/Mview/View.php index dade475a20482..e3e8726332783 100644 --- a/lib/internal/Magento/Framework/Mview/View.php +++ b/lib/internal/Magento/Framework/Mview/View.php @@ -9,7 +9,11 @@ namespace Magento\Framework\Mview; use InvalidArgumentException; +use Magento\Framework\App\ObjectManager; use Magento\Framework\DataObject; +use Magento\Framework\Mview\View\ChangeLogBatchWalkerFactory; +use Magento\Framework\Mview\View\ChangeLogBatchWalkerInterface; +use Magento\Framework\Mview\View\ChangelogInterface; use Magento\Framework\Mview\View\ChangelogTableNotExistsException; use Magento\Framework\Mview\View\SubscriptionFactory; use Exception; @@ -27,11 +31,6 @@ class View extends DataObject implements ViewInterface */ const DEFAULT_BATCH_SIZE = 1000; - /** - * Max versions to load from database at a time - */ - private static $maxVersionQueryBatch = 100000; - /** * @var string */ @@ -67,6 +66,11 @@ class View extends DataObject implements ViewInterface */ private $changelogBatchSize; + /** + * @var ChangeLogBatchWalkerFactory + */ + private $changeLogBatchWalkerFactory; + /** * @param ConfigInterface $config * @param ActionFactory $actionFactory @@ -75,6 +79,7 @@ class View extends DataObject implements ViewInterface * @param SubscriptionFactory $subscriptionFactory * @param array $data * @param array $changelogBatchSize + * @param ChangeLogBatchWalkerFactory $changeLogBatchWalkerFactory */ public function __construct( ConfigInterface $config, @@ -83,7 +88,8 @@ public function __construct( View\ChangelogInterface $changelog, SubscriptionFactory $subscriptionFactory, array $data = [], - array $changelogBatchSize = [] + array $changelogBatchSize = [], + ChangeLogBatchWalkerFactory $changeLogBatchWalkerFactory = null ) { $this->config = $config; $this->actionFactory = $actionFactory; @@ -92,6 +98,8 @@ public function __construct( $this->subscriptionFactory = $subscriptionFactory; $this->changelogBatchSize = $changelogBatchSize; parent::__construct($data); + $this->changeLogBatchWalkerFactory = $changeLogBatchWalkerFactory ?: + ObjectManager::getInstance()->get(ChangeLogBatchWalkerFactory::class); } /** @@ -296,41 +304,36 @@ private function executeAction(ActionInterface $action, int $lastVersionId, int : self::DEFAULT_BATCH_SIZE; $vsFrom = $lastVersionId; - while ($vsFrom < $currentVersionId) { - $ids = $this->getBatchOfIds($vsFrom, $currentVersionId); - // We run the actual indexer in batches. - // Chunked AFTER loading to avoid duplicates in separate chunks. - $chunks = array_chunk($ids, $batchSize); - foreach ($chunks as $ids) { - $action->execute($ids); - } + $iterator = $this->getWalker($this->getChangelog(), $vsFrom, $currentVersionId, $batchSize); + foreach ($iterator as $batchOfIds) { + $action->execute($batchOfIds); } } /** - * Get batch of entity ids + * Create and validate walker class for changelog * - * @param int $lastVersionId - * @param int $currentVersionId - * @return array - */ - private function getBatchOfIds(int &$lastVersionId, int $currentVersionId): array + * @param ChangelogInterface $changelog + * @param int $vsFrom + * @param int $vsTo + * @param int $batchSize + * @return ChangeLogBatchWalkerInterface|mixed + */ + private function getWalker( + ChangelogInterface $changelog, + int $vsFrom, + int $vsTo, + int $batchSize + ): ChangeLogBatchWalkerInterface { - $ids = []; - $versionBatchSize = self::$maxVersionQueryBatch; - $idsBatchSize = self::$maxVersionQueryBatch; - for ($vsFrom = $lastVersionId; $vsFrom < $currentVersionId; $vsFrom += $versionBatchSize) { - // Don't go past the current version for atomicity. - $versionTo = min($currentVersionId, $vsFrom + $versionBatchSize); - /** To avoid duplicate ids need to flip and merge the array */ - $ids += array_flip($this->getChangelog()->getList($vsFrom, $versionTo)); - $lastVersionId = $versionTo; - if (count($ids) >= $idsBatchSize) { - break; - } - } - - return array_keys($ids); + $config = $this->config->getView($this->changelog->getViewId()); + $walkerClass = $config['walker']; + return $this->changeLogBatchWalkerFactory->create($walkerClass, [ + 'changelog' => $changelog, + 'fromVersionId' => $vsFrom, + 'toVersionId' => $vsTo, + 'batchSize' => $batchSize + ]); } /** diff --git a/lib/internal/Magento/Framework/Mview/View/AdditionalColumnProcessorInterface.php b/lib/internal/Magento/Framework/Mview/View/AdditionalColumnProcessorInterface.php new file mode 100644 index 0000000000000..49af83556d023 --- /dev/null +++ b/lib/internal/Magento/Framework/Mview/View/AdditionalColumnProcessorInterface.php @@ -0,0 +1,37 @@ +resourceConnection = $resourceConnection; + } + + /** + * @inheritDoc + */ + public function getTriggerColumns(string $eventPrefix, array $additionalColumns): array + { + $resource = $this->resourceConnection->getConnection(); + $triggersColumns = [ + 'column_names' => [], + 'column_values' => [] + ]; + + foreach ($additionalColumns as $additionalColumn) { + $triggersColumns['column_names'][$additionalColumn['name']] = $resource->quoteIdentifier( + $additionalColumn['cl_name'] + ); + + $triggersColumns['column_values'][$additionalColumn['name']] = isset($additionalColumn['constant']) ? + $resource->quote($additionalColumn['constant']) : + $eventPrefix . $resource->quoteIdentifier($additionalColumn['name']); + } + + return $triggersColumns; + } + + /** + * Retrieve pre-statement for a trigger in Mview + * + * @return string + */ + public function getPreStatements(): string + { + return ''; + } + + /** + * @inheritDoc + */ + public function processColumnForCLTable(Table $table, string $columnName): void + { + $table->addColumn( + $columnName, + \Magento\Framework\DB\Ddl\Table::TYPE_TEXT, + null, + ['unsigned' => true, 'nullable' => true, 'default' => null], + $columnName + ); + } +} diff --git a/lib/internal/Magento/Framework/Mview/View/AdditionalColumnsProcessor/ProcessorFactory.php b/lib/internal/Magento/Framework/Mview/View/AdditionalColumnsProcessor/ProcessorFactory.php new file mode 100644 index 0000000000000..5907cefbffd50 --- /dev/null +++ b/lib/internal/Magento/Framework/Mview/View/AdditionalColumnsProcessor/ProcessorFactory.php @@ -0,0 +1,38 @@ +objectManager = $objectManager; + } + + /** + * Instantiate additional columns processor + * + * @param string $processorClassName + * @return AdditionalColumnProcessorInterface + */ + public function create(string $processorClassName): AdditionalColumnProcessorInterface + { + return $this->objectManager->create($processorClassName); + } +} diff --git a/lib/internal/Magento/Framework/Mview/View/ChangeLogBatchWalker.php b/lib/internal/Magento/Framework/Mview/View/ChangeLogBatchWalker.php new file mode 100644 index 0000000000000..2f7be835b2a13 --- /dev/null +++ b/lib/internal/Magento/Framework/Mview/View/ChangeLogBatchWalker.php @@ -0,0 +1,104 @@ +resourceConnection = $resourceConnection; + $this->changelog = $changelog; + $this->fromVersionId = $fromVersionId; + $this->toVersionId = $toVersionId; + $this->batchSize = $batchSize; + } + + /** + * @return \Generator|\Traversable + * @throws \Exception + */ + public function getIterator(): \Generator + { + while ($this->fromVersionId < $this->toVersionId) { + $ids = $this->walk(); + + if (empty($ids)) { + break; + } + $this->fromVersionId += $this->batchSize; + yield $ids; + } + } + + /** + * @inheritdoc + */ + private function walk() + { + $connection = $this->resourceConnection->getConnection(); + $changelogTableName = $this->resourceConnection->getTableName($this->changelog->getName()); + + $select = $connection->select()->distinct(true) + ->where( + 'version_id > ?', + $this->fromVersionId + ) + ->where( + 'version_id <= ?', + $this->toVersionId + ) + ->group([$this->changelog->getColumnName()]) + ->limit($this->batchSize); + + $select->from($changelogTableName, [$this->changelog->getColumnName()]); + return $connection->fetchCol($select); + } +} diff --git a/lib/internal/Magento/Framework/Mview/View/ChangeLogBatchWalkerFactory.php b/lib/internal/Magento/Framework/Mview/View/ChangeLogBatchWalkerFactory.php new file mode 100644 index 0000000000000..8670f78368178 --- /dev/null +++ b/lib/internal/Magento/Framework/Mview/View/ChangeLogBatchWalkerFactory.php @@ -0,0 +1,38 @@ +objectManager = $objectManager; + } + + /** + * Instantiate BatchWalker interface + * + * @param string $batchWalkerClassName + * @param array $data + * @return ChangeLogBatchWalkerInterface + */ + public function create(string $batchWalkerClassName, array $data): ChangeLogBatchWalkerInterface + { + return $this->objectManager->create($batchWalkerClassName, $data); + } +} diff --git a/lib/internal/Magento/Framework/Mview/View/ChangeLogBatchWalkerInterface.php b/lib/internal/Magento/Framework/Mview/View/ChangeLogBatchWalkerInterface.php new file mode 100644 index 0000000000000..55d4abfcb4704 --- /dev/null +++ b/lib/internal/Magento/Framework/Mview/View/ChangeLogBatchWalkerInterface.php @@ -0,0 +1,20 @@ +connection = $resource->getConnection(); $this->resource = $resource; $this->checkConnection(); + $this->mviewConfig = $mviewConfig; + $this->additionalColumnsProcessorFactory = $additionalColumnsProcessorFactory ?? + ObjectManager::getInstance()->get(ProcessorFactory::class); } /** @@ -83,7 +110,7 @@ public function create() $table = $this->connection->newTable( $changelogTableName )->addColumn( - 'version_id', + self::VERSION_ID_COLUMN_NAME, \Magento\Framework\DB\Ddl\Table::TYPE_INTEGER, null, ['identity' => true, 'unsigned' => true, 'nullable' => false, 'primary' => true], @@ -95,10 +122,46 @@ public function create() ['unsigned' => true, 'nullable' => false, 'default' => '0'], 'Entity ID' ); + + foreach ($this->initAdditionalColumnData() as $columnData) { + /** @var AdditionalColumnProcessorInterface $processor */ + $processorClass = $columnData['processor']; + $processor = $this->additionalColumnsProcessorFactory->create($processorClass); + $processor->processColumnForCLTable($table, $columnData['cl_name']); + } + $this->connection->createTable($table); } } + /** + * Retrieve additional column data + * + * @return array + * @throws \Exception + */ + private function initAdditionalColumnData(): array + { + $config = $this->mviewConfig->getView($this->getViewId()); + $additionalColumns = []; + + if (!$config) { + return $additionalColumns; + } + + foreach ($config['subscriptions'] as $subscription) { + if (isset($subscription['additional_columns'])) { + foreach ($subscription['additional_columns'] as $additionalColumn) { + //We are gatherig unique change log column names in order to create them later + $additionalColumns[$additionalColumn['cl_name']] = $additionalColumn; + $additionalColumns[$additionalColumn['cl_name']]['processor'] = $subscription['processor']; + } + } + } + + return $additionalColumns; + } + /** * Drop changelog table * @@ -139,7 +202,7 @@ public function clear($versionId) * * @param int $fromVersionId * @param int $toVersionId - * @return int[] + * @return array * @throws ChangelogTableNotExistsException */ public function getList($fromVersionId, $toVersionId) diff --git a/lib/internal/Magento/Framework/Mview/View/ChangelogInterface.php b/lib/internal/Magento/Framework/Mview/View/ChangelogInterface.php index b00c1ca3a2e33..79f998cbe02b6 100644 --- a/lib/internal/Magento/Framework/Mview/View/ChangelogInterface.php +++ b/lib/internal/Magento/Framework/Mview/View/ChangelogInterface.php @@ -11,6 +11,11 @@ */ interface ChangelogInterface { + const ATTRIBUTE_SCOPE_SUPPORT = 'attribute_scope'; + const STORE_SCOPE_SUPPORT = 'store_scope'; + const ATTRIBUTE_COLUMN = 'attribute_id'; + const STORE_COLUMN = 'store_id'; + /** * Create changelog table * diff --git a/lib/internal/Magento/Framework/Mview/View/Subscription.php b/lib/internal/Magento/Framework/Mview/View/Subscription.php index ddfa39f0a089f..eacc376972a71 100644 --- a/lib/internal/Magento/Framework/Mview/View/Subscription.php +++ b/lib/internal/Magento/Framework/Mview/View/Subscription.php @@ -6,14 +6,17 @@ namespace Magento\Framework\Mview\View; +use Magento\Framework\App\ObjectManager; use Magento\Framework\App\ResourceConnection; use Magento\Framework\DB\Ddl\Trigger; +use Magento\Framework\Mview\Config; +use Magento\Framework\Mview\View\AdditionalColumnsProcessor\ProcessorFactory; use Magento\Framework\Mview\View\StateInterface; /** * Class Subscription * - * @package Magento\Framework\Mview\View + * Create triggers for subscription tables for Mview */ class Subscription implements SubscriptionInterface { @@ -69,6 +72,16 @@ class Subscription implements SubscriptionInterface */ protected $resource; + /** + * @var Config + */ + private $mviewConfig; + + /** + * @var ProcessorFactory|null + */ + private $additionalColumnsProcessorFactory; + /** * @param ResourceConnection $resource * @param \Magento\Framework\DB\Ddl\TriggerFactory $triggerFactory @@ -77,6 +90,10 @@ class Subscription implements SubscriptionInterface * @param string $tableName * @param string $columnName * @param array $ignoredUpdateColumns + * @param Config|null $mviewConfig + * @param ProcessorFactory|null $additionalColumnsProcessorFactory + * + * @SuppressWarnings(PHPMD.ExcessiveParameterList) */ public function __construct( ResourceConnection $resource, @@ -85,7 +102,9 @@ public function __construct( \Magento\Framework\Mview\ViewInterface $view, $tableName, $columnName, - $ignoredUpdateColumns = [] + $ignoredUpdateColumns = [], + Config $mviewConfig = null, + ProcessorFactory $additionalColumnsProcessorFactory = null ) { $this->connection = $resource->getConnection(); $this->triggerFactory = $triggerFactory; @@ -95,6 +114,9 @@ public function __construct( $this->columnName = $columnName; $this->resource = $resource; $this->ignoredUpdateColumns = $ignoredUpdateColumns; + $this->mviewConfig = $mviewConfig ?? ObjectManager::getInstance()->get(Config::class); + $this->additionalColumnsProcessorFactory = $additionalColumnsProcessorFactory ?? + ObjectManager::getInstance()->get(ProcessorFactory::class); } /** @@ -189,6 +211,41 @@ protected function getLinkedViews() return $this->linkedViews; } + /** + * Prepare columns for trigger statement. Should be protected in order to serve new approach + * + * @param ChangelogInterface $changelog + * @param string $event + * @return array + * @throws \Exception + */ + protected function prepareColumns(ChangelogInterface $changelog, string $event): array + { + $prefix = $event === Trigger::EVENT_DELETE ? 'OLD.' : 'NEW.'; + $subscriptionData = $this->mviewConfig->getView( + $changelog->getViewId() + )['subscriptions'][$this->getTableName()]; + + $columns = [ + 'column_names' => [ + 'entity_id' => $this->connection->quoteIdentifier($changelog->getColumnName()) + ], + 'column_values' => [ + 'entity_id' => $this->getEntityColumn($prefix) + ] + ]; + + if (!empty($subscriptionData['additional_columns'])) { + $processor = $this->getProcessor(); + $columns = array_replace_recursive( + $columns, + $processor->getTriggerColumns($prefix, $subscriptionData['additional_columns']) + ); + } + + return $columns; + } + /** * Build trigger statement for INSERT, UPDATE, DELETE events * @@ -198,13 +255,10 @@ protected function getLinkedViews() */ protected function buildStatement($event, $changelog) { + $trigger = "%sINSERT IGNORE INTO %s (%s) VALUES (%s);"; switch ($event) { - case Trigger::EVENT_INSERT: - $trigger = "INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);"; - break; case Trigger::EVENT_UPDATE: $tableName = $this->resource->getTableName($this->getTableName()); - $trigger = "INSERT IGNORE INTO %s (%s) VALUES (NEW.%s);"; if ($this->connection->isTableExists($tableName) && $describe = $this->connection->describeTable($tableName) ) { @@ -226,20 +280,40 @@ protected function buildStatement($event, $changelog) } } break; - case Trigger::EVENT_DELETE: - $trigger = "INSERT IGNORE INTO %s (%s) VALUES (OLD.%s);"; - break; - default: - return ''; } + $columns = $this->prepareColumns($changelog, $event); return sprintf( $trigger, + $this->getProcessor()->getPreStatements(), $this->connection->quoteIdentifier($this->resource->getTableName($changelog->getName())), - $this->connection->quoteIdentifier($changelog->getColumnName()), - $this->connection->quoteIdentifier($this->getColumnName()) + implode(", ", $columns['column_names']), + implode(", ", $columns['column_values']) ); } + /** + * Instantiate and retrieve additional columns processor + * + * @return AdditionalColumnProcessorInterface + */ + private function getProcessor(): AdditionalColumnProcessorInterface + { + $subscriptionData = $this->mviewConfig->getView($this->getView()->getId())['subscriptions']; + $processorClass = $subscriptionData[$this->getTableName()]['processor']; + return $this->additionalColumnsProcessorFactory->create($processorClass); + } + + /** + * Retrieves entity column for subscription tables + * + * @param string $prefix + * @return string + */ + public function getEntityColumn(string $prefix): string + { + return $prefix . $this->connection->quoteIdentifier($this->getColumnName()); + } + /** * Build an "after" event for the given table and event * diff --git a/lib/internal/Magento/Framework/Mview/etc/mview.xsd b/lib/internal/Magento/Framework/Mview/etc/mview.xsd index dfff4964f6587..04754fa499249 100644 --- a/lib/internal/Magento/Framework/Mview/etc/mview.xsd +++ b/lib/internal/Magento/Framework/Mview/etc/mview.xsd @@ -46,6 +46,7 @@ + @@ -76,9 +77,25 @@ Table declaration. + + + + + + + + + + + + + + + +