From 6b2625a7ebf96b841db387b6a89c2d9a996fb888 Mon Sep 17 00:00:00 2001 From: Dan Wallis Date: Mon, 23 Dec 2019 09:10:18 +0000 Subject: [PATCH 1/7] Invalidate indexers in RecurringData Calling `$indexer->reindexAll();` can take a long time with a large database. This code is run during `php bin/magento setup:upgrade`, which is part of a typical deployment. This change makes the step significantly quicker, while still achieving the same result. Minimising downtime during deployments seems sensible. --- app/code/Magento/Customer/Setup/RecurringData.php | 2 +- app/code/Magento/Theme/Setup/RecurringData.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/code/Magento/Customer/Setup/RecurringData.php b/app/code/Magento/Customer/Setup/RecurringData.php index fbef4c05d126d..77356e4765155 100644 --- a/app/code/Magento/Customer/Setup/RecurringData.php +++ b/app/code/Magento/Customer/Setup/RecurringData.php @@ -38,6 +38,6 @@ public function __construct(IndexerRegistry $indexerRegistry) public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) { $indexer = $this->indexerRegistry->get(Customer::CUSTOMER_GRID_INDEXER_ID); - $indexer->reindexAll(); + $indexer->invalidate(); } } diff --git a/app/code/Magento/Theme/Setup/RecurringData.php b/app/code/Magento/Theme/Setup/RecurringData.php index 972b60dc67f89..2e405c5cac619 100644 --- a/app/code/Magento/Theme/Setup/RecurringData.php +++ b/app/code/Magento/Theme/Setup/RecurringData.php @@ -48,7 +48,7 @@ public function __construct(Registration $themeRegistration, IndexerRegistry $in public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) { $indexer = $this->indexerRegistry->get(Config::DESIGN_CONFIG_GRID_INDEXER_ID); - $indexer->reindexAll(); + $indexer->invalidate(); $this->themeRegistration->register(); } } From 2dc822f9fd318d6b2a04e44764e9cab236010feb Mon Sep 17 00:00:00 2001 From: Dan Wallis Date: Mon, 23 Dec 2019 09:41:08 +0000 Subject: [PATCH 2/7] Update docblocks for linter --- app/code/Magento/Customer/Setup/RecurringData.php | 2 +- app/code/Magento/Theme/Setup/RecurringData.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/code/Magento/Customer/Setup/RecurringData.php b/app/code/Magento/Customer/Setup/RecurringData.php index 77356e4765155..4290602dc113a 100644 --- a/app/code/Magento/Customer/Setup/RecurringData.php +++ b/app/code/Magento/Customer/Setup/RecurringData.php @@ -33,7 +33,7 @@ public function __construct(IndexerRegistry $indexerRegistry) } /** - * {@inheritdoc} + * @inheritDoc */ public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) { diff --git a/app/code/Magento/Theme/Setup/RecurringData.php b/app/code/Magento/Theme/Setup/RecurringData.php index 2e405c5cac619..8e85b6b3cfd52 100644 --- a/app/code/Magento/Theme/Setup/RecurringData.php +++ b/app/code/Magento/Theme/Setup/RecurringData.php @@ -43,7 +43,7 @@ public function __construct(Registration $themeRegistration, IndexerRegistry $in } /** - * {@inheritdoc} + * @inheritDoc */ public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) { From deb1b238e4ca88f1e1b6dcd6ca21b5e0cb6f208c Mon Sep 17 00:00:00 2001 From: Dan Wallis Date: Mon, 23 Dec 2019 20:59:58 +0000 Subject: [PATCH 3/7] Create InstallData scripts for integration tests --- .../Magento/Customer/Setup/InstallData.php | 43 +++++++++++++++ app/code/Magento/Theme/Setup/InstallData.php | 54 +++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 app/code/Magento/Customer/Setup/InstallData.php create mode 100644 app/code/Magento/Theme/Setup/InstallData.php diff --git a/app/code/Magento/Customer/Setup/InstallData.php b/app/code/Magento/Customer/Setup/InstallData.php new file mode 100644 index 0000000000000..b56b96dbf31b4 --- /dev/null +++ b/app/code/Magento/Customer/Setup/InstallData.php @@ -0,0 +1,43 @@ +indexerRegistry = $indexerRegistry; + } + + /** + * @inheritDoc + */ + public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) + { + $indexer = $this->indexerRegistry->get(Customer::CUSTOMER_GRID_INDEXER_ID); + $indexer->reindexAll(); + } +} diff --git a/app/code/Magento/Theme/Setup/InstallData.php b/app/code/Magento/Theme/Setup/InstallData.php new file mode 100644 index 0000000000000..ff7da64bf03a6 --- /dev/null +++ b/app/code/Magento/Theme/Setup/InstallData.php @@ -0,0 +1,54 @@ +themeRegistration = $themeRegistration; + $this->indexerRegistry = $indexerRegistry; + } + + /** + * @inheritDoc + */ + public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) + { + $indexer = $this->indexerRegistry->get(Config::DESIGN_CONFIG_GRID_INDEXER_ID); + $indexer->reindexAll(); + $this->themeRegistration->register(); + } +} From 77f0d38ff4c698d9e0fdc7ac01af7499512114fe Mon Sep 17 00:00:00 2001 From: Dan Wallis Date: Mon, 23 Dec 2019 21:33:29 +0000 Subject: [PATCH 4/7] Replace InstallData scripts with Data Patches --- .../Magento/Customer/Setup/InstallData.php | 43 -------------- .../Setup/Patch/Data/ReindexCustomerGrid.php | 54 +++++++++++++++++ app/code/Magento/Theme/Setup/InstallData.php | 54 ----------------- .../Setup/Patch/Data/ReindexConfigGrid.php | 58 +++++++++++++++++++ 4 files changed, 112 insertions(+), 97 deletions(-) delete mode 100644 app/code/Magento/Customer/Setup/InstallData.php create mode 100644 app/code/Magento/Customer/Setup/Patch/Data/ReindexCustomerGrid.php delete mode 100644 app/code/Magento/Theme/Setup/InstallData.php create mode 100644 app/code/Magento/Theme/Setup/Patch/Data/ReindexConfigGrid.php diff --git a/app/code/Magento/Customer/Setup/InstallData.php b/app/code/Magento/Customer/Setup/InstallData.php deleted file mode 100644 index b56b96dbf31b4..0000000000000 --- a/app/code/Magento/Customer/Setup/InstallData.php +++ /dev/null @@ -1,43 +0,0 @@ -indexerRegistry = $indexerRegistry; - } - - /** - * @inheritDoc - */ - public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) - { - $indexer = $this->indexerRegistry->get(Customer::CUSTOMER_GRID_INDEXER_ID); - $indexer->reindexAll(); - } -} diff --git a/app/code/Magento/Customer/Setup/Patch/Data/ReindexCustomerGrid.php b/app/code/Magento/Customer/Setup/Patch/Data/ReindexCustomerGrid.php new file mode 100644 index 0000000000000..9e42070010542 --- /dev/null +++ b/app/code/Magento/Customer/Setup/Patch/Data/ReindexCustomerGrid.php @@ -0,0 +1,54 @@ +indexerRegistry = $indexerRegistry; + } + + /** + * @inheritDoc + */ + public function apply() + { + $indexer = $this->indexerRegistry->get(Customer::CUSTOMER_GRID_INDEXER_ID); + $indexer->reindexAll(); + } + + /** + * @inheritDoc + */ + public static function getDependencies() + { + return []; + } + + /** + * @inheritDoc + */ + public function getAliases() + { + return []; + } +} diff --git a/app/code/Magento/Theme/Setup/InstallData.php b/app/code/Magento/Theme/Setup/InstallData.php deleted file mode 100644 index ff7da64bf03a6..0000000000000 --- a/app/code/Magento/Theme/Setup/InstallData.php +++ /dev/null @@ -1,54 +0,0 @@ -themeRegistration = $themeRegistration; - $this->indexerRegistry = $indexerRegistry; - } - - /** - * @inheritDoc - */ - public function install(ModuleDataSetupInterface $setup, ModuleContextInterface $context) - { - $indexer = $this->indexerRegistry->get(Config::DESIGN_CONFIG_GRID_INDEXER_ID); - $indexer->reindexAll(); - $this->themeRegistration->register(); - } -} diff --git a/app/code/Magento/Theme/Setup/Patch/Data/ReindexConfigGrid.php b/app/code/Magento/Theme/Setup/Patch/Data/ReindexConfigGrid.php new file mode 100644 index 0000000000000..975e695ce6a47 --- /dev/null +++ b/app/code/Magento/Theme/Setup/Patch/Data/ReindexConfigGrid.php @@ -0,0 +1,58 @@ +indexerRegistry = $indexerRegistry; + } + + /** + * @inheritDoc + */ + public function apply() + { + $indexer = $this->indexerRegistry->get(Config::DESIGN_CONFIG_GRID_INDEXER_ID); + $indexer->reindexAll(); + } + + /** + * @inheritDoc + */ + public static function getDependencies() + { + return []; + } + + /** + * @inheritDoc + */ + public function getAliases() + { + return []; + } +} From aaf0c6e39302f36ef3995099f6d243647a00f20b Mon Sep 17 00:00:00 2001 From: Dan Wallis Date: Mon, 23 Dec 2019 22:20:42 +0000 Subject: [PATCH 5/7] Fix linter complaints --- .../Magento/Customer/Setup/Patch/Data/ReindexCustomerGrid.php | 4 ++++ app/code/Magento/Theme/Setup/Patch/Data/ReindexConfigGrid.php | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/code/Magento/Customer/Setup/Patch/Data/ReindexCustomerGrid.php b/app/code/Magento/Customer/Setup/Patch/Data/ReindexCustomerGrid.php index 9e42070010542..e6d9a3645b384 100644 --- a/app/code/Magento/Customer/Setup/Patch/Data/ReindexCustomerGrid.php +++ b/app/code/Magento/Customer/Setup/Patch/Data/ReindexCustomerGrid.php @@ -10,6 +10,9 @@ use Magento\Framework\Indexer\IndexerRegistry; use Magento\Framework\Setup\Patch\DataPatchInterface; +/** + * Perform a full reindex to ensure the grid table exists + */ class ReindexCustomerGrid implements DataPatchInterface { /** @@ -34,6 +37,7 @@ public function apply() { $indexer = $this->indexerRegistry->get(Customer::CUSTOMER_GRID_INDEXER_ID); $indexer->reindexAll(); + return $this; } /** diff --git a/app/code/Magento/Theme/Setup/Patch/Data/ReindexConfigGrid.php b/app/code/Magento/Theme/Setup/Patch/Data/ReindexConfigGrid.php index 975e695ce6a47..87053aeb23d17 100644 --- a/app/code/Magento/Theme/Setup/Patch/Data/ReindexConfigGrid.php +++ b/app/code/Magento/Theme/Setup/Patch/Data/ReindexConfigGrid.php @@ -11,8 +11,7 @@ use Magento\Theme\Model\Data\Design\Config; /** - * Class ReindexConfigGrid - * @package Magento\Theme\Setup\Patch + * Perform a full reindex to ensure the grid table exists */ class ReindexConfigGrid implements DataPatchInterface { @@ -38,6 +37,7 @@ public function apply() { $indexer = $this->indexerRegistry->get(Config::DESIGN_CONFIG_GRID_INDEXER_ID); $indexer->reindexAll(); + return $this; } /** From 00fd7f7302ddcbae49a81cecc7863fea03c2ff0c Mon Sep 17 00:00:00 2001 From: Dan Wallis Date: Sun, 29 Dec 2019 16:36:38 +0000 Subject: [PATCH 6/7] Ensure indexer is valid for integration tests --- .../Grid/CollectionReindexOnAccountLockTest.php | 1 + ..._grid_indexer_enabled_update_on_schedule.php | 7 +++++++ ...exer_enabled_update_on_schedule_rollback.php | 7 +++++++ .../customer_grid_indexer_ensure_valid.php | 17 +++++++++++++++++ 4 files changed, 32 insertions(+) create mode 100644 dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_ensure_valid.php diff --git a/dev/tests/integration/testsuite/Magento/Customer/Model/ResourceModel/Grid/CollectionReindexOnAccountLockTest.php b/dev/tests/integration/testsuite/Magento/Customer/Model/ResourceModel/Grid/CollectionReindexOnAccountLockTest.php index a665046ceef0d..d6cde96eab417 100644 --- a/dev/tests/integration/testsuite/Magento/Customer/Model/ResourceModel/Grid/CollectionReindexOnAccountLockTest.php +++ b/dev/tests/integration/testsuite/Magento/Customer/Model/ResourceModel/Grid/CollectionReindexOnAccountLockTest.php @@ -69,6 +69,7 @@ private function getCustomerGridLockExpire(): ?string * Test if customer account lock on too many failed authentication attempts triggers customer grid reindex * * @magentoDataFixture Magento/Customer/_files/customer.php + * @magentoDataFixture Magento/Customer/_files/customer_grid_indexer_ensure_valid.php * @magentoAppIsolation enabled * @magentoDbIsolation disabled */ diff --git a/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_enabled_update_on_schedule.php b/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_enabled_update_on_schedule.php index a7d7caec1fb83..56cad85a66d08 100644 --- a/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_enabled_update_on_schedule.php +++ b/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_enabled_update_on_schedule.php @@ -9,3 +9,10 @@ $indexerRegistry = $objectManager->create(\Magento\Framework\Indexer\IndexerRegistry::class); $indexer = $indexerRegistry->get(\Magento\Customer\Model\Customer::CUSTOMER_GRID_INDEXER_ID); $indexer->setScheduled(true); + +// Because integration tests don't run cron, we need to manually ensure the +// index isn't still invalid as that will cause asumptions in the tests to no +// longer hold. +if ($indexer->isInvalid()) { + $indexer->reindexAll(); +} diff --git a/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_enabled_update_on_schedule_rollback.php b/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_enabled_update_on_schedule_rollback.php index 9d7dc7bdc7d03..e835cb76b3c71 100644 --- a/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_enabled_update_on_schedule_rollback.php +++ b/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_enabled_update_on_schedule_rollback.php @@ -9,3 +9,10 @@ $indexerRegistry = $objectManager->create(\Magento\Framework\Indexer\IndexerRegistry::class); $indexer = $indexerRegistry->get(\Magento\Customer\Model\Customer::CUSTOMER_GRID_INDEXER_ID); $indexer->setScheduled(false); + +// Because integration tests don't run cron, we need to manually ensure the +// index isn't still invalid as that will cause asumptions in the tests to no +// longer hold. +if ($indexer->isInvalid()) { + $indexer->reindexAll(); +} diff --git a/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_ensure_valid.php b/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_ensure_valid.php new file mode 100644 index 0000000000000..94cf690c0c4db --- /dev/null +++ b/dev/tests/integration/testsuite/Magento/Customer/_files/customer_grid_indexer_ensure_valid.php @@ -0,0 +1,17 @@ +create(\Magento\Framework\Indexer\IndexerRegistry::class); +$indexer = $indexerRegistry->get(\Magento\Customer\Model\Customer::CUSTOMER_GRID_INDEXER_ID); + +// Because integration tests don't run cron, we need to manually ensure the +// index isn't still invalid as that will cause asumptions in the tests to no +// longer hold. +if ($indexer->isInvalid()) { + $indexer->reindexAll(); +} From 46b035d822621553dc1ff1d2c22ec03aa660094b Mon Sep 17 00:00:00 2001 From: Dan Wallis Date: Wed, 22 Jan 2020 23:54:33 +0000 Subject: [PATCH 7/7] Add test to cover recurring data task --- .../Magento/Customer/Setup/UpgradeTest.php | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 dev/tests/integration/testsuite/Magento/Customer/Setup/UpgradeTest.php diff --git a/dev/tests/integration/testsuite/Magento/Customer/Setup/UpgradeTest.php b/dev/tests/integration/testsuite/Magento/Customer/Setup/UpgradeTest.php new file mode 100644 index 0000000000000..d21a929a6eb04 --- /dev/null +++ b/dev/tests/integration/testsuite/Magento/Customer/Setup/UpgradeTest.php @@ -0,0 +1,54 @@ +create(IndexerRegistry::class); + $this->indexer = $indexerRegistry->get(Customer::CUSTOMER_GRID_INDEXER_ID); + $this->shell = $objectManager->create(Shell::class); + } + + /** + * Clean up shared dependencies + */ + protected function tearDown() + { + $this->indexer->reindexAll(); + } + + public function testSetupUpgrade() + { + $this->shell->execute( + PHP_BINARY . ' -f %s setup:upgrade', + [BP . '/bin/magento'] + ); + + $this->assertSame( + StateInterface::STATUS_INVALID, + $this->indexer->getStatus() + ); + } +}