diff --git a/lib/internal/Magento/Framework/DB/Adapter/Pdo/Mysql.php b/lib/internal/Magento/Framework/DB/Adapter/Pdo/Mysql.php index 468885548b312..6399824590a69 100644 --- a/lib/internal/Magento/Framework/DB/Adapter/Pdo/Mysql.php +++ b/lib/internal/Magento/Framework/DB/Adapter/Pdo/Mysql.php @@ -26,6 +26,7 @@ use Magento\Framework\Phrase; use Magento\Framework\Stdlib\DateTime; use Magento\Framework\Stdlib\StringUtils; +use Magento\Framework\DB\Query\Generator as QueryGenerator; /** * @SuppressWarnings(PHPMD.ExcessivePublicCount) @@ -199,6 +200,11 @@ class Mysql extends \Zend_Db_Adapter_Pdo_Mysql implements AdapterInterface */ private $exceptionMap; + /** + * @var QueryGenerator + */ + private $queryGenerator; + /** * @param StringUtils $string * @param DateTime $dateTime @@ -3362,57 +3368,32 @@ public function insertFromSelect(Select $select, $table, array $fields = [], $mo * @param int $stepCount * @return \Magento\Framework\DB\Select[] * @throws LocalizedException + * @deprecated */ public function selectsByRange($rangeField, \Magento\Framework\DB\Select $select, $stepCount = 100) { - $fromSelect = $select->getPart(\Magento\Framework\DB\Select::FROM); - if (empty($fromSelect)) { - throw new LocalizedException( - new \Magento\Framework\Phrase('Select object must have correct "FROM" part') - ); - } - - $tableName = []; - $correlationName = ''; - foreach ($fromSelect as $correlationName => $formPart) { - if ($formPart['joinType'] == \Magento\Framework\DB\Select::FROM) { - $tableName = $formPart['tableName']; - break; - } - } - - $selectRange = $this->select() - ->from( - $tableName, - [ - new \Zend_Db_Expr('MIN(' . $this->quoteIdentifier($rangeField) . ') AS min'), - new \Zend_Db_Expr('MAX(' . $this->quoteIdentifier($rangeField) . ') AS max'), - ] - ); - - $rangeResult = $this->fetchRow($selectRange); - $min = $rangeResult['min']; - $max = $rangeResult['max']; - + $iterator = $this->getQueryGenerator()->generate($rangeField, $select, $stepCount); $queries = []; - while ($min <= $max) { - $partialSelect = clone $select; - $partialSelect->where( - $this->quoteIdentifier($correlationName) . '.' - . $this->quoteIdentifier($rangeField) . ' >= ?', - $min - ) - ->where( - $this->quoteIdentifier($correlationName) . '.' - . $this->quoteIdentifier($rangeField) . ' < ?', - $min + $stepCount - ); - $queries[] = $partialSelect; - $min += $stepCount; + foreach ($iterator as $query) { + $queries[] = $query; } return $queries; } + /** + * Get query generator + * + * @return QueryGenerator + * @deprecated + */ + private function getQueryGenerator() + { + if ($this->queryGenerator === null) { + $this->queryGenerator = \Magento\Framework\App\ObjectManager::getInstance()->create(QueryGenerator::class); + } + return $this->queryGenerator; + } + /** * Get update table query using select object for join and update * diff --git a/lib/internal/Magento/Framework/DB/Query/BatchIterator.php b/lib/internal/Magento/Framework/DB/Query/BatchIterator.php new file mode 100644 index 0000000000000..cf5e88c1b9944 --- /dev/null +++ b/lib/internal/Magento/Framework/DB/Query/BatchIterator.php @@ -0,0 +1,192 @@ +batchSize = $batchSize; + $this->select = $select; + $this->correlationName = $correlationName; + $this->rangeField = $rangeField; + $this->rangeFieldAlias = $rangeFieldAlias; + $this->connection = $select->getConnection(); + } + + /** + * @return Select + */ + public function current() + { + if (null == $this->currentSelect) { + $this->currentSelect = $this->initSelectObject(); + $itemsCount = $this->calculateBatchSize($this->currentSelect); + $this->isValid = $itemsCount > 0; + } + return $this->currentSelect; + } + + /** + * @return Select + */ + public function next() + { + if (null == $this->currentSelect) { + $this->current(); + } + $select = $this->initSelectObject(); + $itemsCountInSelect = $this->calculateBatchSize($select); + $this->isValid = $itemsCountInSelect > 0; + if ($this->isValid) { + $this->iteration++; + $this->currentSelect = $select; + } else { + $this->currentSelect = null; + } + return $this->currentSelect; + } + + /** + * @return int + */ + public function key() + { + return $this->iteration; + } + + /** + * @return bool + */ + public function valid() + { + return $this->isValid; + } + + /** + * @return void + */ + public function rewind() + { + $this->minValue = 0; + $this->currentSelect = null; + $this->iteration = 0; + $this->isValid = true; + } + + /** + * Calculate batch size for select. + * + * @param Select $select + * @return int + */ + private function calculateBatchSize(Select $select) + { + $wrapperSelect = $this->connection->select(); + $wrapperSelect->from( + $select, + [ + new \Zend_Db_Expr('MAX(' . $this->rangeFieldAlias . ') as max'), + new \Zend_Db_Expr('COUNT(*) as cnt') + ] + ); + $row = $this->connection->fetchRow($wrapperSelect); + $this->minValue = $row['max']; + return intval($row['cnt']); + } + + /** + * Initialize select object. + * + * @return \Magento\Framework\DB\Select + */ + private function initSelectObject() + { + $object = clone $this->select; + $object->where( + $this->connection->quoteIdentifier($this->correlationName) + . '.' . $this->connection->quoteIdentifier($this->rangeField) + . ' > ?', + $this->minValue + ); + $object->limit($this->batchSize); + /** + * Reset sort order section from origin select object + */ + $object->order($this->correlationName . '.' . $this->rangeField . ' ' . \Magento\Framework\DB\Select::SQL_ASC); + return $object; + } +} diff --git a/lib/internal/Magento/Framework/DB/Query/BatchIteratorFactory.php b/lib/internal/Magento/Framework/DB/Query/BatchIteratorFactory.php new file mode 100644 index 0000000000000..a51a650197def --- /dev/null +++ b/lib/internal/Magento/Framework/DB/Query/BatchIteratorFactory.php @@ -0,0 +1,51 @@ +objectManager = $objectManager; + $this->instanceName = $instanceName; + } + + /** + * Create class instance with specified parameters + * + * @param array $data + * @return \Magento\Framework\DB\Query\BatchIterator + */ + public function create(array $data = []) + { + return $this->objectManager->create($this->instanceName, $data); + } +} diff --git a/lib/internal/Magento/Framework/DB/Query/Generator.php b/lib/internal/Magento/Framework/DB/Query/Generator.php new file mode 100644 index 0000000000000..43f8388d3f449 --- /dev/null +++ b/lib/internal/Magento/Framework/DB/Query/Generator.php @@ -0,0 +1,79 @@ +iteratorFactory = $iteratorFactory; + } + + /** + * Generate select query list with predefined items count in each select item. + * + * @param string $rangeField + * @param \Magento\Framework\DB\Select $select + * @param int $batchSize + * @return BatchIterator + * @throws LocalizedException + */ + public function generate($rangeField, \Magento\Framework\DB\Select $select, $batchSize = 100) + { + $fromSelect = $select->getPart(\Magento\Framework\DB\Select::FROM); + if (empty($fromSelect)) { + throw new LocalizedException( + new \Magento\Framework\Phrase('Select object must have correct "FROM" part') + ); + } + + $fieldCorrelationName = ''; + foreach ($fromSelect as $correlationName => $fromPart) { + if ($fromPart['joinType'] == \Magento\Framework\DB\Select::FROM) { + $fieldCorrelationName = $correlationName; + break; + } + } + + $columns = $select->getPart(\Magento\Framework\DB\Select::COLUMNS); + /** + * Calculate $rangeField alias + */ + $rangeFieldAlias = $rangeField; + foreach ($columns as $column) { + list($table, $columnName, $alias) = $column; + if ($table == $fieldCorrelationName && $columnName == $rangeField) { + $rangeFieldAlias = $alias ?: $rangeField; + break; + } + } + + return $this->iteratorFactory->create( + [ + 'select' => $select, + 'batchSize' => $batchSize, + 'correlationName' => $fieldCorrelationName, + 'rangeField' => $rangeField, + 'rangeFieldAlias' => $rangeFieldAlias + ] + ); + } +} diff --git a/lib/internal/Magento/Framework/Test/Unit/DB/Query/BatchIteratorTest.php b/lib/internal/Magento/Framework/Test/Unit/DB/Query/BatchIteratorTest.php new file mode 100644 index 0000000000000..e522daf97a802 --- /dev/null +++ b/lib/internal/Magento/Framework/Test/Unit/DB/Query/BatchIteratorTest.php @@ -0,0 +1,200 @@ +batchSize = 10; + $this->correlationName = 'correlationName'; + $this->rangeField = 'rangeField'; + $this->rangeFieldAlias = 'rangeFieldAlias'; + + $this->selectMock = $this->getMock(Select::class, [], [], '', false, false); + $this->wrapperSelectMock = $this->getMock(Select::class, [], [], '', false, false); + $this->connectionMock = $this->getMock(AdapterInterface::class); + $this->connectionMock->expects($this->any())->method('select')->willReturn($this->wrapperSelectMock); + $this->selectMock->expects($this->once())->method('getConnection')->willReturn($this->connectionMock); + $this->connectionMock->expects($this->any())->method('quoteIdentifier')->willReturnArgument(0); + + $this->model = new BatchIterator( + $this->selectMock, + $this->batchSize, + $this->correlationName, + $this->rangeField, + $this->rangeFieldAlias + ); + } + + /** + * Test steps: + * 1. $iterator->current(); + * 2. $iterator->current(); + * 3. $iterator->key(); + * @return void + */ + public function testCurrent() + { + $filed = $this->correlationName . '.' . $this->rangeField; + + $this->selectMock->expects($this->once())->method('where')->with($filed . ' > ?', 0); + $this->selectMock->expects($this->once())->method('limit')->with($this->batchSize); + $this->selectMock->expects($this->once())->method('order')->with($filed . ' ASC'); + $this->assertEquals($this->selectMock, $this->model->current()); + $this->assertEquals($this->selectMock, $this->model->current()); + $this->assertEquals(0, $this->model->key()); + } + + /** + * SQL: select * from users + * Batch size: 10 + * IDS: [1 - 25] + * Items count: 25 + * Expected batches: [1-10, 11-20, 20-25] + * + * Test steps: + * 1. $iterator->rewind(); + * 2. $iterator->valid(); + * 3. $iterator->current(); + * 4. $iterator->key(); + * + * 1. $iterator->next() + * 2. $iterator->valid(); + * 3. $iterator->current(); + * 4. $iterator->key(); + * + * 1. $iterator->next() + * 2. $iterator->valid(); + * 3. $iterator->current(); + * 4. $iterator->key(); + * + * + * 1. $iterator->next() + * 2. $iterator->valid(); + * @return void + */ + public function testIterations() + { + $startCallIndex = 3; + $stepCall = 4; + + $this->connectionMock->expects($this->at($startCallIndex)) + ->method('fetchRow') + ->willReturn(['max' => 10, 'cnt' => 10]); + + $this->connectionMock->expects($this->at($startCallIndex += $stepCall)) + ->method('fetchRow') + ->willReturn(['max' => 20, 'cnt' => 10]); + + $this->connectionMock->expects($this->at($startCallIndex += $stepCall)) + ->method('fetchRow') + ->willReturn(['max' => 25, 'cnt' => 5]); + + $this->connectionMock->expects($this->at($startCallIndex += $stepCall)) + ->method('fetchRow') + ->willReturn(['max' => null, 'cnt' => 0]); + + /** + * Test 3 iterations + * [1-10, 11-20, 20-25] + */ + $iteration = 0; + $result = []; + foreach ($this->model as $key => $select) { + $result[] = $select; + $this->assertEquals($iteration, $key); + $iteration++; + } + $this->assertCount(3, $result); + } + + /** + * Test steps: + * 1. $iterator->next(); + * 2. $iterator->key() + * 3. $iterator->next(); + * 4. $iterator->current() + * 5. $iterator->key() + * @return void + */ + public function testNext() + { + $filed = $this->correlationName . '.' . $this->rangeField; + $this->selectMock->expects($this->at(0))->method('where')->with($filed . ' > ?', 0); + $this->selectMock->expects($this->exactly(3))->method('limit')->with($this->batchSize); + $this->selectMock->expects($this->exactly(3))->method('order')->with($filed . ' ASC'); + $this->selectMock->expects($this->at(3))->method('where')->with($filed . ' > ?', 25); + + $this->wrapperSelectMock->expects($this->exactly(3))->method('from')->with( + $this->selectMock, + [ + new \Zend_Db_Expr('MAX(' . $this->rangeFieldAlias . ') as max'), + new \Zend_Db_Expr('COUNT(*) as cnt') + ] + ); + $this->connectionMock->expects($this->exactly(3)) + ->method('fetchRow') + ->with($this->wrapperSelectMock) + ->willReturn(['max' => 25, 'cnt' => 10]); + + $this->assertEquals($this->selectMock, $this->model->next()); + $this->assertEquals(1, $this->model->key()); + + $this->assertEquals($this->selectMock, $this->model->next()); + $this->assertEquals($this->selectMock, $this->model->current()); + $this->assertEquals(2, $this->model->key()); + } +} diff --git a/lib/internal/Magento/Framework/Test/Unit/DB/Query/GeneratorTest.php b/lib/internal/Magento/Framework/Test/Unit/DB/Query/GeneratorTest.php new file mode 100644 index 0000000000000..4be22eb14ca85 --- /dev/null +++ b/lib/internal/Magento/Framework/Test/Unit/DB/Query/GeneratorTest.php @@ -0,0 +1,168 @@ +factoryMock = $this->getMock(BatchIteratorFactory::class, [], [], '', false, false); + $this->selectMock = $this->getMock(Select::class, [], [], '', false, false); + $this->iteratorMock = $this->getMock(BatchIterator::class, [], [], '', false, false); + $this->model = new Generator($this->factoryMock); + } + + /** + * Test success generate. + * @return void + */ + public function testGenerate() + { + $map = [ + [ + Select::FROM, + [ + 'cp' => ['joinType' => Select::FROM] + ] + ], + [ + Select::COLUMNS, + [ + ['cp', 'entity_id', 'product_id'] + ] + ] + ]; + $this->selectMock->expects($this->exactly(2))->method('getPart')->willReturnMap($map); + $this->factoryMock->expects($this->once())->method('create')->with( + [ + 'select' => $this->selectMock, + 'batchSize' => 100, + 'correlationName' => 'cp', + 'rangeField' => 'entity_id', + 'rangeFieldAlias' => 'product_id' + ] + )->willReturn($this->iteratorMock); + $this->assertEquals($this->iteratorMock, $this->model->generate('entity_id', $this->selectMock, 100)); + } + + /** + * Test batch generation with invalid select object. + * + * @expectedException \Magento\Framework\Exception\LocalizedException + * @expectedExceptionMessage Select object must have correct "FROM" part + * @return void + */ + public function testGenerateWithoutFromPart() + { + $map = [ + [Select::FROM, []], + [ + Select::COLUMNS, + [ + ['cp', 'entity_id', 'product_id'] + ] + ] + ]; + $this->selectMock->expects($this->any())->method('getPart')->willReturnMap($map); + $this->factoryMock->expects($this->never())->method('create'); + $this->model->generate('entity_id', $this->selectMock, 100); + } + + /** + * Test generate batches with rangeField without alias. + * @return void + */ + public function testGenerateWithRangeFieldWithoutAlias() + { + $map = [ + [ + Select::FROM, + [ + 'cp' => ['joinType' => Select::FROM] + ] + ], + [ + Select::COLUMNS, + [ + ['cp', 'entity_id', null] + ] + ] + ]; + $this->selectMock->expects($this->exactly(2))->method('getPart')->willReturnMap($map); + $this->factoryMock->expects($this->once())->method('create')->with( + [ + 'select' => $this->selectMock, + 'batchSize' => 100, + 'correlationName' => 'cp', + 'rangeField' => 'entity_id', + 'rangeFieldAlias' => 'entity_id' + ] + )->willReturn($this->iteratorMock); + $this->assertEquals($this->iteratorMock, $this->model->generate('entity_id', $this->selectMock, 100)); + } + + /** + * Test generate batches with wild-card. + * + * @return void + */ + public function testGenerateWithInvalidWithWildcard() + { + $map = [ + [ + Select::FROM, + [ + 'cp' => ['joinType' => Select::FROM] + ] + ], + [ + Select::COLUMNS, + [ + ['cp', '*', null] + ] + ] + ]; + $this->selectMock->expects($this->exactly(2))->method('getPart')->willReturnMap($map); + $this->factoryMock->expects($this->once())->method('create')->with( + [ + 'select' => $this->selectMock, + 'batchSize' => 100, + 'correlationName' => 'cp', + 'rangeField' => 'entity_id', + 'rangeFieldAlias' => 'entity_id' + ] + )->willReturn($this->iteratorMock); + $this->assertEquals($this->iteratorMock, $this->model->generate('entity_id', $this->selectMock, 100)); + } +}