Skip to content

Commit 69a02ea

Browse files
adifucanadifucan
authored
Merge pull request #7872 from magento-performance/ACPT-99-2
ACPT-99: Import API Parallelization
2 parents 62aafe2 + f661c0e commit 69a02ea

File tree

20 files changed

+292
-52
lines changed

20 files changed

+292
-52
lines changed

‎app/code/Magento/AdvancedPricingImportExport/Model/Import/AdvancedPricing.php‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -316,7 +316,6 @@ protected function _importData()
316316
} elseif (\Magento\ImportExport\Model\Import::BEHAVIOR_APPEND == $this->getBehavior()) {
317317
$this->saveAdvancedPricing();
318318
}
319-
320319
return true;
321320
}
322321

@@ -342,7 +341,7 @@ public function deleteAdvancedPricing()
342341
{
343342
$this->_cachedSkuToDelete = null;
344343
$listSku = [];
345-
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
344+
while ($bunch = $this->_dataSourceModel->getNextUniqueBunch($this->getIds())) {
346345
foreach ($bunch as $rowNum => $rowData) {
347346
$this->validateRow($rowData, $rowNum);
348347
if (!$this->getErrorAggregator()->isRowInvalid($rowNum)) {
@@ -389,7 +388,7 @@ protected function saveAndReplaceAdvancedPrices()
389388
}
390389
$listSku = [];
391390
$tierPrices = [];
392-
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
391+
while ($bunch = $this->_dataSourceModel->getNextUniqueBunch($this->getIds())) {
393392
$bunchTierPrices = [];
394393
foreach ($bunch as $rowNum => $rowData) {
395394
if (!$this->validateRow($rowData, $rowNum)) {

‎app/code/Magento/AdvancedPricingImportExport/Test/Unit/Model/Import/AdvancedPricingTest.php‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ class AdvancedPricingTest extends AbstractImportTestCase
3939
/**
4040
* DB Table data
4141
*/
42-
const TABLE_NAME = 'tableName';
43-
const LINK_FIELD = 'linkField';
42+
public const TABLE_NAME = 'tableName';
43+
public const LINK_FIELD = 'linkField';
4444

4545
/**
4646
* @var ResourceFactory|MockObject
@@ -328,7 +328,7 @@ public function testSaveAndReplaceAdvancedPricesAddRowErrorCall(): void
328328
]
329329
];
330330
$this->dataSourceModel
331-
->method('getNextBunch')
331+
->method('getNextUniqueBunch')
332332
->willReturnOnConsecutiveCalls($testBunch);
333333
$this->advancedPricing->expects($this->once())->method('validateRow')->willReturn(false);
334334
$this->advancedPricing->method('saveProductPrices')->willReturnSelf();
@@ -400,7 +400,7 @@ public function testSaveAndReplaceAdvancedPricesAppendBehaviourDataAndCalls(
400400
->method('getBehavior')
401401
->willReturn(Import::BEHAVIOR_APPEND);
402402
$this->dataSourceModel
403-
->method('getNextBunch')
403+
->method('getNextUniqueBunch')
404404
->willReturnOnConsecutiveCalls($data);
405405
$advancedPricing->method('validateRow')->willReturn(true);
406406

@@ -524,7 +524,7 @@ public function testSaveAndReplaceAdvancedPricesReplaceBehaviourInternalCalls():
524524
Import::BEHAVIOR_REPLACE
525525
);
526526
$this->dataSourceModel
527-
->method('getNextBunch')
527+
->method('getNextUniqueBunch')
528528
->willReturnOnConsecutiveCalls($data);
529529
$this->advancedPricing->expects($this->once())->method('validateRow')->willReturn(true);
530530

@@ -577,7 +577,7 @@ public function testDeleteAdvancedPricingFormListSkuToDelete(): void
577577
];
578578

579579
$this->dataSourceModel
580-
->method('getNextBunch')
580+
->method('getNextUniqueBunch')
581581
->willReturnOnConsecutiveCalls($data);
582582
$this->advancedPricing->method('validateRow')->willReturn(true);
583583
$expectedSkuList = ['sku value'];

‎app/code/Magento/CatalogImportExport/Model/Import/Product.php‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
use Magento\Framework\Exception\NoSuchEntityException;
2525
use Magento\Framework\Filesystem;
2626
use Magento\Framework\Filesystem\Driver\File;
27-
use Magento\Framework\Filesystem\DriverPool;
2827
use Magento\Framework\Intl\DateTimeFactory;
2928
use Magento\Framework\Model\ResourceModel\Db\ObjectRelationProcessor;
3029
use Magento\Framework\Model\ResourceModel\Db\TransactionManagerInterface;
@@ -1062,7 +1061,7 @@ protected function _deleteProducts()
10621061
{
10631062
$productEntityTable = $this->_resourceFactory->create()->getEntityTable();
10641063

1065-
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
1064+
while ($bunch = $this->_dataSourceModel->getNextUniqueBunch($this->getIds())) {
10661065
$idsToDelete = [];
10671066

10681067
foreach ($bunch as $rowNum => $rowData) {
@@ -1158,13 +1157,18 @@ protected function _saveProductsData()
11581157
foreach ($this->_productTypeModels as $productTypeModel) {
11591158
$productTypeModel->saveData();
11601159
}
1161-
$this->linkProcessor->saveLinks($this, $this->_dataSourceModel, $this->getProductEntityLinkField());
1160+
$this->linkProcessor->saveLinks(
1161+
$this,
1162+
$this->_dataSourceModel,
1163+
$this->getProductEntityLinkField(),
1164+
$this->getIds()
1165+
);
11621166
$this->_saveStockItem();
11631167
if ($this->_replaceFlag) {
11641168
$this->getOptionEntity()->clearProductsSkuToId();
11651169
}
1170+
$this->getOptionEntity()->setIds($this->getIds());
11661171
$this->getOptionEntity()->importData();
1167-
11681172
return $this;
11691173
}
11701174

@@ -1300,7 +1304,7 @@ protected function _prepareRowForDb(array $rowData)
13001304
*/
13011305
protected function _saveLinks()
13021306
{
1303-
$this->linkProcessor->saveLinks($this, $this->_dataSourceModel, $this->getProductEntityLinkField());
1307+
$this->linkProcessor->saveLinks($this, $this->_dataSourceModel, $this->getProductEntityLinkField(), []);
13041308
return $this;
13051309
}
13061310

@@ -1586,7 +1590,7 @@ protected function _saveProducts()
15861590
$productsQty = null;
15871591
$entityLinkField = $this->getProductEntityLinkField();
15881592

1589-
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
1593+
while ($bunch = $this->_dataSourceModel->getNextUniqueBunch($this->getIds())) {
15901594
$entityRowsIn = [];
15911595
$entityRowsUp = [];
15921596
$attributes = [];
@@ -1847,7 +1851,6 @@ protected function _saveProducts()
18471851
$mediaGalleryStoreData['disabled'] = 0;
18481852
$mediaGallery[Store::DEFAULT_STORE_ID][$rowSku][$uploadedFile] = $mediaGalleryStoreData;
18491853
}
1850-
18511854
}
18521855
}
18531856
}
@@ -2336,7 +2339,7 @@ protected function _saveProductWebsites(array $websiteData)
23362339
*/
23372340
protected function _saveStockItem()
23382341
{
2339-
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
2342+
while ($bunch = $this->_dataSourceModel->getNextUniqueBunch($this->getIds())) {
23402343
$stockData = [];
23412344
$importedData = [];
23422345
$productIdsToReindex = [];
@@ -2475,7 +2478,7 @@ public function getNewSku($sku = null)
24752478
*/
24762479
public function getNextBunch()
24772480
{
2478-
return $this->_dataSourceModel->getNextBunch();
2481+
return $this->_dataSourceModel->getNextUniqueBunch($this->getIds());
24792482
}
24802483

24812484
/**

‎app/code/Magento/CatalogImportExport/Model/Import/Product/LinkProcessor.php‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,13 +78,15 @@ public function __construct(
7878
* @param Product $importEntity
7979
* @param Data $dataSourceModel
8080
* @param string $linkField
81+
* @param array $ids
8182
* @return void
8283
* @throws LocalizedException
8384
*/
8485
public function saveLinks(
8586
Product $importEntity,
8687
Data $dataSourceModel,
87-
string $linkField
88+
string $linkField,
89+
array $ids
8890
): void {
8991
$resource = $this->linkFactory->create();
9092
$mainTable = $resource->getMainTable();
@@ -101,7 +103,7 @@ public function saveLinks(
101103
$bind = [':link_id' => $linkId, ':position' => 'position'];
102104
$positionAttrId[$linkId] = $importEntity->getConnection()->fetchOne($select, $bind);
103105
}
104-
while ($bunch = $dataSourceModel->getNextBunch()) {
106+
while ($bunch = $dataSourceModel->getNextUniqueBunch($ids)) {
105107
$nextLinkId = $this->resourceHelper->getNextAutoincrement($mainTable);
106108
$this->processLinkBunches($importEntity, $linkField, $bunch, $resource, $nextLinkId, $positionAttrId);
107109
}
@@ -111,6 +113,7 @@ public function saveLinks(
111113
* Add link types (exists for backwards compatibility)
112114
*
113115
* @deprecated 101.1.0 Use DI to inject to the constructor
116+
* @see Nothing
114117
* @param array $nameToIds
115118
*/
116119
public function addNameToIds(array $nameToIds): void

‎app/code/Magento/CatalogImportExport/Model/Import/Product/Option.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1256,7 +1256,7 @@ protected function _importData()
12561256
$prevOptionId = 0;
12571257
$optionId = null;
12581258
$valueId = null;
1259-
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
1259+
while ($bunch = $this->_dataSourceModel->getNextUniqueBunch($this->getIds())) {
12601260
$products = [];
12611261
$options = [];
12621262
$titles = [];

‎app/code/Magento/CatalogImportExport/Test/Unit/Model/Import/Product/LinkProcessorTest.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ public function testSaveLinks($expectedCallCount, $linkToNameId)
121121

122122
$dataSourceModel = $this->createMock(Data::class);
123123

124-
$this->linkProcessor->saveLinks($importEntity, $dataSourceModel, '_related_');
124+
$this->linkProcessor->saveLinks($importEntity, $dataSourceModel, '_related_', []);
125125
}
126126

127127
/**

‎app/code/Magento/CatalogImportExport/Test/Unit/Model/Import/Product/Type/OptionTest.php‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class OptionTest extends AbstractImportTestCase
4040
/**
4141
* Path to csv file to import
4242
*/
43-
const PATH_TO_CSV_FILE = '/_files/product_with_custom_options.csv';
43+
public const PATH_TO_CSV_FILE = '/_files/product_with_custom_options.csv';
4444

4545
/**
4646
* Parameters for Test stores.
@@ -409,12 +409,12 @@ protected function _getSourceDataMocks(bool $addExpectations, bool $doubleOption
409409
{
410410
$csvData = $this->_loadCsvFile();
411411

412-
$dataSourceModel = $this->getMockBuilder(\stdClass::class)->addMethods(['getNextBunch'])
412+
$dataSourceModel = $this->getMockBuilder(\stdClass::class)->addMethods(['getNextUniqueBunch'])
413413
->disableOriginalConstructor()
414414
->getMock();
415415
if ($addExpectations) {
416416
$dataSourceModel
417-
->method('getNextBunch')
417+
->method('getNextUniqueBunch')
418418
->willReturnOnConsecutiveCalls($csvData['data'], null);
419419
}
420420

@@ -1039,11 +1039,11 @@ public function validateAmbiguousDataDataProvider(): array
10391039
*/
10401040
public function testParseRequiredData(): void
10411041
{
1042-
$modelData = $this->getMockBuilder(\stdClass::class)->addMethods(['getNextBunch'])
1042+
$modelData = $this->getMockBuilder(\stdClass::class)->addMethods(['getNextUniqueBunch'])
10431043
->disableOriginalConstructor()
10441044
->getMock();
10451045
$modelData
1046-
->method('getNextBunch')
1046+
->method('getNextUniqueBunch')
10471047
->willReturnOnConsecutiveCalls(
10481048
[
10491049
[

‎app/code/Magento/CustomerImportExport/Model/Import/Address.php‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ class Address extends AbstractCustomer
160160
*
161161
* @var array
162162
* @deprecated 100.3.4 field not in use
163+
* @see Nothing
163164
*/
164165
protected $_regionParameters;
165166

@@ -190,18 +191,21 @@ class Address extends AbstractCustomer
190191
/**
191192
* @var \Magento\Eav\Model\Config
192193
* @deprecated 100.3.4 field not-in use
194+
* @see Nothing
193195
*/
194196
protected $_eavConfig;
195197

196198
/**
197199
* @var \Magento\Customer\Model\AddressFactory
198200
* @deprecated 100.3.4 not utilized anymore
201+
* @see Nothing
199202
*/
200203
protected $_addressFactory;
201204

202205
/**
203206
* @var \Magento\Framework\Stdlib\DateTime
204207
* @deprecated 100.3.4 the property isn't used
208+
* @see Nothing
205209
*/
206210
protected $dateTime;
207211

@@ -512,7 +516,7 @@ protected function _importData()
512516
{
513517
//Preparing data for mass validation/import.
514518
$rows = [];
515-
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
519+
while ($bunch = $this->_dataSourceModel->getNextUniqueBunch($this->getIds())) {
516520
$rows[] = $bunch;
517521
}
518522

@@ -521,7 +525,7 @@ protected function _importData()
521525
$this->_dataSourceModel->getIterator()->rewind();
522526

523527
//Importing
524-
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
528+
while ($bunch = $this->_dataSourceModel->getNextUniqueBunch($this->getIds())) {
525529
$newRows = [];
526530
$updateRows = [];
527531
$attributes = [];

‎app/code/Magento/CustomerImportExport/Model/Import/Customer.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -502,12 +502,13 @@ protected function _prepareDataForUpdate(array $rowData)
502502
* Import data rows
503503
*
504504
* @return bool
505+
* @throws \Exception
505506
* @SuppressWarnings(PHPMD.CyclomaticComplexity)
506507
* @SuppressWarnings(PHPMD.NPathComplexity)
507508
*/
508509
protected function _importData()
509510
{
510-
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
511+
while ($bunch = $this->_dataSourceModel->getNextUniqueBunch($this->getIds())) {
511512
$this->prepareCustomerData($bunch);
512513
$entitiesToCreate = [];
513514
$entitiesToUpdate = [];

‎app/code/Magento/CustomerImportExport/Model/ResourceModel/Import/CustomerComposite/Data.php‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,14 @@
1010
class Data extends \Magento\ImportExport\Model\ResourceModel\Import\Data
1111
{
1212
/**
13-
* Entity type
13+
* Entity
1414
*
1515
* @var string
1616
*/
1717
protected $_entityType = CustomerComposite::COMPONENT_ENTITY_CUSTOMER;
1818

1919
/**
20-
* Customer attributes
20+
* Customer attributes data
2121
*
2222
* @var array
2323
*/
@@ -27,7 +27,7 @@ class Data extends \Magento\ImportExport\Model\ResourceModel\Import\Data
2727
* Class constructor
2828
*
2929
* @param \Magento\Framework\Model\ResourceModel\Db\Context $context
30-
* @param \Magento\Framework\Json\Helper\Data $coreHelper
30+
* @param \Magento\Framework\Json\Helper\Data $jsonHelper
3131
* @param string $connectionName
3232
* @param array $arguments
3333
*/
@@ -50,11 +50,12 @@ public function __construct(
5050
/**
5151
* Get next bunch of validated rows.
5252
*
53+
* @param array|null $ids
5354
* @return array|null
5455
*/
55-
public function getNextBunch()
56+
public function getNextUniqueBunch($ids = null)
5657
{
57-
$bunchRows = parent::getNextBunch();
58+
$bunchRows = parent::getNextUniqueBunch($ids);
5859
if ($bunchRows != null) {
5960
$rows = [];
6061
foreach ($bunchRows as $rowNumber => $rowData) {

0 commit comments

Comments
 (0)