From a383793c85e4f68db7dd6c84112671ed6c71c830 Mon Sep 17 00:00:00 2001
From: Fanis Strezos <Fanis.Strezos@dotmailer.com>
Date: Wed, 22 Apr 2020 10:23:39 +0000
Subject: [PATCH] Merged PR 20956: Batch deletion of orphaned product rows in
 catalog sync

## What's being changed

There's a step at the start of catalog sync where we scan the `email_catalog` table and check whether the `product_id` matches an `entity_id` in `catalog_product_entity`, deleting from `email_catalog` if we can't find a match. This step is being optimised using batching.

## Why it's being changed

For merchants with large data sets, the delete query can cause SQL locking.

## How to review / test this change

- Delete a number of rows manually from `catalog_product_entity`, from different points e.g. delete entity ids 200 and 700
- Run catalog sync
- Ensure the deleted product ids are also removed from email_catalog

Related work items: #113976
---
 code/Dotdigitalgroup/Email/Model/Catalog.php | 72 +++++++++++++-------
 1 file changed, 47 insertions(+), 25 deletions(-)

diff --git a/code/Dotdigitalgroup/Email/Model/Catalog.php b/code/Dotdigitalgroup/Email/Model/Catalog.php
index 73bc8c9f6..2771d92ea 100755
--- a/code/Dotdigitalgroup/Email/Model/Catalog.php
+++ b/code/Dotdigitalgroup/Email/Model/Catalog.php
@@ -69,31 +69,8 @@ public function sync()
         );
 
         if ($enabled && $sync) {
-            //remove product with product id set and no product
-            $coreResource = Mage::getSingleton('core/resource');
-            $write        = $coreResource->getConnection('core_write');
-            $catalogTable = $coreResource->getTableName(
-                'ddg_automation/catalog'
-            );
-            //@codingStandardsIgnoreStart
-            $select       = $write->select();
-            $select->reset()
-                ->from(
-                    array('c' => $catalogTable),
-                    array('c.product_id')
-                )
-                ->joinLeft(
-                    array('e' => $coreResource->getTableName(
-                        'catalog/product'
-                    )),
-                    "c.product_id = e.entity_id"
-                )
-                ->where('e.entity_id is NULL');
-            //delete sql statement
-            $deleteSql = $select->deleteFromSelect('c');
-            //run query
-            $write->query($deleteSql);
-            //@codingStandardsIgnoreEnd
+
+            $this->removeOrphanedProducts();
 
             $scope = Mage::getStoreConfig(
                 Dotdigitalgroup_Email_Helper_Config::XML_PATH_CONNECTOR_SYNC_CATALOG_VALUES
@@ -469,4 +446,49 @@ public function handleConfigSaveAfter(Varien_Event_Observer $observer)
 
         return $this;
     }
+
+    /**
+     * Remove orphaned records from email_catalog table.
+     * These are rows in email_catalog for which there is no longer a matching product in catalog_product_entity.
+     */
+    protected function removeOrphanedProducts()
+    {
+        $coreResource = Mage::getSingleton('core/resource');
+        $write        = $coreResource->getConnection('core_write');
+        $catalogTable = $coreResource->getTableName(
+            'ddg_automation/catalog'
+        );
+
+        $batchSize = 500;
+        $startPoint = 0;
+        $endPoint = $startPoint + $batchSize;
+
+        do {
+            $select = $write->select();
+            $batching = $select->reset()
+                ->from(
+                    array('c' => $catalogTable),
+                    array('c.id', 'c.product_id')
+                )
+                ->joinLeft(
+                    array('e' => $coreResource->getTableName(
+                        'catalog/product'
+                    )),
+                    "c.product_id = e.entity_id"
+                )
+                ->where('c.id >= ?', $startPoint)
+                ->where('c.id < ?', $endPoint);
+
+            $rowCount = $write->query($batching)->rowCount();
+
+            $select = $batching->where('e.entity_id is NULL');
+
+            $deleteSql = $select->deleteFromSelect('c');
+            $write->query($deleteSql);
+            
+            $startPoint += $batchSize;
+            $endPoint += $batchSize;
+
+        } while ($rowCount > 0);
+    }
 }