Merge pull request #3305 from magento-performance/pr-2.3-develop

### Task * [MAGETWO-95275](https://jira.corp.magento.com/browse/MAGETWO-95275) Varnish "Connection reset by peer" error when large catalog is reindexed on schedule
magento · Oct 23, 2018 · c515b40 · c515b40
2 parents 7fc1ef6 + 287a5dc
commit c515b40
Show file tree

Hide file tree

Showing 7 changed files with 87 additions and 26 deletions.
diff --git a/app/code/Magento/CacheInvalidate/Model/PurgeCache.php b/app/code/Magento/CacheInvalidate/Model/PurgeCache.php
@@ -7,6 +7,9 @@
 
 use Magento\Framework\Cache\InvalidateLogger;
 
+/**
+ * Class PurgeCache
+ */
 class PurgeCache
 {
     const HEADER_X_MAGENTO_TAGS_PATTERN = 'X-Magento-Tags-Pattern';
@@ -26,6 +29,18 @@ class PurgeCache
      */
     private $logger;
 
+    /**
+     * Batch size of the purge request.
+     *
+     * Based on default Varnish 4 http_req_hdr_len size minus a 512 bytes margin for method,
+     * header name, line feeds etc.
+     *
+     * @see https://varnish-cache.org/docs/4.1/reference/varnishd.html
+     *
+     * @var int
+     */
+    private $requestSize = 7680;
+
     /**
      * Constructor
      *
@@ -44,18 +59,65 @@ public function __construct(
     }
 
     /**
-     * Send curl purge request
-     * to invalidate cache by tags pattern
+     * Send curl purge request to invalidate cache by tags pattern
      *
      * @param string $tagsPattern
      * @return bool Return true if successful; otherwise return false
      */
     public function sendPurgeRequest($tagsPattern)
     {
+        $successful = true;
         $socketAdapter = $this->socketAdapterFactory->create();
         $servers = $this->cacheServer->getUris();
-        $headers = [self::HEADER_X_MAGENTO_TAGS_PATTERN => $tagsPattern];
         $socketAdapter->setOptions(['timeout' => 10]);
+
+        $formattedTagsChunks = $this->splitTags($tagsPattern);
+        foreach ($formattedTagsChunks as $formattedTagsChunk) {
+            if (!$this->sendPurgeRequestToServers($socketAdapter, $servers, $formattedTagsChunk)) {
+                $successful = false;
+            }
+        }
+
+        return $successful;
+    }
+
+    /**
+     * Split tags by batches
+     *
+     * @param string $tagsPattern
+     * @return \Generator
+     */
+    private function splitTags($tagsPattern)
+    {
+        $tagsBatchSize = 0;
+        $formattedTagsChunk = [];
+        $formattedTags = explode('|', $tagsPattern);
+        foreach ($formattedTags as $formattedTag) {
+            if ($tagsBatchSize + strlen($formattedTag) > $this->requestSize - count($formattedTagsChunk) - 1) {
+                yield implode('|', array_unique($formattedTagsChunk));
+                $formattedTagsChunk = [];
+                $tagsBatchSize = 0;
+            }
+
+            $tagsBatchSize += strlen($formattedTag);
+            $formattedTagsChunk[] = $formattedTag;
+        }
+        if (!empty($formattedTagsChunk)) {
+            yield implode('|', array_unique($formattedTagsChunk));
+        }
+    }
+
+    /**
+     * Send curl purge request to servers to invalidate cache by tags pattern
+     *
+     * @param \Zend\Http\Client\Adapter\Socket $socketAdapter
+     * @param \Zend\Uri\Uri[] $servers
+     * @param string $formattedTagsChunk
+     * @return bool Return true if successful; otherwise return false
+     */
+    private function sendPurgeRequestToServers($socketAdapter, $servers, $formattedTagsChunk)
+    {
+        $headers = [self::HEADER_X_MAGENTO_TAGS_PATTERN => $formattedTagsChunk];
         foreach ($servers as $server) {
             $headers['Host'] = $server->getHost();
             try {
@@ -69,12 +131,11 @@ public function sendPurgeRequest($tagsPattern)
                 $socketAdapter->read();
                 $socketAdapter->close();
             } catch (\Exception $e) {
-                $this->logger->critical($e->getMessage(), compact('server', 'tagsPattern'));
+                $this->logger->critical($e->getMessage(), compact('server', 'formattedTagsChunk'));
                 return false;
             }
         }
-
-        $this->logger->execute(compact('servers', 'tagsPattern'));
+        $this->logger->execute(compact('servers', 'formattedTagsChunk'));
         return true;
     }
 }
diff --git a/setup/performance-toolkit/config/description.xml b/setup/performance-toolkit/config/description.xml
@@ -7,16 +7,16 @@
 -->
 <description>
     <paragraphs>
-        <count-min>4</count-min>
-        <count-max>10</count-max>
+        <count-min>7</count-min>
+        <count-max>7</count-max>
 
         <sentences>
-            <count-min>10</count-min>
-            <count-max>15</count-max>
+            <count-min>12</count-min>
+            <count-max>12</count-max>
 
             <words>
-                <count-min>5</count-min>
-                <count-max>7</count-max>
+                <count-min>6</count-min>
+                <count-max>6</count-max>
             </words>
         </sentences>
     </paragraphs>

diff --git a/setup/performance-toolkit/profiles/ce/extra_large.xml b/setup/performance-toolkit/profiles/ce/extra_large.xml
@@ -40,9 +40,9 @@
         <cart_price_rules>20</cart_price_rules> <!-- Number of cart price rules -->
         <cart_price_rules_floor>2</cart_price_rules_floor>
 
-        <product_attribute_sets>200</product_attribute_sets> <!-- Number of product attribute sets -->
-        <product_attribute_sets_attributes>50</product_attribute_sets_attributes> <!-- Number of attributes per set -->
-        <product_attribute_sets_attributes_values>2</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
+        <product_attribute_sets>100</product_attribute_sets> <!-- Number of product attribute sets -->
+        <product_attribute_sets_attributes>30</product_attribute_sets_attributes> <!-- Number of attributes per set -->
+        <product_attribute_sets_attributes_values>15</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
 
         <order_quotes_enable>true</order_quotes_enable>
         <order_simple_product_count_from>2</order_simple_product_count_from>

diff --git a/setup/performance-toolkit/profiles/ce/large.xml b/setup/performance-toolkit/profiles/ce/large.xml
@@ -40,9 +40,9 @@
         <cart_price_rules>20</cart_price_rules> <!-- Number of cart price rules -->
         <cart_price_rules_floor>2</cart_price_rules_floor>
 
-        <product_attribute_sets>200</product_attribute_sets> <!-- Number of product attribute sets -->
-        <product_attribute_sets_attributes>50</product_attribute_sets_attributes> <!-- Number of attributes per set -->
-        <product_attribute_sets_attributes_values>2</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
+        <product_attribute_sets>50</product_attribute_sets> <!-- Number of product attribute sets -->
+        <product_attribute_sets_attributes>20</product_attribute_sets_attributes> <!-- Number of attributes per set -->
+        <product_attribute_sets_attributes_values>15</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
 
         <order_quotes_enable>true</order_quotes_enable>
         <order_simple_product_count_from>2</order_simple_product_count_from>

diff --git a/setup/performance-toolkit/profiles/ce/medium.xml b/setup/performance-toolkit/profiles/ce/medium.xml
@@ -40,9 +40,9 @@
         <cart_price_rules>20</cart_price_rules> <!-- Number of cart price rules -->
         <cart_price_rules_floor>2</cart_price_rules_floor>
 
-        <product_attribute_sets>100</product_attribute_sets> <!-- Number of product attribute sets -->
-        <product_attribute_sets_attributes>50</product_attribute_sets_attributes> <!-- Number of attributes per set -->
-        <product_attribute_sets_attributes_values>2</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
+        <product_attribute_sets>30</product_attribute_sets> <!-- Number of product attribute sets -->
+        <product_attribute_sets_attributes>10</product_attribute_sets_attributes> <!-- Number of attributes per set -->
+        <product_attribute_sets_attributes_values>8</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
 
         <order_quotes_enable>true</order_quotes_enable>
         <order_simple_product_count_from>2</order_simple_product_count_from>

diff --git a/setup/performance-toolkit/profiles/ce/medium_msite.xml b/setup/performance-toolkit/profiles/ce/medium_msite.xml
@@ -46,9 +46,9 @@
         <cart_price_rules>20</cart_price_rules> <!-- Number of cart price rules -->
         <cart_price_rules_floor>2</cart_price_rules_floor>
 
-        <product_attribute_sets>100</product_attribute_sets> <!-- Number of product attribute sets -->
-        <product_attribute_sets_attributes>50</product_attribute_sets_attributes> <!-- Number of attributes per set -->
-        <product_attribute_sets_attributes_values>2</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
+        <product_attribute_sets>30</product_attribute_sets> <!-- Number of product attribute sets -->
+        <product_attribute_sets_attributes>10</product_attribute_sets_attributes> <!-- Number of attributes per set -->
+        <product_attribute_sets_attributes_values>8</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
 
         <order_quotes_enable>true</order_quotes_enable>
         <order_simple_product_count_from>2</order_simple_product_count_from>

diff --git a/setup/performance-toolkit/profiles/ce/small.xml b/setup/performance-toolkit/profiles/ce/small.xml
@@ -41,8 +41,8 @@
         <cart_price_rules_floor>2</cart_price_rules_floor>
 
         <product_attribute_sets>10</product_attribute_sets> <!-- Number of product attribute sets -->
-        <product_attribute_sets_attributes>10</product_attribute_sets_attributes> <!-- Number of attributes per set -->
-        <product_attribute_sets_attributes_values>2</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
+        <product_attribute_sets_attributes>5</product_attribute_sets_attributes> <!-- Number of attributes per set -->
+        <product_attribute_sets_attributes_values>5</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
 
         <order_quotes_enable>true</order_quotes_enable>
         <order_simple_product_count_from>2</order_simple_product_count_from>