ACP2E-4153: [Cloud] Sitemap generation performance is significantly degraded

Author: thiaramus

app/code/Magento/Sitemap/Model/ResourceModel/Catalog/Product.php

@@ -1,13 +1,16 @@
 <?php
 /**
- * Copyright © Magento, Inc. All rights reserved.
- * See COPYING.txt for license details.
+ * Copyright 2011 Adobe
+ * All Rights Reserved.
  */
 namespace Magento\Sitemap\Model\ResourceModel\Catalog;
 
 use Magento\Catalog\Model\Product\Image\UrlBuilder;
 use Magento\CatalogUrlRewrite\Model\ProductUrlRewriteGenerator;
 use Magento\Framework\App\ObjectManager;
+use Magento\Framework\DataObject;
+use Magento\Framework\Exception\LocalizedException;
+use Magento\Sitemap\Model\SitemapConfigReaderInterface;
 use Magento\Store\Model\Store;
 
 /**
@@ -21,6 +24,11 @@ class Product extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb
 {
     public const NOT_SELECTED_IMAGE = 'no_selection';
 
+    /**
+     * Batch size for loading product images to avoid database IN() clause limits
+     */
+    private const IMAGE_BATCH_SIZE = 1000;
+
     /**
      * Collection Zend Db select
      *
@@ -35,6 +43,13 @@ class Product extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb
      */
     protected $_attributesCache = [];
 
+    /**
+     * Cached product images to avoid N+1 queries
+     *
+     * @var array
+     */
+    private $_productImagesCache = [];
+
     /**
      * @var \Magento\Catalog\Model\Product\Gallery\ReadHandler
      * @since 100.1.0
@@ -46,6 +61,11 @@ class Product extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb
      */
     protected $_sitemapData = null;
 
+    /**
+     * @var SitemapConfigReaderInterface
+     */
+    private $sitemapConfigReader;
+
     /**
      * @var \Magento\Catalog\Model\ResourceModel\Product
      */
@@ -107,6 +127,7 @@ class Product extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb
      * @param \Magento\Framework\App\Config\ScopeConfigInterface|null $scopeConfig
      * @param UrlBuilder $urlBuilder
      * @param ProductSelectBuilder $productSelectBuilder
+     * @param SitemapConfigReaderInterface $sitemapConfigReader
      * @SuppressWarnings(PHPMD.ExcessiveParameterList)
      * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      */
@@ -125,7 +146,8 @@ public function __construct(
         ?\Magento\Catalog\Helper\Image $catalogImageHelper = null,
         ?\Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig = null,
         ?UrlBuilder $urlBuilder = null,
-        ?ProductSelectBuilder $productSelectBuilder = null
+        ?ProductSelectBuilder $productSelectBuilder = null,
+        ?SitemapConfigReaderInterface $sitemapConfigReader = null
     ) {
         $this->_productResource = $productResource;
         $this->_storeManager = $storeManager;
@@ -138,6 +160,8 @@ public function __construct(
         $this->imageUrlBuilder = $urlBuilder ?? ObjectManager::getInstance()->get(UrlBuilder::class);
         $this->productSelectBuilder = $productSelectBuilder ??
             ObjectManager::getInstance()->get(ProductSelectBuilder::class);
+        $this->sitemapConfigReader = $sitemapConfigReader ??
+            ObjectManager::getInstance()->get(SitemapConfigReaderInterface::class);
 
         parent::__construct($context, $connectionName);
     }
@@ -307,7 +331,7 @@ public function getCollection($storeId)
         $this->_addFilter($store->getId(), 'status', $this->_productStatus->getVisibleStatusIds(), 'in');
 
         // Join product images required attributes
-        $imageIncludePolicy = $this->_sitemapData->getProductImageIncludePolicy($store->getId());
+        $imageIncludePolicy = $this->sitemapConfigReader->getProductImageIncludePolicy($store->getId());
         if (\Magento\Sitemap\Model\Source\Product\Image\IncludeImage::INCLUDE_NONE != $imageIncludePolicy) {
             $this->_joinAttribute($store->getId(), 'name', 'name');
             if (\Magento\Sitemap\Model\Source\Product\Image\IncludeImage::INCLUDE_ALL == $imageIncludePolicy) {
@@ -318,7 +342,23 @@ public function getCollection($storeId)
         }
 
         $query = $connection->query($this->prepareSelectStatement($this->_select));
+
+        // First, collect all product data without loading images
+        $productRows = [];
+        $productIds = [];
         while ($row = $query->fetch()) {
+            $productRows[] = $row;
+            $productIds[] = $row[$this->getIdFieldName()];
+        }
+
+        // Pre-load all images in batch to avoid N+1 queries
+        $imageIncludePolicy = $this->sitemapConfigReader->getProductImageIncludePolicy($store->getId());
+        if (\Magento\Sitemap\Model\Source\Product\Image\IncludeImage::INCLUDE_NONE != $imageIncludePolicy) {
+            $this->_preloadAllProductImages($productIds, $store->getId());
+        }
+
+        // Now create products with cached image data
+        foreach ($productRows as $row) {
             $product = $this->_prepareProduct($row, $store->getId());
             $products[$product->getId()] = $product;
         }
@@ -332,12 +372,12 @@ public function getCollection($storeId)
      * @param array $productRow
      * @param int $storeId
      *
-     * @return \Magento\Framework\DataObject
+     * @return DataObject
      * @throws \Magento\Framework\Exception\LocalizedException
      */
     protected function _prepareProduct(array $productRow, $storeId)
     {
-        $product = new \Magento\Framework\DataObject();
+        $product = new DataObject();
 
         $product['id'] = $productRow[$this->getIdFieldName()];
         if (empty($productRow['url'])) {
@@ -352,16 +392,14 @@ protected function _prepareProduct(array $productRow, $storeId)
     /**
      * Load product images
      *
-     * @param \Magento\Framework\DataObject $product
+     * @param DataObject $product
      * @param int $storeId
      * @return void
      */
     protected function _loadProductImages($product, $storeId)
     {
         $this->_storeManager->setCurrentStore($storeId);
-        /** @var $helper \Magento\Sitemap\Helper\Data */
-        $helper = $this->_sitemapData;
-        $imageIncludePolicy = $helper->getProductImageIncludePolicy($storeId);
+        $imageIncludePolicy = $this->sitemapConfigReader->getProductImageIncludePolicy($storeId);
 
         // Get product images
         $imagesCollection = [];
@@ -372,7 +410,7 @@ protected function _loadProductImages($product, $storeId)
             $product->getImage() != self::NOT_SELECTED_IMAGE
         ) {
             $imagesCollection = [
-                new \Magento\Framework\DataObject(
+                new DataObject(
                     ['url' => $this->getProductImageUrl($product->getImage())]
                 ),
             ];
@@ -388,7 +426,7 @@ protected function _loadProductImages($product, $storeId)
             }
 
             $product->setImages(
-                new \Magento\Framework\DataObject(
+                new DataObject(
                     ['collection' => $imagesCollection, 'title' => $product->getName(), 'thumbnail' => $thumbnail]
                 )
             );
@@ -404,22 +442,38 @@ protected function _loadProductImages($product, $storeId)
      */
     protected function _getAllProductImages($product, $storeId)
     {
-        $product->setStoreId($storeId);
-        $gallery = $this->mediaGalleryResourceModel->loadProductGalleryByAttributeId(
-            $product,
-            $this->mediaGalleryReadHandler->getAttribute()->getId()
-        );
-
+        $productId = $product->getId();
         $imagesCollection = [];
-        if ($gallery) {
+
+        // Use cached images if available (from batch loading)
+        if (isset($this->_productImagesCache[$productId])) {
+            $gallery = $this->_productImagesCache[$productId];
             foreach ($gallery as $image) {
-                $imagesCollection[] = new \Magento\Framework\DataObject(
+                $imagesCollection[] = new DataObject(
                     [
                         'url' => $this->getProductImageUrl($image['file']),
                         'caption' => $image['label'] ? $image['label'] : $image['label_default'],
                     ]
                 );
             }
+        } else {
+            // Fallback to individual query (should rarely happen now)
+            $product->setStoreId($storeId);
+            $gallery = $this->mediaGalleryResourceModel->loadProductGalleryByAttributeId(
+                $product,
+                $this->mediaGalleryReadHandler->getAttribute()->getId()
+            );
+
+            if ($gallery) {
+                foreach ($gallery as $image) {
+                    $imagesCollection[] = new DataObject(
+                        [
+                            'url' => $this->getProductImageUrl($image['file']),
+                            'caption' => $image['label'] ? $image['label'] : $image['label_default'],
+                        ]
+                    );
+                }
+            }
         }
 
         return $imagesCollection;
@@ -449,6 +503,58 @@ public function prepareSelectStatement(\Magento\Framework\DB\Select $select)
         return $select;
     }
 
+    /**
+     * Pre-load all product images in batched queries to avoid N+1 problem while respecting DB limits
+     *
+     * @param array $productIds
+     * @param int $storeId
+     * @return void
+     * @throws LocalizedException
+     */
+    private function _preloadAllProductImages($productIds, $storeId)
+    {
+        if (empty($productIds)) {
+            return;
+        }
+
+        // Split into smaller batches to avoid hitting database IN() clause limits
+        $productBatches = array_chunk($productIds, self::IMAGE_BATCH_SIZE);
+
+        $linkField = $this->_productResource->getLinkField();
+        $connection = $this->getConnection();
+
+        foreach ($productBatches as $batch) {
+            // Use the existing createBatchBaseSelect method for each batch
+            $select = $this->mediaGalleryResourceModel->createBatchBaseSelect(
+                $storeId,
+                $this->mediaGalleryReadHandler->getAttribute()->getId()
+            );
+
+            $select->where('entity.' . $linkField . ' IN (?)', $batch);
+
+            // Add ordering to ensure consistent results
+            $select->order(['entity.' . $linkField, 'position']);
+
+            $result = $connection->fetchAll($select);
+
+            // Group images by product ID
+            foreach ($result as $row) {
+                $productId = $row[$linkField];
+                if (!isset($this->_productImagesCache[$productId])) {
+                    $this->_productImagesCache[$productId] = [];
+                }
+                $this->_productImagesCache[$productId][] = $row;
+            }
+        }
+
+        // Ensure all requested products have an entry (even if empty)
+        foreach ($productIds as $productId) {
+            if (!isset($this->_productImagesCache[$productId])) {
+                $this->_productImagesCache[$productId] = [];
+            }
+        }
+    }
+
     /**
      * Get product image URL from image filename
      *