ACP2E-1028: Current Indexer implementation for large catalogs is causing items to not appear on site.

dhorytskyi · dhorytskyi · commit 9cb64adfa13b · 2022-07-20T16:31:35.000-05:00
diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php
@@ -170,7 +170,8 @@ abstract public function execute($ids);
      * @param array $processIds
      * @return AbstractAction
      * @SuppressWarnings(PHPMD.UnusedFormalParameter)
-     * @deprecated 102.0.6 Used only for backward compatibility for indexer, which not support indexation by dimensions
+     * @deprecated 102.0.6
+     * @see Used only for backward compatibility for indexer, which not support indexation by dimensions
      */
     protected function _syncData(array $processIds = [])
     {
@@ -386,10 +387,6 @@ protected function _reindexRows($changedIds = [])
         $changedIds = array_unique(array_merge($changedIds, ...array_values($parentProductsTypes)));
         $productsTypes = array_merge_recursive($productsTypes, $parentProductsTypes);
 
-        if ($changedIds) {
-            $this->deleteIndexData($changedIds);
-        }
-
         $typeIndexers = $this->getTypeIndexers();
         foreach ($typeIndexers as $productType => $indexer) {
             $entityIds = $productsTypes[$productType] ?? [];
@@ -403,14 +400,13 @@ protected function _reindexRows($changedIds = [])
                     $temporaryTable = $this->tableMaintainer->getMainTmpTable($dimensions);
                     $this->_emptyTable($temporaryTable);
                     $indexer->executeByDimensions($dimensions, \SplFixedArray::fromArray($entityIds, false));
-                    // copy to index
-                    $this->_insertFromTable(
-                        $temporaryTable,
-                        $this->tableMaintainer->getMainTableByDimensions($dimensions)
-                    );
+                    $mainTable = $this->tableMaintainer->getMainTableByDimensions($dimensions);
+                    $this->_insertFromTable($temporaryTable, $mainTable);
+                    $this->deleteOutdatedData($entityIds, $temporaryTable, $mainTable);
                 }
             } else {
                 // handle 3d-party indexers for backward compatibility
+                $this->deleteIndexData($changedIds);
                 $this->_emptyTable($this->_defaultIndexerResource->getIdxTable());
                 $this->_copyRelationIndexData($entityIds);
                 $indexer->reindexEntity($entityIds);
@@ -421,6 +417,30 @@ protected function _reindexRows($changedIds = [])
         return $changedIds;
     }
 
+    /**
+     * Delete records that do not exist anymore
+     *
+     * @param array $entityIds
+     * @param string $temporaryTable
+     * @param string $mainTable
+     * @return void
+     */
+    private function deleteOutdatedData(array $entityIds, string $temporaryTable, string $mainTable): void
+    {
+        $joinCondition = [
+            'tmp_table.entity_id = main_table.entity_id',
+            'tmp_table.customer_group_id = main_table.customer_group_id',
+            'tmp_table.website_id = main_table.website_id',
+        ];
+        $select = $this->getConnection()->select()
+            ->from(['main_table' => $mainTable], null)
+            ->joinLeft(['tmp_table' => $temporaryTable], implode(' AND ', $joinCondition), null)
+            ->where('tmp_table.entity_id IS NULL')
+            ->where('main_table.entity_id IN (?)', $entityIds, \Zend_Db::INT_TYPE);
+        $query = $select->deleteFromSelect('main_table');
+        $this->getConnection()->query($query);
+    }
+
     /**
      * Delete Index data index for list of entities
      *
@@ -445,7 +465,8 @@ private function deleteIndexData(array $entityIds)
      * @param null|array $parentIds
      * @param array $excludeIds
      * @return \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction
-     * @deprecated 102.0.6 Used only for backward compatibility for do not broke custom indexer implementation
+     * @deprecated 102.0.6
+     * @see Used only for backward compatibility for do not broke custom indexer implementation
      * which do not work by dimensions.
      * For indexers, which support dimensions all composite products read data directly from main price indexer table
      * or replica table for partial or full reindex correspondingly.
diff --git a/app/code/Magento/Catalog/Test/Unit/Model/Indexer/Product/Price/Action/RowsTest.php b/app/code/Magento/Catalog/Test/Unit/Model/Indexer/Product/Price/Action/RowsTest.php
@@ -7,6 +7,8 @@
 
 namespace Magento\Catalog\Test\Unit\Model\Indexer\Product\Price\Action;
 
+use Magento\Framework\Indexer\DimensionalIndexerInterface;
+use Magento\Framework\Search\Request\Dimension;
 use Magento\Store\Model\StoreManagerInterface;
 use Magento\Directory\Model\CurrencyFactory;
 use Magento\Catalog\Model\Product\Type;
@@ -94,36 +96,17 @@ class RowsTest extends TestCase
 
     protected function setUp(): void
     {
-        $this->config = $this->getMockBuilder(ScopeConfigInterface::class)
-            ->getMockForAbstractClass();
-        $this->storeManager = $this->getMockBuilder(StoreManagerInterface::class)
-            ->getMockForAbstractClass();
-        $this->currencyFactory = $this->getMockBuilder(CurrencyFactory::class)
-            ->disableOriginalConstructor()
-            ->getMock();
-        $this->localeDate = $this->getMockBuilder(TimezoneInterface::class)
-            ->getMockForAbstractClass();
-        $this->dateTime = $this->getMockBuilder(DateTime::class)
-            ->disableOriginalConstructor()
-            ->getMock();
-        $this->catalogProductType = $this->getMockBuilder(Type::class)
-            ->disableOriginalConstructor()
-            ->getMock();
-        $this->indexerPriceFactory = $this->getMockBuilder(Factory::class)
-            ->disableOriginalConstructor()
-            ->getMock();
-        $this->defaultIndexerResource = $this->getMockBuilder(DefaultPrice::class)
-            ->disableOriginalConstructor()
-            ->getMock();
-        $this->tierPriceIndexResource = $this->getMockBuilder(TierPrice::class)
-            ->disableOriginalConstructor()
-            ->getMock();
-        $this->dimensionCollectionFactory = $this->getMockBuilder(DimensionCollectionFactory::class)
-            ->disableOriginalConstructor()
-            ->getMock();
-        $this->tableMaintainer = $this->getMockBuilder(TableMaintainer::class)
-            ->disableOriginalConstructor()
-            ->getMock();
+        $this->config = $this->createMock(ScopeConfigInterface::class);
+        $this->storeManager = $this->createMock(StoreManagerInterface::class);
+        $this->currencyFactory = $this->createMock(CurrencyFactory::class);
+        $this->localeDate = $this->createMock(TimezoneInterface::class);
+        $this->dateTime = $this->createMock(DateTime::class);
+        $this->catalogProductType = $this->createMock(Type::class);
+        $this->indexerPriceFactory = $this->createMock(Factory::class);
+        $this->defaultIndexerResource = $this->createMock(DefaultPrice::class);
+        $this->tierPriceIndexResource = $this->createMock(TierPrice::class);
+        $this->dimensionCollectionFactory = $this->createMock(DimensionCollectionFactory::class);
+        $this->tableMaintainer = $this->createMock(TableMaintainer::class);
         $batchSize = 2;
 
         $this->actionRows = new Rows(
@@ -144,52 +127,72 @@ protected function setUp(): void
 
     public function testEmptyIds()
     {
-        $this->expectException('Magento\Framework\Exception\InputException');
+        $this->expectException(\Magento\Framework\Exception\InputException::class);
         $this->expectExceptionMessage('Bad value was supplied.');
         $this->actionRows->execute(null);
     }
 
     public function testBatchProcessing()
     {
         $ids = [1, 2, 3, 4];
-        $select = $this->getMockBuilder(Select::class)
-            ->disableOriginalConstructor()
-            ->getMock();
-        $select->expects($this->any())->method('from')->willReturnSelf();
-        $select->expects($this->any())->method('where')->willReturnSelf();
-        $select->expects($this->any())->method('join')->willReturnSelf();
-        $adapter = $this->getMockBuilder(AdapterInterface::class)->getMockForAbstractClass();
-        $adapter->expects($this->any())->method('select')->willReturn($select);
-        $this->defaultIndexerResource->expects($this->any())
-            ->method('getConnection')
-            ->willReturn($adapter);
-        $adapter->expects($this->any())
-            ->method('fetchAll')
-            ->with($select)
-            ->willReturn([]);
-        $adapter->expects($this->any())
+
+        $select = $this->createMock(Select::class);
+        $select->method('from')->willReturnSelf();
+        $select->method('joinLeft')->willReturnSelf();
+        $select->method('where')->willReturnSelf();
+        $select->method('join')->willReturnSelf();
+        $adapter = $this->createMock(AdapterInterface::class);
+        $adapter->method('select')->willReturn($select);
+        $adapter->method('describeTable')->willReturn([]);
+        $this->defaultIndexerResource->method('getConnection')->willReturn($adapter);
+        $adapter->method('fetchAll')->with($select)->willReturn([]);
+
+        $adapter->expects($this->exactly(4))
             ->method('fetchPairs')
             ->with($select)
-            ->willReturn([]);
-        $multiDimensionProvider = $this->getMockBuilder(MultiDimensionProvider::class)
-            ->disableOriginalConstructor()
-            ->getMock();
-        $this->dimensionCollectionFactory->expects($this->exactly(2))
+            ->willReturnOnConsecutiveCalls(
+                [1 => 'simple', 2 => 'virtual'],
+                [],
+                [3 => 'simple', 4 => 'virtual'],
+                [],
+            );
+        $multiDimensionProvider = $this->createMock(MultiDimensionProvider::class);
+        $this->dimensionCollectionFactory->expects($this->exactly(4))
             ->method('create')
             ->willReturn($multiDimensionProvider);
-        $iterator = new \ArrayIterator([]);
-        $multiDimensionProvider->expects($this->exactly(2))
+        $dimension = $this->createMock(Dimension::class);
+        $dimension->method('getName')->willReturn('default');
+        $dimension->method('getValue')->willReturn('0');
+        $iterator = new \ArrayIterator([[$dimension]]);
+        $multiDimensionProvider->expects($this->exactly(4))
             ->method('getIterator')
             ->willReturn($iterator);
-        $this->catalogProductType->expects($this->any())
+        $this->catalogProductType->expects($this->once())
             ->method('getTypesByPriority')
-            ->willReturn([]);
+            ->willReturn(
+                [
+                    'virtual' => ['price_indexer' => '\Price\Indexer'],
+                    'simple' => ['price_indexer' => '\Price\Indexer'],
+                ]
+            );
+        $priceIndexer = $this->createMock(DimensionalIndexerInterface::class);
+        $this->indexerPriceFactory->expects($this->exactly(2))
+            ->method('create')
+            ->with('\Price\Indexer', ['fullReindexAction' => false])
+            ->willReturn($priceIndexer);
+        $priceIndexer->expects($this->exactly(4))
+            ->method('executeByDimensions');
+        $select->expects($this->exactly(4))
+            ->method('deleteFromSelect')
+            ->with('main_table')
+            ->willReturn('');
         $adapter->expects($this->exactly(2))
             ->method('getIndexList')
             ->willReturn(['entity_id'=>['COLUMNS_LIST'=>['test']]]);
         $adapter->expects($this->exactly(2))
             ->method('getPrimaryKeyName')
             ->willReturn('entity_id');
+
         $this->actionRows->execute($ids);
     }
 }
