ACP2E-1675: [Cloud] Deployment issue due to exceeded memory and large tables

viktym · viktym · commit fb1537dd8f5a · 2023-08-24T10:50:13.000-05:00
diff --git a/app/code/Magento/Catalog/Setup/Patch/Data/UpdateMultiselectAttributesBackendTypes.php b/app/code/Magento/Catalog/Setup/Patch/Data/UpdateMultiselectAttributesBackendTypes.php
@@ -10,54 +10,37 @@
 use Magento\Eav\Setup\EavSetup;
 use Magento\Eav\Setup\EavSetupFactory;
 use Magento\Framework\DB\Adapter\AdapterInterface;
-use Magento\Framework\DB\Query\Generator;
-use Magento\Framework\DB\Select;
-use Magento\Framework\DB\Sql\Expression;
 use Magento\Framework\Setup\ModuleDataSetupInterface;
 use Magento\Framework\Setup\Patch\DataPatchInterface;
 use Magento\Framework\Setup\Patch\NonTransactionableInterface;
 
 class UpdateMultiselectAttributesBackendTypes implements DataPatchInterface, NonTransactionableInterface
 {
-    private const BATCH_SIZE = 10000;
-
     /**
      * @var ModuleDataSetupInterface
      */
     private $dataSetup;
-
     /**
      * @var EavSetupFactory
      */
     private $eavSetupFactory;
 
     /**
-     * @var Generator
-     */
-    private Generator $batchQueryGenerator;
-
-    /**
-     * @var int
+     * @var array
      */
-    private int $batchSize;
+    private $triggersRestoreQueries = [];
 
     /**
      * MigrateMultiselectAttributesData constructor.
      * @param ModuleDataSetupInterface $dataSetup
      * @param EavSetupFactory $eavSetupFactory
-     * @param Generator $batchQueryGenerator
-     * @param int $batchSize
      */
     public function __construct(
         ModuleDataSetupInterface $dataSetup,
-        EavSetupFactory $eavSetupFactory,
-        Generator $batchQueryGenerator,
-        int $batchSize = self::BATCH_SIZE
+        EavSetupFactory $eavSetupFactory
     ) {
         $this->dataSetup = $dataSetup;
         $this->eavSetupFactory = $eavSetupFactory;
-        $this->batchQueryGenerator = $batchQueryGenerator;
-        $this->batchSize = $batchSize;
     }
 
     /**
@@ -84,6 +67,7 @@ public function apply()
         $this->dataSetup->startSetup();
         $setup = $this->dataSetup;
         $connection = $setup->getConnection();
+        $this->triggersRestoreQueries = [];
 
         $attributeTable = $setup->getTable('eav_attribute');
         /** @var EavSetup $eavSetup */
@@ -97,43 +81,30 @@ public function apply()
                 ->where('backend_type = ?', 'varchar')
                 ->where('frontend_input = ?', 'multiselect')
         );
+        $attributesToMigrate = array_map('intval', $attributesToMigrate);
 
         $varcharTable = $setup->getTable('catalog_product_entity_varchar');
         $textTable = $setup->getTable('catalog_product_entity_text');
 
         $columns = $connection->describeTable($varcharTable);
-        $primaryKey = 'value_id';
-        unset($columns[$primaryKey]);
-        $columnNames = array_keys($columns);
-        $batchIterator = $this->batchQueryGenerator->generate(
-            $primaryKey,
-            $connection->select()
-                ->from($varcharTable)
-                ->where('attribute_id in (?)', $attributesToMigrate),
-            $this->batchSize
-        );
-        foreach ($batchIterator as $select) {
-            $selectForInsert = clone $select;
-            $selectForInsert->reset(Select::COLUMNS);
-            $selectForInsert->columns($columnNames);
+        unset($columns['value_id']);
+        $this->dropTriggers($textTable);
+        $this->dropTriggers($varcharTable);
+        try {
             $connection->query(
                 $connection->insertFromSelect(
-                    $selectForInsert,
+                    $connection->select()
+                        ->from($varcharTable, array_keys($columns))
+                        ->where('attribute_id in (?)', $attributesToMigrate),
                     $textTable,
-                    $columnNames,
+                    array_keys($columns),
                     AdapterInterface::INSERT_ON_DUPLICATE
                 )
             );
-            $selectForDelete = clone $select;
-            $selectForDelete->reset(Select::COLUMNS);
-            $selectForDelete->columns($primaryKey);
-            $selectForDelete = $connection->select()
-                ->from($selectForDelete, $primaryKey);
-
-            $connection->delete(
-                $varcharTable,
-                new Expression($primaryKey . ' IN ('  . $selectForDelete->assemble() . ')')
-            );
+            $connection->delete($varcharTable, ['attribute_id IN (?)' => $attributesToMigrate]);
+        } finally {
+            $this->restoreTriggers($textTable);
+            $this->restoreTriggers($varcharTable);
         }
 
         foreach ($attributesToMigrate as $attributeId) {
@@ -144,4 +115,48 @@ public function apply()
 
         return $this;
     }
+
+    /**
+     * Drop table triggers
+     *
+     * @param string $tableName
+     * @return void
+     * @throws \Zend_Db_Statement_Exception
+     */
+    private function dropTriggers(string $tableName): void
+    {
+        $triggers = $this->dataSetup->getConnection()
+            ->query('SHOW TRIGGERS LIKE \''. $tableName . '\'')
+            ->fetchAll();
+
+        if (!$triggers) {
+            return;
+        }
+
+        foreach ($triggers as $trigger) {
+            $triggerData = $this->dataSetup->getConnection()
+                ->query('SHOW CREATE TRIGGER '. $trigger['Trigger'])
+                ->fetch();
+            $this->triggersRestoreQueries[$tableName][] =
+                preg_replace('/DEFINER=[^\s]*/', '', $triggerData['SQL Original Statement']);
+            // phpcs:ignore Magento2.SQL.RawQuery.FoundRawSql
+            $this->dataSetup->getConnection()->query('DROP TRIGGER IF EXISTS ' . $trigger['Trigger']);
+        }
+    }
+
+    /**
+     * Restore table triggers.
+     *
+     * @param string $tableName
+     * @return void
+     * @throws \Zend_Db_Statement_Exception
+     */
+    private function restoreTriggers(string $tableName): void
+    {
+        if (array_key_exists($tableName, $this->triggersRestoreQueries)) {
+            foreach ($this->triggersRestoreQueries[$tableName] as $query) {
+                $this->dataSetup->getConnection()->multiQuery($query);
+            }
+        }
+    }
 }
diff --git a/app/code/Magento/Catalog/Test/Unit/Setup/Patch/Data/UpdateMultiselectAttributesBackendTypesTest.php b/app/code/Magento/Catalog/Test/Unit/Setup/Patch/Data/UpdateMultiselectAttributesBackendTypesTest.php
@@ -11,30 +11,22 @@
 use Magento\Eav\Setup\EavSetup;
 use Magento\Eav\Setup\EavSetupFactory;
 use Magento\Framework\DB\Adapter\AdapterInterface;
-use Magento\Framework\DB\Query\BatchIteratorInterface;
-use Magento\Framework\DB\Query\Generator;
 use Magento\Framework\DB\Select;
 use Magento\Framework\Setup\ModuleDataSetupInterface;
-use PHPUnit\Framework\MockObject\MockObject;
 use PHPUnit\Framework\TestCase;
 
 class UpdateMultiselectAttributesBackendTypesTest extends TestCase
 {
     /**
-     * @var ModuleDataSetupInterface|MockObject
+     * @var ModuleDataSetupInterface|\PHPUnit\Framework\MockObject\MockObject
      */
     private $dataSetup;
 
     /**
-     * @var EavSetupFactory|MockObject
+     * @var EavSetupFactory|\PHPUnit\Framework\MockObject\MockObject
      */
     private $eavSetupFactory;
 
-    /**
-     * @var Generator|MockObject
-     */
-    private $batchQueryGenerator;
-
     /**
      * @var UpdateMultiselectAttributesBackendTypes
      */
@@ -48,64 +40,37 @@ protected function setUp(): void
         parent::setUp();
         $this->dataSetup = $this->createMock(ModuleDataSetupInterface::class);
         $this->eavSetupFactory = $this->createMock(EavSetupFactory::class);
-        $this->batchQueryGenerator = $this->createMock(Generator::class);
-        $this->model = new UpdateMultiselectAttributesBackendTypes(
-            $this->dataSetup,
-            $this->eavSetupFactory,
-            $this->batchQueryGenerator
-        );
+        $this->model = new UpdateMultiselectAttributesBackendTypes($this->dataSetup, $this->eavSetupFactory);
     }
 
-    /**
-     * @SuppressWarnings(PHPMD.ExcessiveMethodLength)
-     * @return void
-     */
     public function testApply(): void
     {
         $attributeIds = [3, 7];
         $entityTypeId = 4;
         $eavSetup = $this->createMock(EavSetup::class);
         $connection = $this->createMock(AdapterInterface::class);
-        $selectAttributes = $this->createMock(Select::class);
-        $selectAttributesValues = $this->createMock(Select::class);
-        $selectForInsert1 = $this->createMock(Select::class);
-        $selectForInsert2 = $this->createMock(Select::class);
-        $selectForDelete1 = $this->createMock(Select::class);
-        $selectForDelete2 = $this->createMock(Select::class);
-        $batchIterator = $this->getMockForAbstractClass(BatchIteratorInterface::class);
-        $batchIterator->method('current')
-            ->willReturnOnConsecutiveCalls($selectForInsert1, $selectForInsert2);
-        $batchIterator->method('valid')
-            ->willReturnOnConsecutiveCalls(true, true, false);
-        $this->eavSetupFactory->expects($this->once())
-            ->method('create')
+        $select1 = $this->createMock(Select::class);
+        $select2 = $this->createMock(Select::class);
+        $select3 = $this->createMock(Select::class);
+        $statement = $this->createMock(\Zend_Db_Statement_Interface::class);
+
+        $this->eavSetupFactory->method('create')
             ->willReturn($eavSetup);
-        $this->dataSetup->expects($this->once())
-            ->method('getConnection')
+        $this->dataSetup->method('getConnection')
             ->willReturn($connection);
         $this->dataSetup->method('getTable')
             ->willReturnArgument(0);
-        $this->batchQueryGenerator->expects($this->once())
-            ->method('generate')
-            ->willReturn($batchIterator);
         $eavSetup->method('getEntityTypeId')
             ->willReturn(4);
-        $eavSetup->expects($this->exactly(2))
-            ->method('updateAttribute')
+        $eavSetup->method('updateAttribute')
             ->withConsecutive(
                 [$entityTypeId, 3, 'backend_type', 'text'],
                 [$entityTypeId, 7, 'backend_type', 'text']
             );
-        $connection->expects($this->exactly(4))
+        $connection->expects($this->exactly(2))
             ->method('select')
-            ->willReturnOnConsecutiveCalls(
-                $selectAttributes,
-                $selectAttributesValues,
-                $selectForDelete1,
-                $selectForDelete2
-            );
-        $connection->expects($this->once())
-            ->method('describeTable')
+            ->willReturnOnConsecutiveCalls($select1, $select2, $select3);
+        $connection->method('describeTable')
             ->willReturn(
                 [
                     'value_id' => [],
@@ -115,78 +80,39 @@ public function testApply(): void
                     'row_id' => [],
                 ]
             );
-        $connection->expects($this->once())
-            ->method('fetchCol')
-            ->with($selectAttributes)
+        $connection->method('query')
+            ->willReturn($statement);
+        $connection->method('fetchAll')
+            ->willReturn([]);
+        $connection->method('fetchCol')
+            ->with($select1)
             ->willReturn($attributeIds);
-        $connection->expects($this->exactly(2))
-            ->method('insertFromSelect')
-            ->withConsecutive(
-                [$selectForInsert1, 'catalog_product_entity_text', ['attribute_id', 'store_id', 'value', 'row_id']],
-                [$selectForInsert2, 'catalog_product_entity_text', ['attribute_id', 'store_id', 'value', 'row_id']],
-            )
+        $connection->method('insertFromSelect')
+            ->with($select3, 'catalog_product_entity_text', ['attribute_id', 'store_id', 'value', 'row_id'])
             ->willReturn('');
-        $connection->expects($this->exactly(2))
-            ->method('delete')
-            ->withConsecutive(
-                ['catalog_product_entity_varchar', 'value_id IN (select1)'],
-                ['catalog_product_entity_varchar', 'value_id IN (select2)'],
-            )
+        $connection->method('deleteFromSelect')
+            ->with($select2, 'catalog_product_entity_varchar')
             ->willReturn('');
-        $selectAttributes->expects($this->once())
-            ->method('from')
+        $select1->method('from')
             ->with('eav_attribute', ['attribute_id'])
             ->willReturnSelf();
-        $selectAttributes->expects($this->exactly(3))
-            ->method('where')
+        $select1->method('where')
             ->withConsecutive(
                 ['entity_type_id = ?', $entityTypeId],
                 ['backend_type = ?', 'varchar'],
                 ['frontend_input = ?', 'multiselect']
             )
             ->willReturnSelf();
-        $selectForInsert1->expects($this->exactly(2))
-            ->method('reset')
-            ->with(Select::COLUMNS)
-            ->willReturnSelf();
-        $selectForInsert1->expects($this->exactly(2))
-            ->method('columns')
-            ->withConsecutive(
-                [['attribute_id', 'store_id', 'value', 'row_id']],
-                ['value_id']
-            )
-            ->willReturnSelf();
-        $selectForInsert2->expects($this->exactly(2))
-            ->method('reset')
-            ->with(Select::COLUMNS)
+        $select2->method('from')
+            ->with('catalog_product_entity_varchar')
             ->willReturnSelf();
-        $selectForInsert2->expects($this->exactly(2))
-            ->method('columns')
-            ->withConsecutive(
-                [['attribute_id', 'store_id', 'value', 'row_id']],
-                ['value_id']
-            )
-            ->willReturnSelf();
-        $selectForDelete1->expects($this->once())
-            ->method('from')
-            ->with($selectForInsert1, 'value_id')
-            ->willReturnSelf();
-        $selectForDelete1->expects($this->once())
-            ->method('assemble')
-            ->willReturn('select1');
-        $selectForDelete2->expects($this->once())
-            ->method('from')
-            ->with($selectForInsert2, 'value_id')
+        $select2->method('where')
+            ->with('attribute_id in (?)', $attributeIds)
             ->willReturnSelf();
-        $selectForDelete2->expects($this->once())
-            ->method('assemble')
-            ->willReturn('select2');
-        $selectAttributesValues->expects($this->once())
-            ->method('from')
-            ->with('catalog_product_entity_varchar', '*')
+        $select3->method('from')
+            ->with('catalog_product_entity_varchar', ['attribute_id', 'store_id', 'value', 'row_id'])
             ->willReturnSelf();
-        $selectAttributesValues->expects($this->once())
-            ->method('where')
+        $select3->method('where')
             ->with('attribute_id in (?)', $attributeIds)
             ->willReturnSelf();
         $this->model->apply();
