ACPT-820: Add id to Import Start request for Adminhtml

Also added updated_at column to importexport_importdata.
When import is done through admin ui,  we no longer delete all rows from importexport_importdata before validation - only completed and rows older than 1 day.
Also removed dependency on Base64EncodedCsvData class that was removed.

Author: JacobBrownAustin

app/code/Magento/ImportExport/Block/Adminhtml/Import/Edit/Form.php

@@ -211,7 +211,6 @@ protected function _prepareForm()
             );
             $fieldsets[$behaviorCode] = $fieldset;
         }
-
         // fieldset for file uploading
         $fieldset = $form->addFieldset(
             'upload_file_fieldset',
@@ -257,11 +256,19 @@ protected function _prepareForm()
                 ),
             ]
         );
+        $fieldset->addField(
+            Import::FIELD_IMPORT_IDS,
+            'hidden',
+            [
+                'name' => Import::FIELD_IMPORT_IDS,
+                'label' => __('Import id'),
+                'title' => __('Import id'),
+                'value' => '',
+            ]
+        );
         $fieldsets['upload'] = $fieldset;
-
         $form->setUseContainer(true);
         $this->setForm($form);
-
         return parent::_prepareForm();
     }
 

app/code/Magento/ImportExport/Controller/Adminhtml/Import/Validate.php

@@ -42,15 +42,15 @@ public function execute()
         //phpcs:disable Magento2.Security.Superglobal
         if ($data) {
             // common actions
-            $resultBlock->addAction(
-                'show',
-                'import_validation_container'
-            );
-
+            $resultBlock->addAction('show', 'import_validation_container');
             $import = $this->getImport()->setData($data);
             try {
                 $source = $import->uploadFileAndGetSource();
                 $this->processValidationResult($import->validateSource($source), $resultBlock);
+                $ids = $import->getValidatedIds();
+                if (count($ids) > 0) {
+                    $resultBlock->addAction('value', Import::FIELD_IMPORT_IDS, $ids);
+                }
             } catch (\Magento\Framework\Exception\LocalizedException $e) {
                 $resultBlock->addError($e->getMessage());
             } catch (\Exception $e) {
@@ -117,7 +117,6 @@ private function processValidationResult($validationResult, $resultBlock)
      * Provides import model.
      *
      * @return Import
-     * @deprecated 100.1.0
      */
     private function getImport()
     {

app/code/Magento/ImportExport/Model/Import.php

@@ -98,6 +98,11 @@ class Import extends AbstractModel
      */
     public const FIELD_EMPTY_ATTRIBUTE_VALUE_CONSTANT = '_import_empty_attribute_value_constant';
 
+    /**
+     * Id of the `importexport_importdata` row after validation.
+     */
+    public const FIELD_IMPORT_IDS = '_import_ids';
+
     /**
      * Allow multiple values wrapping in double quotes for additional attributes.
      */
@@ -476,6 +481,12 @@ public function getWorkingDir()
     public function importSource()
     {
         $ids = $this->_getEntityAdapter()->getIds();
+        if (empty($ids)) {
+            $idsFromPostData = $this->getData(self::FIELD_IMPORT_IDS);
+            if (null !== $idsFromPostData && '' !== $idsFromPostData) {
+                $ids = explode(",", $idsFromPostData);;
+            }
+        }
         $this->setData('entity', $this->getDataSourceModel()->getEntityTypeCode($ids));
         $this->setData('behavior', $this->getDataSourceModel()->getBehavior($ids));
 
@@ -852,4 +863,14 @@ public function getDeletedItemsCount()
     {
         return $this->_getEntityAdapter()->getDeletedItemsCount();
     }
+
+    /**
+     * Retrieve Ids of Validated Rows
+     *
+     * @return int[]
+     */
+    public function getValidatedIds() : array
+    {
+        return $this->_getEntityAdapter()->getIds() ?? [];
+    }
 }

app/code/Magento/ImportExport/Model/Import/AbstractEntity.php

@@ -421,9 +421,7 @@ protected function _saveValidatedBunches()
         $startNewBunch = false;
 
         $source->rewind();
-        if (!$source instanceof Base64EncodedCsvData) {
-            $this->_dataSourceModel->cleanBunches();
-        }
+        $this->_dataSourceModel->cleanProcessedBunches();
         $mainAttributeCode = $this->getMasterAttributeCode();
 
         while ($source->valid() || count($bunchRows) || isset($entityGroup)) {

app/code/Magento/ImportExport/Model/Import/Entity/AbstractEntity.php

@@ -395,10 +395,7 @@ protected function _saveValidatedBunches()
         $skuSet = [];
 
         $source->rewind();
-        if (!$source instanceof Base64EncodedCsvData) {
-            $this->_dataSourceModel->cleanBunches();
-        }
-
+        $this->_dataSourceModel->cleanProcessedBunches();
         while ($source->valid() || $bunchRows) {
             if ($startNewBunch || !$source->valid()) {
                 $this->ids[] =

app/code/Magento/ImportExport/Model/ResourceModel/Import/Data.php

@@ -125,9 +125,7 @@ public function cleanProcessedBunches()
     {
         $this->getConnection()->delete(
             $this->getMainTable(),
-            [
-                'is_processed' => '1'
-            ]
+            'is_processed = 1 OR TIMESTAMPADD(DAY, 1, updated_at) < NOW() '
         );
     }
 

app/code/Magento/ImportExport/etc/db_schema.xml

@@ -13,6 +13,8 @@
         <column xsi:type="varchar" name="behavior" nullable="false" length="10" default="append" comment="Behavior"/>
         <column xsi:type="longtext" name="data" nullable="true" comment="Data"/>
         <column xsi:type="boolean" name="is_processed" nullable="false" default="true" comment="Is Row Processed"/>
+        <column xsi:type="timestamp" name="updated_at" on_update="true" nullable="false" default="CURRENT_TIMESTAMP"
+                comment="timestamp of last update"/>
         <constraint xsi:type="primary" referenceId="PRIMARY">
             <column name="id"/>
         </constraint>

app/code/Magento/ImportExport/etc/db_schema_whitelist.json

@@ -5,7 +5,8 @@
             "entity": true,
             "behavior": true,
             "data": true,
-            "is_processed": true
+            "is_processed": true,
+            "updated_at": true
         },
         "constraint": {
             "PRIMARY": true

dev/tests/integration/testsuite/Magento/CatalogImportExport/Model/Import/ProductTest/ProductOptionsTest.php

@@ -125,7 +125,9 @@ public function testSaveCustomOptions(string $importFile, string $sku, int $expe
 
         // Make sure that after importing existing options again, option IDs and option value IDs are not changed
         $customOptionValues = $this->getCustomOptionValues($sku);
-        $this->createImportModel($pathToFile)->importData();
+        $importModel = $this->createImportModel($pathToFile);
+        $importModel->validateData();
+        $importModel->importData();
         $this->assertEquals($customOptionValues, $this->getCustomOptionValues($sku));
 
         // Cleanup imported products
@@ -222,7 +224,9 @@ public function testSaveCustomOptionsWithMultipleStoreViews()
 
             // Make sure that after importing existing options again, option IDs and option value IDs are not changed
             $customOptionValues = $this->getCustomOptionValues($sku);
-            $this->createImportModel($pathToFile)->importData();
+            $importModel = $this->createImportModel($pathToFile);
+            $importModel->validateData();
+            $importModel->importData();
             $this->assertEquals(
                 $customOptionValues,
                 $this->getCustomOptionValues($sku),