Skip to content

Commit 0d04908

Browse files
vzabaznovvzabaznov
authored
Merge pull request #7748 from magento-performance/ACPT-6
ACPT-6: Improve Data Import Performance
2 parents c6aeb6a + e393b30 commit 0d04908

File tree

11 files changed

+360
-125841
lines changed

11 files changed

+360
-125841
lines changed

‎app/code/Magento/ImportExport/Model/Import.php‎

Lines changed: 21 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use Magento\Framework\HTTP\Adapter\FileTransferFactory;
1919
use Magento\Framework\Indexer\IndexerRegistry;
2020
use Magento\Framework\Math\Random;
21+
use Magento\Framework\Message\ManagerInterface;
2122
use Magento\Framework\Stdlib\DateTime\DateTime;
2223
use Magento\ImportExport\Helper\Data as DataHelper;
2324
use Magento\ImportExport\Model\Export\Adapter\CsvFactory;
@@ -29,11 +30,10 @@
2930
use Magento\ImportExport\Model\Import\Entity\Factory;
3031
use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingError;
3132
use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingErrorAggregatorInterface;
32-
use Magento\Framework\Message\ManagerInterface;
3333
use Magento\ImportExport\Model\ResourceModel\Import\Data;
3434
use Magento\ImportExport\Model\Source\Import\AbstractBehavior;
3535
use Magento\ImportExport\Model\Source\Import\Behavior\Factory as BehaviorFactory;
36-
use Magento\MediaStorage\Model\File\Uploader;
36+
use Magento\ImportExport\Model\Source\Upload;
3737
use Magento\MediaStorage\Model\File\UploaderFactory;
3838
use Psr\Log\LoggerInterface;
3939

@@ -122,6 +122,7 @@ class Import extends AbstractModel
122122
protected $_entityAdapter;
123123

124124
/**
125+
* @Deprecated Property isn't used
125126
* @var DataHelper
126127
*/
127128
protected $_importExportData = null;
@@ -152,6 +153,7 @@ class Import extends AbstractModel
152153
protected $_csvFactory;
153154

154155
/**
156+
* @Deprecated Property isn't used
155157
* @var FileTransferFactory
156158
*/
157159
protected $_httpFactory;
@@ -192,10 +194,16 @@ class Import extends AbstractModel
192194
private $messageManager;
193195

194196
/**
197+
* @Deprecated Property isn't used
195198
* @var Random
196199
*/
197200
private $random;
198201

202+
/**
203+
* @var Upload
204+
*/
205+
private $upload;
206+
199207
/**
200208
* @param LoggerInterface $logger
201209
* @param Filesystem $filesystem
@@ -214,6 +222,7 @@ class Import extends AbstractModel
214222
* @param array $data
215223
* @param ManagerInterface|null $messageManager
216224
* @param Random|null $random
225+
* @param Upload|null $upload
217226
* @SuppressWarnings(PHPMD.ExcessiveParameterList)
218227
*/
219228
public function __construct(
@@ -233,7 +242,8 @@ public function __construct(
233242
DateTime $localeDate,
234243
array $data = [],
235244
ManagerInterface $messageManager = null,
236-
Random $random = null
245+
Random $random = null,
246+
Upload $upload = null
237247
) {
238248
$this->_importExportData = $importExportData;
239249
$this->_coreConfig = $coreConfig;
@@ -252,6 +262,8 @@ public function __construct(
252262
->get(ManagerInterface::class);
253263
$this->random = $random ?: ObjectManager::getInstance()
254264
->get(Random::class);
265+
$this->upload = $upload ?: ObjectManager::getInstance()
266+
->get(Upload::class);
255267
parent::__construct($logger, $filesystem, $data);
256268
}
257269

@@ -303,6 +315,8 @@ protected function _getEntityAdapter()
303315
/**
304316
* Returns source adapter object.
305317
*
318+
* @Deprecated
319+
* @see \Magento\ImportExport\Model\Import\Source\Factory::create()
306320
* @param string $sourceFile Full path to source file
307321
* @return AbstractSource
308322
* @throws FileSystemException
@@ -550,61 +564,12 @@ public function getErrorAggregator()
550564
*/
551565
public function uploadSource()
552566
{
553-
/** @var $adapter \Zend_File_Transfer_Adapter_Http */
554-
$adapter = $this->_httpFactory->create();
555-
if (!$adapter->isValid(self::FIELD_NAME_SOURCE_FILE)) {
556-
$errors = $adapter->getErrors();
557-
if ($errors[0] == \Zend_Validate_File_Upload::INI_SIZE) {
558-
$errorMessage = $this->_importExportData->getMaxUploadSizeMessage();
559-
} else {
560-
$errorMessage = __('The file was not uploaded.');
561-
}
562-
throw new LocalizedException($errorMessage);
563-
}
564-
565567
$entity = $this->getEntity();
566-
/** @var $uploader Uploader */
567-
$uploader = $this->_uploaderFactory->create(['fileId' => self::FIELD_NAME_SOURCE_FILE]);
568-
$uploader->setAllowedExtensions(['csv', 'zip']);
569-
$uploader->skipDbProcessing(true);
570-
$fileName = $this->random->getRandomString(32) . '.' . $uploader->getFileExtension();
571-
try {
572-
$result = $uploader->save($this->getWorkingDir(), $fileName);
573-
} catch (\Exception $e) {
574-
throw new LocalizedException(__('The file cannot be uploaded.'));
575-
}
576-
577-
$extension = '';
578-
$uploadedFile = '';
579-
if ($result !== false) {
580-
// phpcs:ignore Magento2.Functions.DiscouragedFunction
581-
$extension = pathinfo($result['file'], PATHINFO_EXTENSION);
582-
$uploadedFile = $result['path'] . $result['file'];
583-
}
584-
585-
if (!$extension) {
586-
$this->_varDirectory->delete($uploadedFile);
587-
throw new LocalizedException(__('The file you uploaded has no extension.'));
588-
}
589-
$sourceFile = $this->getWorkingDir() . $entity;
590-
591-
$sourceFile .= '.' . $extension;
568+
$result = $this->upload->uploadSource($entity);
569+
// phpcs:ignore Magento2.Functions.DiscouragedFunction
570+
$extension = pathinfo($result['file'], PATHINFO_EXTENSION);
571+
$sourceFile = $this->getWorkingDir() . $entity . '.' . $extension;
592572
$sourceFileRelative = $this->_varDirectory->getRelativePath($sourceFile);
593-
594-
if (strtolower($uploadedFile) != strtolower($sourceFile)) {
595-
if ($this->_varDirectory->isExist($sourceFileRelative)) {
596-
$this->_varDirectory->delete($sourceFileRelative);
597-
}
598-
599-
try {
600-
$this->_varDirectory->renameFile(
601-
$this->_varDirectory->getRelativePath($uploadedFile),
602-
$sourceFileRelative
603-
);
604-
} catch (FileSystemException $e) {
605-
throw new LocalizedException(__('The source file moving process failed.'));
606-
}
607-
}
608573
$this->_removeBom($sourceFile);
609574
$this->createHistoryReport($sourceFileRelative, $entity, $extension, $result);
610575
return $sourceFile;

‎app/code/Magento/ImportExport/Model/Import/Adapter.php‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66
namespace Magento\ImportExport\Model\Import;
77

88
use Magento\Framework\Filesystem\Directory\Write;
9+
use Magento\ImportExport\Model\Import\Source\Factory;
910

1011
/**
1112
* Import adapter model
12-
*
13+
* @Deprecated
14+
* @see \Magento\ImportExport\Model\Import\Source\Factory
1315
* @author Magento Core Team <core@magentocommerce.com>
1416
*/
1517
class Adapter
@@ -23,7 +25,7 @@ class Adapter
2325
* @param mixed $options OPTIONAL Adapter constructor options
2426
*
2527
* @return AbstractSource
26-
*
28+
* phpcs:disable Magento2.Functions.StaticFunction
2729
* @throws \Magento\Framework\Exception\LocalizedException
2830
*/
2931
public static function factory($type, $directory, $source, $options = null)
@@ -56,11 +58,11 @@ public static function factory($type, $directory, $source, $options = null)
5658
* @param string $source Source file path.
5759
* @param Write $directory
5860
* @param mixed $options OPTIONAL Adapter constructor options
59-
*
61+
* phpcs:disable Magento2.Functions.StaticFunction
6062
* @return AbstractSource
6163
*/
6264
public static function findAdapterFor($source, $directory, $options = null)
6365
{
64-
return self::factory(pathinfo($source, PATHINFO_EXTENSION), $directory, $source, $options);
66+
return self::factory(pathinfo($source, PATHINFO_EXTENSION), $directory, $source, $options); // phpcs:ignore
6567
}
6668
}

‎app/code/Magento/ImportExport/Model/Import/Source/Base64EncodedCsvData.php‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
declare(strict_types=1);
7+
8+
namespace Magento\ImportExport\Model\Import\Source;
9+
10+
use Magento\ImportExport\Model\Import\AbstractSource;
11+
12+
class Base64EncodedCsvData extends AbstractSource
13+
{
14+
/**
15+
* @var array
16+
*/
17+
private $rows;
18+
19+
/**
20+
* @var string
21+
*/
22+
private $delimiter = ',';
23+
24+
/**
25+
* Read Data and detect column names
26+
*
27+
* @param string $file
28+
*/
29+
public function __construct(string $file)
30+
{
31+
// phpcs:ignore Magento2.Functions.DiscouragedFunction
32+
$source = trim(base64_decode($file));
33+
$rowsData = preg_split("/\r\n|\n|\r/", $source);
34+
$colNames = explode($this->delimiter, $rowsData[0]);
35+
$this->rows = array_splice($rowsData, 1);
36+
parent::__construct($colNames);
37+
}
38+
39+
/**
40+
* Read next line from CSV data
41+
*
42+
* @return array
43+
*/
44+
public function _getNextRow()
45+
{
46+
if ($this->_key===count($this->rows)) {
47+
return [];
48+
}
49+
$parsed =str_getcsv($this->rows[$this->_key], ',', '"');
50+
if (is_array($parsed) && count($parsed) != $this->_colQty) {
51+
foreach ($parsed as $element) {
52+
if ($element && strpos($element, "'") !== false) {
53+
$this->_foundWrongQuoteFlag = true;
54+
break;
55+
}
56+
}
57+
} else {
58+
$this->_foundWrongQuoteFlag = false;
59+
}
60+
return is_array($parsed) ? $parsed : [];
61+
}
62+
}

‎app/code/Magento/ImportExport/Model/Import/Source/Factory.php‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
declare(strict_types=1);
8+
9+
namespace Magento\ImportExport\Model\Import\Source;
10+
11+
use Magento\Framework\Filesystem\Directory\Write;
12+
use Magento\Framework\ObjectManagerInterface;
13+
use Magento\ImportExport\Model\Import\AbstractSource;
14+
15+
class Factory
16+
{
17+
/**
18+
* Object Manager Instance
19+
*
20+
* @var \Magento\Framework\ObjectManagerInterface
21+
*/
22+
private $objectManager;
23+
24+
/**
25+
* @param ObjectManagerInterface $objectManager
26+
*/
27+
public function __construct(
28+
ObjectManagerInterface $objectManager
29+
) {
30+
$this->objectManager = $objectManager;
31+
}
32+
33+
/**
34+
* Create class instance with specified parameters
35+
*
36+
* @param string $source
37+
* @param Write $directory
38+
* @param mixed $options
39+
* @return AbstractSource
40+
* @phpcs:disable Magento2.Functions.DiscouragedFunction
41+
* @throws \Magento\Framework\Exception\LocalizedException
42+
*/
43+
public function create($source, $directory = null, $options = null): AbstractSource
44+
{
45+
$adapterClass = 'Magento\ImportExport\Model\Import\Source\\';
46+
if (file_exists($source)) {
47+
$type = ucfirst(strtolower(pathinfo($source, PATHINFO_EXTENSION)));
48+
} else {
49+
$type = 'Base64EncodedCsvData';
50+
}
51+
if (!is_string($source) || !$source) {
52+
throw new \Magento\Framework\Exception\LocalizedException(
53+
__('The source type must be a non-empty string.')
54+
);
55+
}
56+
$adapterClass.= $type;
57+
if (!class_exists($adapterClass)) {
58+
throw new \Magento\Framework\Exception\LocalizedException(
59+
__('\'%1\' file extension is not supported', $type)
60+
);
61+
}
62+
return $this->objectManager->create(
63+
$adapterClass,
64+
[
65+
'file' => $source,
66+
'directory' => $directory,
67+
'options' => $options
68+
]
69+
);
70+
}
71+
}

0 commit comments

Comments
 (0)