Missing options fix (#239)

akaplya · LucianRadu · web-flow · commit 9f22a65e8de1 · 2023-03-27T18:28:01.000-05:00
* MDEE-415: Add whole additional data json for swatch (#242)
---------
Co-authored-by: LucianRadu &lt;lucian.radu91@gmail.com&gt;
diff --git a/CatalogDataExporter/etc/et_schema.xml b/CatalogDataExporter/etc/et_schema.xml
@@ -197,6 +197,7 @@
         <field name="required" type="Boolean" />
         <field name="renderType" type="String" />
         <field name="type" type="String" />
+        <field name="swatchType" type="String" />
         <field name="values" type="ProductOptionValue" repeated="true" />
     </record>
 
@@ -205,7 +206,9 @@
         <field name="label" type="String" />
         <field name="sortOrder" type="Int" />
         <field name="isDefault" type="Boolean" />
+        <field name="colorHex" type="String" />
         <field name="imageUrl" type="String" />
+        <field name="customSwatchValue" type="String" />
         <field name="qtyMutability" type="Boolean" />
         <field name="qty" type="Float" />
         <field name="infoUrl" type="String" />
diff --git a/ConfigurableProductDataExporter/Model/Provider/Product/Options.php b/ConfigurableProductDataExporter/Model/Provider/Product/Options.php
@@ -8,30 +8,22 @@
 namespace Magento\ConfigurableProductDataExporter\Model\Provider\Product;
 
 use Exception;
-use Generator;
 use Magento\CatalogDataExporter\Model\Provider\Product\OptionProviderInterface;
 use Magento\ConfigurableProduct\Model\Product\Type\Configurable;
 use Magento\ConfigurableProductDataExporter\Model\Query\ProductOptionQuery;
 use Magento\ConfigurableProductDataExporter\Model\Query\ProductOptionValueQuery;
 use Magento\DataExporter\Exception\UnableRetrieveData;
-use Magento\Framework\App\ResourceConnection;
-use Magento\Framework\DB\Query\BatchIteratorInterface;
-use Magento\Framework\DB\Query\Generator as QueryGenerator;
-use Magento\Framework\DB\Select;
 use Magento\DataExporter\Model\Logging\CommerceDataExportLoggerInterface as LoggerInterface;
-use Throwable;
+use Magento\Framework\App\ResourceConnection;
+use Magento\Framework\DB\Sql\ColumnValueExpression;
+use Magento\Swatches\Helper\Media as MediaHelper;
+use Magento\Swatches\Model\Swatch;
 
 /**
  * Configurable product options data provider
  */
 class Options implements OptionProviderInterface
 {
-    /**
-     * Batch sizing for performing queries
-     *
-     * @var int
-     */
-    private $batchSize;
 
     /**
      * @var ResourceConnection
@@ -53,70 +45,93 @@ class Options implements OptionProviderInterface
      */
     private $logger;
 
-    /**
-     * @var QueryGenerator
-     */
-    private $queryGenerator;
-
     /**
      * @var ConfigurableOptionValueUid
      */
     private $optionValueUid;
 
-    private static $optionValuesPerAttributesCache = [];
+    /**
+     * @var MediaHelper
+     */
+    private $mediaHelper;
 
     /**
      * @param ResourceConnection $resourceConnection
      * @param ProductOptionQuery $productOptionQuery
      * @param ProductOptionValueQuery $productOptionValueQuery
-     * @param QueryGenerator $queryGenerator
      * @param ConfigurableOptionValueUid $optionValueUid
+     * @param MediaHelper $mediaHelper
      * @param LoggerInterface $logger
-     * @param int $batchSize
      */
     public function __construct(
         ResourceConnection $resourceConnection,
         ProductOptionQuery $productOptionQuery,
         ProductOptionValueQuery $productOptionValueQuery,
-        QueryGenerator $queryGenerator,
         ConfigurableOptionValueUid $optionValueUid,
-        LoggerInterface $logger,
-        int $batchSize
+        MediaHelper $mediaHelper,
+        LoggerInterface $logger
     ) {
         $this->resourceConnection = $resourceConnection;
         $this->productOptionQuery = $productOptionQuery;
         $this->productOptionValueQuery = $productOptionValueQuery;
-        $this->queryGenerator = $queryGenerator;
         $this->optionValueUid = $optionValueUid;
+        $this->mediaHelper = $mediaHelper;
         $this->logger = $logger;
-        $this->batchSize = $batchSize;
     }
 
     /**
-     * Retrieve query data in batches
+     * Returns table name
      *
-     * @param Select $select
-     * @param string $rangeField
-     * @return Generator
-     * @throws UnableRetrieveData
+     * @param string $reference
+     * @return string
      */
-    private function getBatchedQueryData(Select $select, string $rangeField): Generator
+    private function getTable(string $reference)
     {
-        try {
-            $connection = $this->resourceConnection->getConnection();
-            $iterator = $this->queryGenerator->generate(
-                $rangeField,
-                $select,
-                $this->batchSize,
-                BatchIteratorInterface::NON_UNIQUE_FIELD_ITERATOR
+        return $this->resourceConnection->getTableName($reference);
+    }
+
+    /**
+     * Returns possible attribute valies for a product
+     *
+     * @param int $entityId
+     * @param int $attributeId
+     * @return array
+     */
+    private function getPossibleAttributeValues($entityId, $attributeId)
+    {
+        $connection = $this->resourceConnection->getConnection();
+        $joinField = $connection->getAutoIncrementField($this->getTable('catalog_product_entity'));
+        $select = $connection->select()
+            ->from(['cpe' => $this->getTable('catalog_product_entity')], [])
+            ->join(
+                ['psl' => $this->getTable('catalog_product_super_link')],
+                sprintf('psl.parent_id = cpe.%s', $joinField),
+                []
+            )
+            ->join(
+                ['iss' => $this->getTable('cataloginventory_stock_status')],
+                'iss.product_id = psl.product_id AND iss.stock_status = 1',
+                []
+            )
+            ->join(
+                ['cpc' => $this->getTable('catalog_product_entity')],
+                'cpc.entity_id = psl.product_id',
+                []
+            )
+            ->join(
+                ['cpi' => $this->getTable('catalog_product_entity_int')],
+                sprintf(
+                    'cpi.%1$s = cpc.%1$s AND cpi.store_id = 0 AND cpi.attribute_id = %2$d',
+                    $joinField,
+                    $attributeId
+                ),
+                []
+            )
+            ->where('cpe.entity_id = ?', $entityId)
+            ->columns(
+                new ColumnValueExpression('DISTINCT cpi.value')
             );
-            foreach ($iterator as $batchSelect) {
-                yield $connection->fetchAll($batchSelect);
-            }
-        } catch (Throwable $exception) {
-            $this->logger->error($exception->getMessage(), ['exception' => $exception]);
-            throw new UnableRetrieveData('Unable to retrieve configurable option data');
-        }
+        return $connection->fetchCol($select);
     }
 
     /**
@@ -128,37 +143,32 @@ private function getBatchedQueryData(Select $select, string $rangeField): Genera
      */
     private function getOptionValuesData(array $arguments): array
     {
-        $attributeIdsOrigin = $this->getAttributeIds($arguments);
-        $attributeIds = \array_diff(
-            $attributeIdsOrigin,
-            \array_keys(self::$optionValuesPerAttributesCache)
-        );
+        $arguments['attributes'] = $this->getAttributeIds($arguments);
+        $select = $this->productOptionValueQuery->getQuery($arguments);
 
-        if (!$attributeIds) {
-            // get from cache
-            return $this->getOptionValuesFromCache($attributeIdsOrigin);
-        }
-        $arguments['attributes'] = $attributeIds;
+        $cursor = $this->resourceConnection->getConnection()->query($select);
 
-        $select = $this->productOptionValueQuery->getQuery($arguments);
-        // ad hoc solution to check application cache size. should be replaced with generic approach
-        $cacheSize = \strlen(\json_encode(self::$optionValuesPerAttributesCache));
-        if ($cacheSize > 1024 * 1024 * 20) {
-            self::$optionValuesPerAttributesCache = [];
-        }
-        foreach ($this->getBatchedQueryData($select, 'attribute_id') as $batchData) {
-            foreach ($batchData as $row) {
-                self::$optionValuesPerAttributesCache[$row['attribute_id']][$row['storeViewCode']][$row['optionId']] = [
-                    'id' => $this->optionValueUid->resolve($row['attribute_id'], $row['optionId']),
-                    'label' => $row['label'],
-                ];
-            }
+        $data = [];
+        while ($row = $cursor->fetch()) {
+            $data[$row['attribute_id']][$row['storeViewCode']][$row['optionId']] = [
+                'id' => $this->optionValueUid->resolve($row['attribute_id'], $row['optionId']),
+                'label' => $row['label'],
+                'colorHex' => $row['swatchType'] == Swatch::SWATCH_TYPE_VISUAL_COLOR
+                    ? $row['swatchValue'] : null,
+                'imageUrl' => $row['swatchType'] == Swatch::SWATCH_TYPE_VISUAL_IMAGE
+                    ? $this->mediaHelper->getSwatchMediaUrl() . $row['swatchValue'] : null,
+                'customSwatchValue' => !in_array(
+                    $row['swatchType'],
+                    [Swatch::SWATCH_TYPE_TEXTUAL, Swatch::SWATCH_TYPE_VISUAL_COLOR, Swatch::SWATCH_TYPE_VISUAL_IMAGE]
+                ) ? $row['swatchValue'] : null
+            ];
         }
-
-        return $this->getOptionValuesFromCache($attributeIdsOrigin);
+        return $data;
     }
 
     /**
+     * Returns attribute IDs associated with this product
+     *
      * @param array $arguments
      * @return array
      */
@@ -200,7 +210,8 @@ private function formatOptionsRow($row): array
                 'id' => $row['attribute_code'],
                 'type' => ConfigurableOptionValueUid::OPTION_TYPE,
                 'label' => $row['label'],
-                'sortOrder' => $row['position']
+                'sortOrder' => $row['position'],
+                'swatchType' => $row['swatchType']
             ],
         ];
     }
@@ -221,11 +232,12 @@ private function getOptionKey($row): string
      */
     public function get(array $values): array
     {
+        $temp = [];
 
         $queryArguments = [];
         foreach ($values as $value) {
             if (!isset($value['productId'], $value['type'], $value['storeViewCode'])
-                || $value['type'] !== Configurable::TYPE_CODE ) {
+                || $value['type'] !== Configurable::TYPE_CODE) {
                 continue;
             }
             $queryArguments['productId'][$value['productId']] = $value['productId'];
@@ -240,17 +252,24 @@ public function get(array $values): array
             $optionValuesData = $this->getOptionValuesData($queryArguments);
 
             $select = $this->productOptionQuery->getQuery($queryArguments);
-            foreach ($this->getBatchedQueryData($select, 'entity_id') as $batchData) {
-                foreach ($batchData as $row) {
-                    $key = $this->getOptionKey($row);
-                    $options[$key] = $options[$key] ?? $this->formatOptionsRow($row);
 
-                    if (isset($optionValuesData[$row['attribute_id']][$row['storeViewCode']])) {
-                        $options[$key]['optionsV2']['values'] = $this->getAssignedAttributeValues(
-                            $optionValuesData[$row['attribute_id']][$row['storeViewCode']],
-                            explode(',', $row['attributeValues'])
-                        );
-                    }
+            $cursor = $this->resourceConnection->getConnection()->query($select);
+
+            while ($row = $cursor->fetch()) {
+                if (!isset($temp[$row['productId'] . '-' . $row['attribute_id']])) {
+                    $temp[$row['productId'] . '-' . $row['attribute_id']] =
+                        $this->getPossibleAttributeValues($row['productId'], $row['attribute_id']);
+                }
+                $filter = $temp[$row['productId'] . '-' . $row['attribute_id']];
+
+                $key = $this->getOptionKey($row);
+                $options[$key] = $options[$key] ?? $this->formatOptionsRow($row);
+
+                if (isset($optionValuesData[$row['attribute_id']][$row['storeViewCode']])) {
+                    $options[$key]['optionsV2']['values'] = $this->getAssignedAttributeValues(
+                        $optionValuesData[$row['attribute_id']][$row['storeViewCode']],
+                        $filter
+                    );
                 }
             }
         } catch (Exception $exception) {
@@ -261,15 +280,8 @@ public function get(array $values): array
     }
 
     /**
-     * @param array|null $attributeIds
-     * @return array
-     */
-    private function getOptionValuesFromCache(?array $attributeIds): array
-    {
-        return \array_intersect_key(self::$optionValuesPerAttributesCache, \array_flip($attributeIds));
-    }
-
-    /**
+     * Filter values
+     *
      * @param array $attributeValuesList
      * @param array $assignedAttributeValuesId
      * @return array
diff --git a/ConfigurableProductDataExporter/Model/Query/ProductOptionQuery.php b/ConfigurableProductDataExporter/Model/Query/ProductOptionQuery.php
@@ -38,8 +38,8 @@ public function __construct(
      */
     public function getQuery(array $arguments): Select
     {
-        $productIds = isset($arguments['productId']) ? $arguments['productId'] : [];
-        $storeViewCodes = isset($arguments['storeViewCode']) ? $arguments['storeViewCode'] : [];
+        $productIds = $arguments['productId'] ?? [];
+        $storeViewCodes = $arguments['storeViewCode'] ?? [];
         $connection = $this->resourceConnection->getConnection();
         $joinField = $connection->getAutoIncrementField(
             $this->resourceConnection->getTableName('catalog_product_entity')
@@ -60,6 +60,10 @@ public function getQuery(array $arguments): Select
                 ['eav' => $this->resourceConnection->getTableName('eav_attribute')],
                 'eav.attribute_id = super_attribute.attribute_id',
                 ['attribute_code' => 'eav.attribute_code']
+            )->join(
+                ['cea' => $this->resourceConnection->getTableName('catalog_eav_attribute')],
+                'cea.attribute_id = super_attribute.attribute_id',
+                ['swatchType' => 'cea.additional_data']
             )->join(
                 ['product_website' => $this->resourceConnection->getTableName('catalog_product_website')],
                 'product_website.product_id = product.entity_id',
@@ -71,19 +75,6 @@ public function getQuery(array $arguments): Select
                      . ' AND s.website_id = product_website.website_id'
                     : 's.store_id != 0' . ' AND s.website_id = product_website.website_id',
                 ['storeViewCode' => 's.code']
-            )->join(
-                ['configurable_link' => $this->resourceConnection->getTableName('catalog_product_super_link')],
-                sprintf('configurable_link.parent_id = product.%s', $joinField),
-                []
-            )->join(
-                ['catalog_product_entity' => $this->resourceConnection->getTableName('catalog_product_entity')],
-                'catalog_product_entity.entity_id = configurable_link.product_id',
-                []
-            )->join(
-                ['super_attribute_value' => $this->resourceConnection->getTableName('catalog_product_entity_int')],
-                sprintf('super_attribute_value.%s = catalog_product_entity.%s', $joinField, $joinField)
-                    . ' AND super_attribute_value.attribute_id = super_attribute.attribute_id',
-                ['attributeValues' => 'GROUP_CONCAT(DISTINCT super_attribute_value.value)']
             )
             ->joinLeft(
                 ['attr_label' => $this->resourceConnection->getTableName('eav_attribute_label')],
@@ -96,8 +87,7 @@ public function getQuery(array $arguments): Select
                     ),
                 ]
             )
-            ->where('product.entity_id IN (?)', $productIds)
-            ->group(['product.entity_id', 's.store_id', 'eav.attribute_code']);
+            ->where('product.entity_id IN (?)', $productIds);
         return $select;
     }
 }
diff --git a/ConfigurableProductDataExporter/Model/Query/ProductOptionValueQuery.php b/ConfigurableProductDataExporter/Model/Query/ProductOptionValueQuery.php
@@ -67,6 +67,19 @@ public function getQuery(array $arguments): Select
                     'label' => new Expression('CASE WHEN ovs.value IS NULL THEN ovd.value ELSE ovs.value END'),
                 ]
             )
+            ->joinLeft(
+                ['aod' => $this->resourceConnection->getTableName('eav_attribute_option_swatch')],
+                'aod.option_id = eao.option_id AND aod.store_id = 0',
+                []
+            )
+            ->joinLeft(
+                ['aos' => $this->resourceConnection->getTableName('eav_attribute_option_swatch')],
+                'aos.option_id = eao.option_id AND aos.store_id = s.store_id',
+                [
+                    'swatchValue' => new Expression('CASE WHEN aos.value IS NULL THEN aod.value ELSE aos.value END'),
+                    'swatchType' => new Expression('CASE WHEN aos.value IS NULL THEN aod.type ELSE aos.type END'),
+                ]
+            )
             ->where('eao.attribute_id IN (?)', $attributeIds)
             ->where('s.code IN (?)', $storeViewCodes);
         return $select;
diff --git a/ConfigurableProductDataExporter/composer.json b/ConfigurableProductDataExporter/composer.json
@@ -18,6 +18,7 @@
         "php": "~8.1.0||~8.2.0",
         "magento/framework": ">=103.0.4",
         "magento/module-configurable-product": ">=100.4.4",
+        "magento/module-swatches": ">=100.4.4",
         "magento/module-catalog": ">=104.0.4",
         "magento/module-eav": ">=102.1.4",
         "magento/module-store": ">=101.1.4",
