From 0b1f3b048e95ee972e830d5cefdf3ec223baf862 Mon Sep 17 00:00:00 2001 From: Nathan Day Date: Wed, 30 Apr 2025 11:14:01 +0100 Subject: [PATCH 001/119] Refactor Product With Children to remove Double Conversion --- .../PriceManager/ProductWithChildren.php | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Helper/Entity/Product/PriceManager/ProductWithChildren.php b/Helper/Entity/Product/PriceManager/ProductWithChildren.php index 2c1f52ac1..62a298978 100755 --- a/Helper/Entity/Product/PriceManager/ProductWithChildren.php +++ b/Helper/Entity/Product/PriceManager/ProductWithChildren.php @@ -58,8 +58,17 @@ protected function getMinMaxPrices(Product $product, $withTax, $subProducts, $cu } else { $minPrice = $specialPrice[0]; } - $price = $minPrice ?? $this->getTaxPrice($product, $subProduct->getFinalPrice(), $withTax); - $basePrice = $this->getTaxPrice($product, $subProduct->getPrice(), $withTax); + + $finalPrice = $subProduct->getFinalPrice(); + $basePrice = $subProduct->getPrice(); + + if ($currencyCode !== $this->baseCurrencyCode) { + $finalPrice = $this->convertPrice($finalPrice, $currencyCode); + $basePrice = $this->convertPrice($basePrice, $currencyCode); + } + + $price = $minPrice ?? $this->getTaxPrice($product, $finalPrice, $withTax); + $basePrice = $this->getTaxPrice($product, $basePrice, $withTax); $min = min($min, $price); $original = min($original, $basePrice); $max = max($max, $price); @@ -68,14 +77,7 @@ protected function getMinMaxPrices(Product $product, $withTax, $subProducts, $cu } else { $originalMax = $original = $min = $max; } - if ($currencyCode !== $this->baseCurrencyCode) { - $min = $this->convertPrice($min, $currencyCode); - $original = $this->convertPrice($original, $currencyCode); - if ($min !== $max) { - $max = $this->convertPrice($max, $currencyCode); - $originalMax = $this->convertPrice($originalMax, $currencyCode); - } - } + return [$min, $max, $original, $originalMax]; } From 49e22d6c4f4853ebc9c6bf67263c4d804ec48614 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 4 Jun 2025 16:32:23 -0400 Subject: [PATCH 002/119] MAGE-1083 Refactoring batch handling --- Model/Indexer/Product.php | 28 +++++--- Service/Product/BatchQueueProcessor.php | 96 ++++++++++++++++--------- 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/Model/Indexer/Product.php b/Model/Indexer/Product.php index 860c8947e..fb3155fb8 100755 --- a/Model/Indexer/Product.php +++ b/Model/Indexer/Product.php @@ -2,9 +2,8 @@ namespace Algolia\AlgoliaSearch\Model\Indexer; +use Algolia\AlgoliaSearch\Api\Processor\BatchQueueProcessorInterface; use Algolia\AlgoliaSearch\Helper\ConfigHelper; -use Algolia\AlgoliaSearch\Service\Product\BatchQueueProcessor as ProductBatchQueueProcessor; -use Magento\Framework\Exception\NoSuchEntityException; use Magento\Store\Model\StoreManagerInterface; /** @@ -16,34 +15,43 @@ class Product implements \Magento\Framework\Indexer\ActionInterface, \Magento\Fr public function __construct( protected StoreManagerInterface $storeManager, protected ConfigHelper $configHelper, - protected ProductBatchQueueProcessor $productBatchQueueProcessor + protected BatchQueueProcessorInterface $productBatchQueueProcessor ) {} /** - * @throws NoSuchEntityException + * {@inheritdoc} */ - public function execute($productIds) + public function execute($ids): void { foreach (array_keys($this->storeManager->getStores()) as $storeId) { - $this->productBatchQueueProcessor->processBatch($storeId, $productIds); + $this->productBatchQueueProcessor->processBatch($storeId, $ids); } } - public function executeFull() + /** + * {@inheritdoc} + */ + public function executeFull(): void { if (!$this->configHelper->isProductsIndexerEnabled()) { return; } - $this->execute(null); + $this->execute([]); } - public function executeList(array $ids) + /** + * {@inheritdoc} + */ + public function executeList(array $ids): void { $this->execute($ids); } - public function executeRow($id) + /** + * {@inheritdoc} + */ + public function executeRow($id): void { $this->execute([$id]); } diff --git a/Service/Product/BatchQueueProcessor.php b/Service/Product/BatchQueueProcessor.php index 15462e5d3..c84a7d47d 100644 --- a/Service/Product/BatchQueueProcessor.php +++ b/Service/Product/BatchQueueProcessor.php @@ -14,6 +14,7 @@ use Algolia\AlgoliaSearch\Model\Queue; use Algolia\AlgoliaSearch\Service\AlgoliaCredentialsManager; use Algolia\AlgoliaSearch\Service\Product\IndexBuilder as ProductIndexBuilder; +use Magento\Catalog\Model\ResourceModel\Product\Collection; use Magento\Framework\Exception\NoSuchEntityException; class BatchQueueProcessor implements BatchQueueProcessorInterface @@ -39,53 +40,89 @@ public function __construct( */ public function processBatch(int $storeId, ?array $entityIds = null): void { - if ($this->dataHelper->isIndexingEnabled($storeId) === false) { + if (!$this->dataHelper->isIndexingEnabled($storeId)) { return; } if (!$this->algoliaCredentialsManager->checkCredentialsWithSearchOnlyAPIKey($storeId)) { $this->algoliaCredentialsManager->displayErrorMessage(self::class, $storeId); - return; } - if ($entityIds && !$this->areParentsLoaded) { - $entityIds = array_unique(array_merge($entityIds, $this->productHelper->getParentProductIds($entityIds))); - $this->areParentsLoaded = true; - } - $productsPerPage = $this->configHelper->getNumberOfElementByPage(); - if (is_array($entityIds) && count($entityIds) > 0) { - foreach (array_chunk($entityIds, $productsPerPage) as $chunk) { - /** @uses ProductIndexBuilder::buildIndexList() */ - $this->queue->addToQueue( - ProductIndexBuilder::class, - 'buildIndexList', - ['storeId' => $storeId, 'entityIds' => $chunk], - count($chunk) - ); - } - + if (!empty($entityIds)) { + $this->handleEntityIds($entityIds, $storeId, $productsPerPage); return; } $useTmpIndex = $this->configHelper->isQueueActive($storeId); - $onlyVisible = !$this->configHelper->includeNonVisibleProductsInIndex(); - $collection = $this->productHelper->getProductCollectionQuery($storeId, $entityIds, $onlyVisible); + $this->syncAlgoliaSettings($storeId, $useTmpIndex); - $timerName = __METHOD__ . ' (Get product collection size)'; - $this->diag->startProfiling($timerName); - $size = $collection->getSize(); - $this->diag->stopProfiling($timerName); + $this->handleFullIndex($storeId, $productsPerPage, $useTmpIndex); + + if ($useTmpIndex) { + $this->moveTempIndex($storeId); + } + } - $pages = ceil($size / $productsPerPage); + /** + * @throws DiagnosticsException + */ + protected function getCollectionSize(Collection $collection): int + { + $this->diag->startProfiling(__METHOD__); + $size = $collection->getSize(); + $this->diag->stopProfiling(__METHOD__); + return $size; + } + protected function syncAlgoliaSettings(int $storeId, bool $useTmpIndex): void + { /** @uses IndicesConfigurator::saveConfigurationToAlgolia() */ $this->queue->addToQueue(IndicesConfigurator::class, 'saveConfigurationToAlgolia', [ 'storeId' => $storeId, 'useTmpIndex' => $useTmpIndex, ], 1, true); + } + + protected function moveTempIndex(int $storeId): void { + /** @uses IndexMover::moveIndexWithSetSettings() */ + $this->queue->addToQueue(IndexMover::class, 'moveIndexWithSetSettings', [ + 'tmpIndexName' => $this->productHelper->getTempIndexName($storeId), + 'indexName' => $this->productHelper->getIndexName($storeId), + 'storeId' => $storeId, + ], 1, true); + } + + protected function handleEntityIds(array $entityIds, int $storeId, int $productsPerPage): void + { + // TODO: Reassess this member bool + if (!$this->areParentsLoaded) { + $entityIds = array_unique(array_merge($entityIds, $this->productHelper->getParentProductIds($entityIds))); + $this->areParentsLoaded = true; + } + + foreach (array_chunk($entityIds, $productsPerPage) as $chunk) { + /** @uses ProductIndexBuilder::buildIndexList() */ + $this->queue->addToQueue( + ProductIndexBuilder::class, + 'buildIndexList', + ['storeId' => $storeId, 'entityIds' => $chunk], + count($chunk) + ); + } + } + + /** + * @throws DiagnosticsException + */ + protected function handleFullIndex(int $storeId, int $productsPerPage, bool $useTmpIndex): void + { + $entityIds = []; // unused in full reindex + $onlyVisible = !$this->configHelper->includeNonVisibleProductsInIndex(); + $collection = $this->productHelper->getProductCollectionQuery($storeId, $entityIds, $onlyVisible); + $pages = ceil($this->getCollectionSize($collection) / $productsPerPage); for ($i = 1; $i <= $pages; $i++) { $data = [ 'storeId' => $storeId, @@ -106,15 +143,6 @@ public function processBatch(int $storeId, ?array $entityIds = null): void true ); } - - if ($useTmpIndex) { - /** @uses IndexMover::moveIndexWithSetSettings() */ - $this->queue->addToQueue(IndexMover::class, 'moveIndexWithSetSettings', [ - 'tmpIndexName' => $this->productHelper->getTempIndexName($storeId), - 'indexName' => $this->productHelper->getIndexName($storeId), - 'storeId' => $storeId, - ], 1, true); - } } /** From 7dfae16105cdc587d92a6c004824e3e49fcf770d Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Thu, 5 Jun 2025 21:04:04 -0400 Subject: [PATCH 003/119] MAGE-1083 Eliminate collection cloning and redundant batching on delta --- Service/Product/BatchQueueProcessor.php | 4 +- Service/Product/IndexBuilder.php | 114 ++++++------------------ 2 files changed, 30 insertions(+), 88 deletions(-) diff --git a/Service/Product/BatchQueueProcessor.php b/Service/Product/BatchQueueProcessor.php index c84a7d47d..c9a5a2b61 100644 --- a/Service/Product/BatchQueueProcessor.php +++ b/Service/Product/BatchQueueProcessor.php @@ -52,7 +52,7 @@ public function processBatch(int $storeId, ?array $entityIds = null): void $productsPerPage = $this->configHelper->getNumberOfElementByPage(); if (!empty($entityIds)) { - $this->handleEntityIds($entityIds, $storeId, $productsPerPage); + $this->handleDeltaIndex($entityIds, $storeId, $productsPerPage); return; } @@ -95,7 +95,7 @@ protected function moveTempIndex(int $storeId): void { ], 1, true); } - protected function handleEntityIds(array $entityIds, int $storeId, int $productsPerPage): void + protected function handleDeltaIndex(array $entityIds, int $storeId, int $productsPerPage): void { // TODO: Reassess this member bool if (!$this->areParentsLoaded) { diff --git a/Service/Product/IndexBuilder.php b/Service/Product/IndexBuilder.php index 2176b440a..34b487746 100644 --- a/Service/Product/IndexBuilder.php +++ b/Service/Product/IndexBuilder.php @@ -3,6 +3,7 @@ namespace Algolia\AlgoliaSearch\Service\Product; use Algolia\AlgoliaSearch\Api\Builder\UpdatableIndexBuilderInterface; +use Algolia\AlgoliaSearch\Exception\DiagnosticsException; use Algolia\AlgoliaSearch\Exception\ProductReindexingException; use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; use Algolia\AlgoliaSearch\Helper\ConfigHelper; @@ -86,71 +87,23 @@ public function buildIndexList(int $storeId, array $entityIds = null, array $opt */ public function buildIndex(int $storeId, ?array $entityIds, ?array $options): void { - if ($this->isIndexingEnabled($storeId) === false) { + if (!$this->isIndexingEnabled($storeId)) { return; } - if (!is_null($entityIds)) { - $this->rebuildEntityIds($storeId, $entityIds); - return; - } + $this->startEmulation($storeId); $onlyVisible = !$this->configHelper->includeNonVisibleProductsInIndex($storeId); - $collection = $this->productHelper->getProductCollectionQuery($storeId, null, $onlyVisible); + $collection = $this->productHelper->getProductCollectionQuery($storeId, $entityIds, $onlyVisible); + $this->buildIndexPage( $storeId, $collection, - $options['page'], - $options['pageSize'], - null, - $options['entityIds'], - $options['useTmpIndex'] + $options['page'] ?? 1, + $options['pageSize'] ?? $this->configHelper->getNumberOfElementByPage(), + $entityIds ); - } - /** - * @param int $storeId - * @param string[] $productIds - * @return void - * @throws \Exception - */ - protected function rebuildEntityIds(int $storeId, array $productIds): void - { - $this->startEmulation($storeId); - $this->logger->start('Indexing'); - try { - $onlyVisible = !$this->configHelper->includeNonVisibleProductsInIndex($storeId); - $collection = $this->productHelper->getProductCollectionQuery($storeId, $productIds, $onlyVisible); - $timerName = __METHOD__ . ' (Get product collection size)'; - $this->logger->startProfiling($timerName); - $size = $collection->getSize(); - $this->logger->stopProfiling($timerName); - - if (!empty($productIds)) { - $size = max(count($productIds), $size); - } - $this->logger->log('Store ' . $this->logger->getStoreName($storeId) . ' collection size : ' . $size); - if ($size > 0) { - $pages = ceil($size / $this->configHelper->getNumberOfElementByPage()); - $collection->clear(); - $page = 1; - while ($page <= $pages) { - $this->buildIndexPage( - $storeId, - $collection, - $page, - $this->configHelper->getNumberOfElementByPage(), - null, - $productIds - ); - $page++; - } - } - } catch (\Exception $e) { - $this->stopEmulation(); - throw $e; - } - $this->logger->stop('Indexing'); $this->stopEmulation(); } @@ -186,42 +139,39 @@ public function deleteInactiveProducts($storeId): void } /** - * @param $storeId - * @param $collectionDefault - * @param $page - * @param $pageSize - * @param $emulationInfo - * @param $productIds - * @param $useTmpIndex + * @param int $storeId + * @param Collection $collection - collection to be paged + * @param int $page + * @param int $pageSize + * @param array|null $productIds - pre-batched product ids - if specified no paging will be applied * @return void - * @throws \Exception + * @throws AlgoliaException + * @throws DiagnosticsException + * @throws NoSuchEntityException */ protected function buildIndexPage( - $storeId, - $collectionDefault, - $page, - $pageSize, - $emulationInfo = null, - $productIds = null, - $useTmpIndex = false + int $storeId, + Collection $collection, + int $page, + int $pageSize, + ?array $productIds = null ): void { if ($this->isIndexingEnabled($storeId) === false) { return; } - $wrapperLogMessage = 'rebuildStoreProductIndexPage: ' . $this->logger->getStoreName($storeId) . ', + $wrapperLogMessage = 'Build products index page: ' . $this->logger->getStoreName($storeId) . ', page ' . $page . ', pageSize ' . $pageSize; $this->logger->start($wrapperLogMessage, true); - if ($emulationInfo === null) { - $this->startEmulation($storeId); - } + $additionalAttributes = $this->configHelper->getProductAdditionalAttributes($storeId); - /** @var Collection $collection */ - $collection = clone $collectionDefault; - $collection->setCurPage($page)->setPageSize($pageSize); + if (empty($productIds)) { + $collection->setCurPage($page)->setPageSize($pageSize); + } + $collection->addCategoryIds(); $collection->addUrlRewrite(); @@ -248,7 +198,7 @@ protected function buildIndexPage( collection page: ' . $page . ', pageSize: ' . $pageSize; $this->logger->start($logMessage); - $collection->load(); + $collection->load(); // eliminate extra query to obtain count $this->logger->log('Loaded ' . count($collection) . ' products'); $this->logger->stop($logMessage); $indexOptions = $this->indexOptionsBuilder->buildEntityIndexOptions($storeId); @@ -269,14 +219,6 @@ protected function buildIndexPage( $this->logger->stop('REMOVE FROM ALGOLIA'); } } - unset($indexData); - $collection->walk('clearInstance'); - $collection->clear(); - unset($collection); - if ($emulationInfo === null) { - $this->stopEmulation(); - } - $this->logger->stop($wrapperLogMessage, true); } From e2ba8ebedca1201540f989b79886d4a4534133af Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Fri, 6 Jun 2025 15:10:47 -0400 Subject: [PATCH 004/119] MAGE-1083 Fix dependency injection --- Console/Command/Indexer/IndexProductsCommand.php | 4 ++-- etc/di.xml | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Console/Command/Indexer/IndexProductsCommand.php b/Console/Command/Indexer/IndexProductsCommand.php index 65c69089b..660dc70bb 100644 --- a/Console/Command/Indexer/IndexProductsCommand.php +++ b/Console/Command/Indexer/IndexProductsCommand.php @@ -2,8 +2,8 @@ namespace Algolia\AlgoliaSearch\Console\Command\Indexer; +use Algolia\AlgoliaSearch\Api\Processor\BatchQueueProcessorInterface; use Algolia\AlgoliaSearch\Service\StoreNameFetcher; -use Algolia\AlgoliaSearch\Service\Product\BatchQueueProcessor as ProductBatchQueueProcessor; use Magento\Framework\App\State; use Magento\Framework\Console\Cli; use Magento\Store\Model\StoreManagerInterface; @@ -13,7 +13,7 @@ class IndexProductsCommand extends AbstractIndexerCommand { public function __construct( - protected ProductBatchQueueProcessor $productBatchQueueProcessor, + protected BatchQueueProcessorInterface $productBatchQueueProcessor, protected StoreManagerInterface $storeManager, State $state, StoreNameFetcher $storeNameFetcher, diff --git a/etc/di.xml b/etc/di.xml index c494e3115..07a3431d7 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -22,22 +22,24 @@ - + - - - - + - + + + Algolia\AlgoliaSearch\Service\Product\BatchQueueProcessor + + + From ca71d79fdf16764a3ed596bcea6bb45faaf7c858 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Fri, 6 Jun 2025 15:21:53 -0400 Subject: [PATCH 005/119] MAGE-1083 Remove member bool to fix issue with subsequent store index operations dropping parent products --- Service/Product/BatchQueueProcessor.php | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Service/Product/BatchQueueProcessor.php b/Service/Product/BatchQueueProcessor.php index c9a5a2b61..ed06a9543 100644 --- a/Service/Product/BatchQueueProcessor.php +++ b/Service/Product/BatchQueueProcessor.php @@ -19,8 +19,6 @@ class BatchQueueProcessor implements BatchQueueProcessorInterface { - protected bool $areParentsLoaded = false; - public function __construct( protected Data $dataHelper, protected ConfigHelper $configHelper, @@ -97,11 +95,7 @@ protected function moveTempIndex(int $storeId): void { protected function handleDeltaIndex(array $entityIds, int $storeId, int $productsPerPage): void { - // TODO: Reassess this member bool - if (!$this->areParentsLoaded) { - $entityIds = array_unique(array_merge($entityIds, $this->productHelper->getParentProductIds($entityIds))); - $this->areParentsLoaded = true; - } + $entityIds = array_unique(array_merge($entityIds, $this->productHelper->getParentProductIds($entityIds))); foreach (array_chunk($entityIds, $productsPerPage) as $chunk) { /** @uses ProductIndexBuilder::buildIndexList() */ From 633eb17855789ca930786cd9a9ba37ccf48b37eb Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Fri, 6 Jun 2025 15:57:06 -0400 Subject: [PATCH 006/119] MAGE-1083 Remove redundant entityIds option and save paging info on delta --- Service/Product/BatchQueueProcessor.php | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Service/Product/BatchQueueProcessor.php b/Service/Product/BatchQueueProcessor.php index ed06a9543..c17849564 100644 --- a/Service/Product/BatchQueueProcessor.php +++ b/Service/Product/BatchQueueProcessor.php @@ -97,12 +97,19 @@ protected function handleDeltaIndex(array $entityIds, int $storeId, int $product { $entityIds = array_unique(array_merge($entityIds, $this->productHelper->getParentProductIds($entityIds))); - foreach (array_chunk($entityIds, $productsPerPage) as $chunk) { + foreach (array_chunk($entityIds, $productsPerPage) as $i => $chunk) { /** @uses ProductIndexBuilder::buildIndexList() */ $this->queue->addToQueue( ProductIndexBuilder::class, 'buildIndexList', - ['storeId' => $storeId, 'entityIds' => $chunk], + [ + 'storeId' => $storeId, + 'entityIds' => $chunk, + 'options' => [ + 'page' => $i + 1, + 'pageSize' => $productsPerPage, + ] + ], count($chunk) ); } @@ -113,17 +120,15 @@ protected function handleDeltaIndex(array $entityIds, int $storeId, int $product */ protected function handleFullIndex(int $storeId, int $productsPerPage, bool $useTmpIndex): void { - $entityIds = []; // unused in full reindex $onlyVisible = !$this->configHelper->includeNonVisibleProductsInIndex(); - $collection = $this->productHelper->getProductCollectionQuery($storeId, $entityIds, $onlyVisible); + $collection = $this->productHelper->getProductCollectionQuery($storeId, [], $onlyVisible); $pages = ceil($this->getCollectionSize($collection) / $productsPerPage); for ($i = 1; $i <= $pages; $i++) { $data = [ 'storeId' => $storeId, 'options' => [ - 'entityIds' => $entityIds, - 'page' => $i, - 'pageSize' => $productsPerPage, + 'page' => $i, + 'pageSize' => $productsPerPage, 'useTmpIndex' => $useTmpIndex, ] ]; From dcae84631067af615529abb2439a820c46e546f7 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Mon, 9 Jun 2025 08:31:25 -0400 Subject: [PATCH 007/119] MAGE-1083 Add query count cache for full reindex --- Model/Cache/IndexCollectionSize.php | 61 +++++++++++++++++++++++++ Model/Cache/Type/Indexer.php | 19 ++++++++ Service/Product/BatchQueueProcessor.php | 14 ++++-- etc/cache.xml | 6 +++ 4 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 Model/Cache/IndexCollectionSize.php create mode 100644 Model/Cache/Type/Indexer.php create mode 100644 etc/cache.xml diff --git a/Model/Cache/IndexCollectionSize.php b/Model/Cache/IndexCollectionSize.php new file mode 100644 index 000000000..2297f6289 --- /dev/null +++ b/Model/Cache/IndexCollectionSize.php @@ -0,0 +1,61 @@ +cache->load($this->getCacheKey($storeId)); + if ($data === false) { + return self::NOT_FOUND; + } + return (int) $data; + } + + public function set(int $storeId, int $value, ?int $ttl = null): void + { + $this->cache->save($value, $this->getCacheKey($storeId), [Indexer::CACHE_TAG], $ttl); + } + + protected function remove(int $storeId): void + { + $this->cache->remove($this->getCacheKey($storeId)); + } + + public function isCacheAvailable(): bool + { + return $this->state->isEnabled(Indexer::TYPE_IDENTIFIER) + && !array_key_exists(Indexer::TYPE_IDENTIFIER, $this->typeList->getInvalidated()); + } + + protected function getCacheKey(int $storeId): string + { + return sprintf('%s_%d', Indexer::TYPE_IDENTIFIER, $storeId); + } + + public function clear(?int $storeId = null): void + { + if (is_null($storeId)) { + $this->typeList->invalidate(Indexer::TYPE_IDENTIFIER); + $this->typeList->cleanType(Indexer::TYPE_IDENTIFIER); + } + else { + $this->remove($storeId); + } + } +} diff --git a/Model/Cache/Type/Indexer.php b/Model/Cache/Type/Indexer.php new file mode 100644 index 000000000..dc434a66a --- /dev/null +++ b/Model/Cache/Type/Indexer.php @@ -0,0 +1,19 @@ +get(self::TYPE_IDENTIFIER), + self::CACHE_TAG + ); + } +} diff --git a/Service/Product/BatchQueueProcessor.php b/Service/Product/BatchQueueProcessor.php index c17849564..b9b576396 100644 --- a/Service/Product/BatchQueueProcessor.php +++ b/Service/Product/BatchQueueProcessor.php @@ -9,6 +9,7 @@ use Algolia\AlgoliaSearch\Helper\Data; use Algolia\AlgoliaSearch\Helper\Entity\ProductHelper; use Algolia\AlgoliaSearch\Logger\DiagnosticsLogger; +use Algolia\AlgoliaSearch\Model\Cache\IndexCollectionSize; use Algolia\AlgoliaSearch\Model\IndexMover; use Algolia\AlgoliaSearch\Model\IndicesConfigurator; use Algolia\AlgoliaSearch\Model\Queue; @@ -26,7 +27,8 @@ public function __construct( protected Queue $queue, protected DiagnosticsLogger $diag, protected AlgoliaCredentialsManager $algoliaCredentialsManager, - protected ProductIndexBuilder $productIndexBuilder + protected ProductIndexBuilder $productIndexBuilder, + protected IndexCollectionSize $indexCollectionSizeCache ){} /** @@ -67,10 +69,14 @@ public function processBatch(int $storeId, ?array $entityIds = null): void /** * @throws DiagnosticsException */ - protected function getCollectionSize(Collection $collection): int + protected function getCollectionSize(int $storeId, Collection $collection): int { $this->diag->startProfiling(__METHOD__); - $size = $collection->getSize(); + $size = $this->indexCollectionSizeCache->get($storeId); + if ($size === IndexCollectionSize::NOT_FOUND) { + $size = $collection->getSize(); + $this->indexCollectionSizeCache->set($storeId, $size); + } $this->diag->stopProfiling(__METHOD__); return $size; } @@ -122,7 +128,7 @@ protected function handleFullIndex(int $storeId, int $productsPerPage, bool $use { $onlyVisible = !$this->configHelper->includeNonVisibleProductsInIndex(); $collection = $this->productHelper->getProductCollectionQuery($storeId, [], $onlyVisible); - $pages = ceil($this->getCollectionSize($collection) / $productsPerPage); + $pages = ceil($this->getCollectionSize($storeId, $collection) / $productsPerPage); for ($i = 1; $i <= $pages; $i++) { $data = [ 'storeId' => $storeId, diff --git a/etc/cache.xml b/etc/cache.xml new file mode 100644 index 000000000..d52341e73 --- /dev/null +++ b/etc/cache.xml @@ -0,0 +1,6 @@ + + + + Algolia product indexing cache + + From f329074313bfce083365358255f707205b9492ea Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Mon, 9 Jun 2025 22:49:05 -0400 Subject: [PATCH 008/119] MAGE-1083 Add plugin to clear cache on product catalog updates --- Plugin/Cache/CacheCleanProductPlugin.php | 113 +++++++++++++++++++++++ Service/Product/BatchQueueProcessor.php | 3 + etc/di.xml | 1 + 3 files changed, 117 insertions(+) create mode 100644 Plugin/Cache/CacheCleanProductPlugin.php diff --git a/Plugin/Cache/CacheCleanProductPlugin.php b/Plugin/Cache/CacheCleanProductPlugin.php new file mode 100644 index 000000000..ebdd15fb7 --- /dev/null +++ b/Plugin/Cache/CacheCleanProductPlugin.php @@ -0,0 +1,113 @@ +originalData[$product->getSku()] = $product->getOrigData(); + } + + public function afterSave(ProductResource $subject, ProductResource $result, Product $product): ProductResource + { + $original = $this->originalData[$product->getSku()] ?? []; + $storeId = $product->getStoreId(); + + $shouldClearCache = + $this->isEligibleNewProduct($product) + || $this->hasEnablementChanged($original, $product->getData()) + || $this->hasVisibilityChanged($original, $product->getData(), $storeId) + || $this->hasStockChanged($original, $product->getData(), $storeId); + + if ($shouldClearCache) { + $this->cache->clear($storeId ?: null); + } + + return $result; + } + + public function afterDelete(ProductResource $subject, ProductResource $result): ProductResource + { + $this->cache->clear(); + return $result; + } + + protected function isEligibleNewProduct(Product $product): bool + { + $storeId = $product->getStoreId(); + return $product->isObjectNew() + && $product->getStatus() === Status::STATUS_ENABLED + && $this->configHelper->includeNonVisibleProductsInIndex($storeId) + || $product->isVisibleInSiteVisibility() + && $this->configHelper->getShowOutOfStock($storeId) + || $product->isInStock(); + } + + protected function hasEnablementChanged(array $orig, array $new): bool + { + $key = 'status'; + return $orig[$key] !== $new[$key]; + } + + protected function hasVisibilityChanged(array $orig, array $new, int $storeId = null): bool + { + if ($this->configHelper->includeNonVisibleProductsInIndex($storeId)) { + return false; + } + + $key = 'visibility'; + return $this->isVisible($orig[$key]) !== $this->isVisible($new[$key]); + } + + /** + * Do not rely on this data point only + * TODO revaluate with MSI support + */ + protected function hasStockChanged(array $orig, array $new, int $storeId): bool + { + if ($this->configHelper->getShowOutOfStock($storeId)) { + return false; + } + + $key = 'quantity_and_stock_status'; + $oldStock = $orig[$key]; + $newStock = $new[$key]; + return $this->canCompareValues($oldStock, $newStock, 'is_in_stock') + && (bool) $oldStock['is_in_stock'] !== (bool) $newStock['is_in_stock'] + || $this->canCompareValues($oldStock, $newStock, 'qty') + && $this->hasStock($oldStock['qty']) !== $this->hasStock($newStock['qty']); + } + + protected function canCompareValues(array $orig, array $new, string $key): bool + { + return array_key_exists($key, $orig) && array_key_exists($key, $new); + } + + protected function isVisible(int $visibility): bool + { + return $visibility !== Visibility::VISIBILITY_NOT_VISIBLE; + } + + /* + * Reduce numeric to comparable boolean + */ + protected function hasStock(int $qty): bool + { + return $qty > 0; + } +} diff --git a/Service/Product/BatchQueueProcessor.php b/Service/Product/BatchQueueProcessor.php index b9b576396..a77b5f014 100644 --- a/Service/Product/BatchQueueProcessor.php +++ b/Service/Product/BatchQueueProcessor.php @@ -90,6 +90,9 @@ protected function syncAlgoliaSettings(int $storeId, bool $useTmpIndex): void ], 1, true); } + /** + * @throws NoSuchEntityException + */ protected function moveTempIndex(int $storeId): void { /** @uses IndexMover::moveIndexWithSetSettings() */ $this->queue->addToQueue(IndexMover::class, 'moveIndexWithSetSettings', [ diff --git a/etc/di.xml b/etc/di.xml index 07a3431d7..3be4d70a9 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -2,6 +2,7 @@ + From f27625e9028cb875ec5f67da5282d301e9e14095 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Tue, 10 Jun 2025 09:04:18 -0400 Subject: [PATCH 009/119] MAGE-1083 Add unit tests for BatchQueueProcessor --- .../Product/BatchQueueProcessorTest.php | 281 ++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 Test/Unit/Service/Product/BatchQueueProcessorTest.php diff --git a/Test/Unit/Service/Product/BatchQueueProcessorTest.php b/Test/Unit/Service/Product/BatchQueueProcessorTest.php new file mode 100644 index 000000000..5ea64e8c3 --- /dev/null +++ b/Test/Unit/Service/Product/BatchQueueProcessorTest.php @@ -0,0 +1,281 @@ +dataHelper = $this->createMock(Data::class); + $this->configHelper = $this->createMock(ConfigHelper::class); + $this->productHelper = $this->createMock(ProductHelper::class); + $this->queue = $this->createMock(Queue::class); + $this->diag = $this->createMock(DiagnosticsLogger::class); + $this->algoliaCredentialsManager = $this->createMock(AlgoliaCredentialsManager::class); + $this->indexBuilder = $this->createMock(IndexBuilder::class); + $this->indexCollectionSizeCache = $this->createMock(IndexCollectionSize::class); + + $this->processor = new BatchQueueProcessor( + $this->dataHelper, + $this->configHelper, + $this->productHelper, + $this->queue, + $this->diag, + $this->algoliaCredentialsManager, + $this->indexBuilder, + $this->indexCollectionSizeCache + ); + } + + /** + * @throws DiagnosticsException + * @throws NoSuchEntityException + */ + public function testProcessBatchSkipsWhenIndexingDisabled() + { + $this->dataHelper->method('isIndexingEnabled')->willReturn(false); + + $this->algoliaCredentialsManager->expects($this->never())->method('checkCredentialsWithSearchOnlyAPIKey'); + + $this->processor->processBatch(1); + } + + /** + * @throws DiagnosticsException + * @throws NoSuchEntityException + */ + public function testProcessBatchSkipsWhenCredentialsInvalid() + { + $this->dataHelper->method('isIndexingEnabled')->willReturn(true); + $this->algoliaCredentialsManager->method('checkCredentialsWithSearchOnlyAPIKey')->willReturn(false); + + $this->algoliaCredentialsManager->expects($this->once()) + ->method('displayErrorMessage') + ->with(BatchQueueProcessor::class, 1); + + $this->processor->processBatch(1); + } + + /** + * @throws NoSuchEntityException + * @throws DiagnosticsException + */ + public function testProcessBatchHandlesDeltaIndexing() + { + $this->setupBasicIndexingConfig(10); + $this->productHelper->method('getParentProductIds')->willReturn([]); + + $this->queue->expects($this->once()) + ->method('addToQueue') + ->with( + IndexBuilder::class, + 'buildIndexList', + $this->arrayHasKey('entityIds') + ); + + $this->processor->processBatch(1, range(1,5)); + } + + public function testProcessBatchHandlesDeltaIndexingPaged() + { + $pageSize = 10; + $this->setupBasicIndexingConfig($pageSize); + $this->productHelper->method('getParentProductIds')->willReturn([]); + + $invocations = $this->exactly(5); + $this->queue->expects($invocations) + ->method('addToQueue') + ->with( + IndexBuilder::class, + 'buildIndexList', + $this->callback(function(array $arg) use ($invocations, $pageSize) { + return array_key_exists('storeId', $arg) + && array_key_exists('entityIds', $arg) + && array_key_exists('options', $arg) + && $arg['options']['pageSize'] === $pageSize + && $arg['options']['page'] === $invocations->getInvocationCount(); + }) + ); + + $this->processor->processBatch(1, range(1,50)); + } + + /** + * @throws DiagnosticsException + * @throws NoSuchEntityException + */ + public function testProcessBatchHandlesFullIndexing() + { + $this->setupBasicIndexingConfig(10); + $this->configHelper->method('isQueueActive')->willReturn(false); + $this->indexCollectionSizeCache->expects($this->once())->method('get')->willReturn(10); + $this->productHelper->method('getProductCollectionQuery')->willReturn($this->getMockCollection()); + + $invocations = $this->exactly(2); + $this->queue->expects($invocations) + ->method('addToQueue') + ->willReturnCallback( + function( + string $className, + string $method, + array $data, + int $dataSize, + bool $isFullReindex) + use ($invocations) { + switch ($invocations->getInvocationCount()) { + case 1: + $this->assertEquals(IndicesConfigurator::class, $className); + $this->assertEquals('saveConfigurationToAlgolia', $method); + $this->assertArrayHasKey('storeId', $data); + break; + case 2: + $this->assertEquals(IndexBuilder::class, $className); + $this->assertEquals('buildIndexFull', $method); + $this->assertArrayHasKey('storeId', $data); + break; + } + } + ); + + $this->processor->processBatch(1); + } + + /** + * @throws DiagnosticsException + * @throws NoSuchEntityException + */ + public function testProcessBatchFullIndexingWithNoCache() + { + $this->setupBasicIndexingConfig(10); + $this->configHelper->method('isQueueActive')->willReturn(false); + $this->indexCollectionSizeCache->expects($this->once())->method('get')->willReturn(IndexCollectionSize::NOT_FOUND); + $this->productHelper->method('getProductCollectionQuery')->willReturn($this->getMockCollection(10, 1)); + + $this->queue->expects($this->exactly(2)) + ->method('addToQueue'); + + $this->processor->processBatch(1); + } + + /** + * @throws DiagnosticsException + * @throws NoSuchEntityException + */ + public function testProcessBatchHandlesFullIndexingPaged() + { + $pageSize = 10; + $this->setupBasicIndexingConfig($pageSize); + $this->configHelper->method('isQueueActive')->willReturn(false); + $this->indexCollectionSizeCache->expects($this->once())->method('get')->willReturn(50); + $this->productHelper->method('getProductCollectionQuery')->willReturn($this->getMockCollection()); + + $invocations = $this->exactly(6); + $this->queue->expects($invocations) + ->method('addToQueue') + ->willReturnCallback( + function( + string $className, + string $method, + array $data, + int $dataSize, + bool $isFullReindex) + use ($invocations, $pageSize) { + $invocation = $invocations->getInvocationCount(); + switch ($invocation) { + case 1: + $this->assertEquals(IndicesConfigurator::class, $className); + $this->assertEquals('saveConfigurationToAlgolia', $method); + break; + default: + $this->assertEquals(IndexBuilder::class, $className); + $this->assertEquals('buildIndexFull', $method); + $this->assertArrayHasKey('options', $data); + $this->assertEquals($pageSize, $data['options']['pageSize']); + $this->assertEquals($invocation - 1, $data['options']['page']); + break; + } + } + ); + + $this->processor->processBatch(1); + } + + /** + * @throws DiagnosticsException + * @throws NoSuchEntityException + */ + public function testProcessBatchMovesTempIndexIfQueueActive() + { + $this->setupBasicIndexingConfig(10); + $this->configHelper->method('isQueueActive')->willReturn(true); + $this->indexCollectionSizeCache->expects($this->once())->method('get')->willReturn(10); + + $this->productHelper->method('getProductCollectionQuery')->willReturn($this->getMockCollection()); + $this->productHelper->method('getTempIndexName')->willReturn('tmp_index'); + $this->productHelper->method('getIndexName')->willReturn('main_index'); + + $invocations = $this->exactly(3); + $this->queue->expects($invocations) + ->method('addToQueue') + ->willReturnCallback( + function( + string $className, + string $method, + array $data, + int $dataSize, + bool $isFullReindex) + use ($invocations) { + if ($invocations->getInvocationCount() === 3) { + $this->assertEquals(IndexMover::class, $className); + $this->assertEquals('moveIndexWithSetSettings', $method); + $this->assertArrayHasKey('tmpIndexName', $data); + $this->assertArrayHasKey('indexName', $data); + $this->assertArrayHasKey('storeId', $data); + } + } + ); + + $this->processor->processBatch(1); + } + + protected function setupBasicIndexingConfig(int $elementsPerPage): void + { + $this->dataHelper->method('isIndexingEnabled')->willReturn(true); + $this->algoliaCredentialsManager->method('checkCredentialsWithSearchOnlyAPIKey')->willReturn(true); + $this->configHelper->method('getNumberOfElementByPage')->willReturn($elementsPerPage); + $this->configHelper->method('includeNonVisibleProductsInIndex')->willReturn(false); + } + + protected function getMockCollection(int $size = 10, int $expectedSizeCalls = 0): Collection + { + $mockCollection = $this->createMock(Collection::class); + $mockCollection->expects($this->exactly($expectedSizeCalls))->method('getSize')->willReturn($size); + return $mockCollection; + } +} From d0670346a274bc92428cdcaffb48ec433b221643 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Tue, 10 Jun 2025 09:34:33 -0400 Subject: [PATCH 010/119] MAGE-1083 Remove redundant DI type --- etc/di.xml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/etc/di.xml b/etc/di.xml index 3be4d70a9..0cddfda4f 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -48,6 +48,7 @@ Algolia\AlgoliaSearch\Model\ResourceModel\Run\Grid\Collection Algolia\AlgoliaSearch\Model\ResourceModel\LandingPage\Grid\Collection Algolia\AlgoliaSearch\Model\ResourceModel\Query\Grid\Collection + Algolia\AlgoliaSearch\Model\ResourceModel\QueueArchive\Grid\Collection @@ -135,13 +136,6 @@ - - - - Algolia\AlgoliaSearch\Model\ResourceModel\QueueArchive\Grid\Collection - - - algoliasearch_queue_archive From fcd975207ac8e1b1dbd48dec63b4b4125587c648 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Tue, 10 Jun 2025 12:37:52 -0400 Subject: [PATCH 011/119] MAGE-1083 Check cache status before read/write --- Model/Cache/IndexCollectionSize.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Model/Cache/IndexCollectionSize.php b/Model/Cache/IndexCollectionSize.php index 2297f6289..c05d5850e 100644 --- a/Model/Cache/IndexCollectionSize.php +++ b/Model/Cache/IndexCollectionSize.php @@ -19,17 +19,24 @@ public function __construct( public function get(int $storeId): int { + if (!$this->isCacheAvailable()) { + return self::NOT_FOUND; + } + /** @var string|false $data */ $data = $this->cache->load($this->getCacheKey($storeId)); if ($data === false) { return self::NOT_FOUND; } + return (int) $data; } public function set(int $storeId, int $value, ?int $ttl = null): void { - $this->cache->save($value, $this->getCacheKey($storeId), [Indexer::CACHE_TAG], $ttl); + if ($this->isCacheAvailable()) { + $this->cache->save($value, $this->getCacheKey($storeId), [Indexer::CACHE_TAG], $ttl); + } } protected function remove(int $storeId): void From 6080ccc9f79c8e3d8685a9152c67f8d11b973046 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 11 Jun 2025 21:35:37 -0400 Subject: [PATCH 012/119] MAGE-1083 Updates per code review --- Model/Cache/{ => Product}/IndexCollectionSize.php | 2 +- Plugin/Cache/CacheCleanProductPlugin.php | 4 ++-- Service/Product/BatchQueueProcessor.php | 2 +- Test/Unit/Service/Product/BatchQueueProcessorTest.php | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) rename Model/Cache/{ => Product}/IndexCollectionSize.php (97%) diff --git a/Model/Cache/IndexCollectionSize.php b/Model/Cache/Product/IndexCollectionSize.php similarity index 97% rename from Model/Cache/IndexCollectionSize.php rename to Model/Cache/Product/IndexCollectionSize.php index c05d5850e..5c15da0a8 100644 --- a/Model/Cache/IndexCollectionSize.php +++ b/Model/Cache/Product/IndexCollectionSize.php @@ -1,6 +1,6 @@ Date: Thu, 12 Jun 2025 07:55:56 -0400 Subject: [PATCH 013/119] MAGE-1083 Apply scoping per code review --- Model/Cache/Product/IndexCollectionSize.php | 2 +- Service/Product/IndexBuilder.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Model/Cache/Product/IndexCollectionSize.php b/Model/Cache/Product/IndexCollectionSize.php index 5c15da0a8..b4661af67 100644 --- a/Model/Cache/Product/IndexCollectionSize.php +++ b/Model/Cache/Product/IndexCollectionSize.php @@ -52,7 +52,7 @@ public function isCacheAvailable(): bool protected function getCacheKey(int $storeId): string { - return sprintf('%s_%d', Indexer::TYPE_IDENTIFIER, $storeId); + return sprintf('%s_%s_%d', Indexer::TYPE_IDENTIFIER, 'product', $storeId); } public function clear(?int $storeId = null): void diff --git a/Service/Product/IndexBuilder.php b/Service/Product/IndexBuilder.php index 34b487746..a9f2c937a 100644 --- a/Service/Product/IndexBuilder.php +++ b/Service/Product/IndexBuilder.php @@ -100,7 +100,7 @@ public function buildIndex(int $storeId, ?array $entityIds, ?array $options): vo $storeId, $collection, $options['page'] ?? 1, - $options['pageSize'] ?? $this->configHelper->getNumberOfElementByPage(), + $options['pageSize'] ?? $this->configHelper->getNumberOfElementByPage($storeId), $entityIds ); From 9193cd29084c1508b8eb35baa16acdb6d9557e1e Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Thu, 12 Jun 2025 08:15:34 -0400 Subject: [PATCH 014/119] MAGE-1083 Implement observer for mass action status change --- .../Product/CacheCleanAttributeUpdate.php | 34 +++++++++++++++++++ etc/events.xml | 5 +++ 2 files changed, 39 insertions(+) create mode 100644 Model/Observer/Product/CacheCleanAttributeUpdate.php diff --git a/Model/Observer/Product/CacheCleanAttributeUpdate.php b/Model/Observer/Product/CacheCleanAttributeUpdate.php new file mode 100644 index 000000000..d435e95f6 --- /dev/null +++ b/Model/Observer/Product/CacheCleanAttributeUpdate.php @@ -0,0 +1,34 @@ +getData('attributes_data'); + $productIds = $observer->getData('product_ids'); + $attributesToObserve = ['status']; + + if ($productIds + && array_intersect(array_keys($attributes), $attributesToObserve)) { + $this->cache->clear(); + } + } +} + + + + diff --git a/etc/events.xml b/etc/events.xml index e20ffd004..20ca422cd 100644 --- a/etc/events.xml +++ b/etc/events.xml @@ -4,4 +4,9 @@ + + + + + From 3a460d2962792440243519cc2b5ab04dc815c096 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Thu, 12 Jun 2025 09:10:25 -0400 Subject: [PATCH 015/119] MAGE-1083 Implement cache clean for attribute change by mass action --- Helper/Entity/Product/CacheHelper.php | 25 ++++++++++++++ .../Cache/CacheCleanBulkAttributePlugin.php | 33 +++++++++++++++++++ Plugin/Cache/CacheCleanProductPlugin.php | 15 ++++++++- etc/adminhtml/di.xml | 6 ++++ etc/di.xml | 2 ++ 5 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 Helper/Entity/Product/CacheHelper.php create mode 100644 Plugin/Cache/CacheCleanBulkAttributePlugin.php diff --git a/Helper/Entity/Product/CacheHelper.php b/Helper/Entity/Product/CacheHelper.php new file mode 100644 index 000000000..1e30a4897 --- /dev/null +++ b/Helper/Entity/Product/CacheHelper.php @@ -0,0 +1,25 @@ +logger->info(sprintf("Clearing product index collection cache on store ID %d for attributes: %s", $storeId, join(',', array_keys($attributes)))); + $this->cache->clear($storeId ?: null); + } + } +} diff --git a/Plugin/Cache/CacheCleanBulkAttributePlugin.php b/Plugin/Cache/CacheCleanBulkAttributePlugin.php new file mode 100644 index 000000000..322f67b78 --- /dev/null +++ b/Plugin/Cache/CacheCleanBulkAttributePlugin.php @@ -0,0 +1,33 @@ +cacheHelper->handleBulkAttributeChange( + $this->attributeHelper->getProductIds(), + $subject->getRequest()->getParam('attributes', []), + $this->attributeHelper->getSelectedStoreId() + ); + + return $result; + } +} diff --git a/Plugin/Cache/CacheCleanProductPlugin.php b/Plugin/Cache/CacheCleanProductPlugin.php index da11ab6f8..03dab9832 100644 --- a/Plugin/Cache/CacheCleanProductPlugin.php +++ b/Plugin/Cache/CacheCleanProductPlugin.php @@ -3,8 +3,10 @@ namespace Algolia\AlgoliaSearch\Plugin\Cache; use Algolia\AlgoliaSearch\Helper\ConfigHelper; +use Algolia\AlgoliaSearch\Helper\Entity\Product\CacheHelper; use Algolia\AlgoliaSearch\Model\Cache\Product\IndexCollectionSize as Cache; use Magento\Catalog\Model\Product; +use Magento\Catalog\Model\Product\Action; use Magento\Catalog\Model\Product\Attribute\Source\Status; use Magento\Catalog\Model\Product\Visibility; use Magento\Catalog\Model\ResourceModel\Product as ProductResource; @@ -15,7 +17,8 @@ class CacheCleanProductPlugin public function __construct( protected Cache $cache, - protected ConfigHelper $configHelper + protected ConfigHelper $configHelper, + protected CacheHelper $cacheHelper ) { } public function beforeSave(ProductResource $subject, Product $product): void @@ -47,6 +50,16 @@ public function afterDelete(ProductResource $subject, ProductResource $result): return $result; } + /** + * Called on mass action "Change Status" + * Called on "Update attributes" if `product_action_attribute.update` consumer is running + */ + public function afterUpdateAttributes(Action $subject, Action $result, array $productIds, array $attributes, int $storeId): Action + { + $this->cacheHelper->handleBulkAttributeChange($productIds, $attributes, $storeId); + return $result; + } + protected function isEligibleNewProduct(Product $product): bool { $storeId = $product->getStoreId(); diff --git a/etc/adminhtml/di.xml b/etc/adminhtml/di.xml index 60e6a16a8..21acfa5d2 100755 --- a/etc/adminhtml/di.xml +++ b/etc/adminhtml/di.xml @@ -13,4 +13,10 @@ Magento\Framework\Url + + + diff --git a/etc/di.xml b/etc/di.xml index 0cddfda4f..efafc7619 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -6,7 +6,9 @@ + + From 68e2d41519c77f28967378dff90e2139a77e8432 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Thu, 12 Jun 2025 09:10:48 -0400 Subject: [PATCH 016/119] MAGE-1083 Remove redundant observer --- .../Product/CacheCleanAttributeUpdate.php | 34 ------------------- etc/events.xml | 5 --- 2 files changed, 39 deletions(-) delete mode 100644 Model/Observer/Product/CacheCleanAttributeUpdate.php diff --git a/Model/Observer/Product/CacheCleanAttributeUpdate.php b/Model/Observer/Product/CacheCleanAttributeUpdate.php deleted file mode 100644 index d435e95f6..000000000 --- a/Model/Observer/Product/CacheCleanAttributeUpdate.php +++ /dev/null @@ -1,34 +0,0 @@ -getData('attributes_data'); - $productIds = $observer->getData('product_ids'); - $attributesToObserve = ['status']; - - if ($productIds - && array_intersect(array_keys($attributes), $attributesToObserve)) { - $this->cache->clear(); - } - } -} - - - - diff --git a/etc/events.xml b/etc/events.xml index 20ca422cd..e20ffd004 100644 --- a/etc/events.xml +++ b/etc/events.xml @@ -4,9 +4,4 @@ - - - - - From 6dbe9bf8cba6d86e0014ba4c021d9666998f2386 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Thu, 26 Jun 2025 15:08:42 -0400 Subject: [PATCH 017/119] MAGE-1341 Fix dropped products issue on indexing queue --- Service/Product/IndexBuilder.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Service/Product/IndexBuilder.php b/Service/Product/IndexBuilder.php index a9f2c937a..f438ceba6 100644 --- a/Service/Product/IndexBuilder.php +++ b/Service/Product/IndexBuilder.php @@ -201,7 +201,7 @@ protected function buildIndexPage( $collection->load(); // eliminate extra query to obtain count $this->logger->log('Loaded ' . count($collection) . ' products'); $this->logger->stop($logMessage); - $indexOptions = $this->indexOptionsBuilder->buildEntityIndexOptions($storeId); + $indexOptions = $this->indexOptionsBuilder->buildEntityIndexOptions($storeId, $useTmpIndex); $indexData = $this->getProductsRecords($storeId, $collection, $productIds); if (!empty($indexData['toIndex'])) { $this->logger->start('ADD/UPDATE TO ALGOLIA'); From c47409fd52d72534877403247ad9f601c02f9dd9 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Tue, 1 Jul 2025 15:55:48 +0200 Subject: [PATCH 018/119] MAGE-1357: Re-add missing boolean --- Service/Product/IndexBuilder.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Service/Product/IndexBuilder.php b/Service/Product/IndexBuilder.php index f438ceba6..f980791f3 100644 --- a/Service/Product/IndexBuilder.php +++ b/Service/Product/IndexBuilder.php @@ -101,7 +101,8 @@ public function buildIndex(int $storeId, ?array $entityIds, ?array $options): vo $collection, $options['page'] ?? 1, $options['pageSize'] ?? $this->configHelper->getNumberOfElementByPage($storeId), - $entityIds + $entityIds, + $options['useTmpIndex'] ); $this->stopEmulation(); @@ -144,6 +145,7 @@ public function deleteInactiveProducts($storeId): void * @param int $page * @param int $pageSize * @param array|null $productIds - pre-batched product ids - if specified no paging will be applied + * @param bool|null $useTmpIndex * @return void * @throws AlgoliaException * @throws DiagnosticsException @@ -154,7 +156,8 @@ protected function buildIndexPage( Collection $collection, int $page, int $pageSize, - ?array $productIds = null + ?array $productIds = null, + ?bool $useTmpIndex = false ): void { if ($this->isIndexingEnabled($storeId) === false) { From e9242bcdd6a402361dba87d9165f4ad7737bccb0 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 3 Jul 2025 14:24:56 +0200 Subject: [PATCH 019/119] MAGE-1138: add use tmp index config --- Helper/ConfigHelper.php | 119 +++++++++++++++++---------- Helper/Configuration/QueueHelper.php | 56 +++++++++++++ Service/Product/IndexBuilder.php | 2 +- etc/adminhtml/system.xml | 17 ++++ etc/config.xml | 1 + 5 files changed, 150 insertions(+), 45 deletions(-) create mode 100644 Helper/Configuration/QueueHelper.php diff --git a/Helper/ConfigHelper.php b/Helper/ConfigHelper.php index 5a1cb30a1..ceb7db730 100755 --- a/Helper/ConfigHelper.php +++ b/Helper/ConfigHelper.php @@ -5,6 +5,7 @@ use Algolia\AlgoliaSearch\Api\Product\ReplicaManagerInterface; use Algolia\AlgoliaSearch\Helper\Configuration\AutocompleteHelper; use Algolia\AlgoliaSearch\Helper\Configuration\InstantSearchHelper; +use Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper; use Algolia\AlgoliaSearch\Service\AlgoliaConnector; use Algolia\AlgoliaSearch\Service\Serializer; use Magento\Cookie\Helper\Cookie as CookieHelper; @@ -42,11 +43,6 @@ class ConfigHelper public const INDEX_EMPTY_CATEGORIES = 'algoliasearch_categories/categories/index_empty_categories'; public const CATEGORY_SEPARATOR = 'algoliasearch_categories/categories/category_separator'; - public const IS_ACTIVE = 'algoliasearch_queue/queue/active'; - public const USE_BUILT_IN_CRON = 'algoliasearch_queue/queue/use_built_in_cron'; - public const NUMBER_OF_JOB_TO_RUN = 'algoliasearch_queue/queue/number_of_job_to_run'; - public const RETRY_LIMIT = 'algoliasearch_queue/queue/number_of_retries'; - public const XML_PATH_IMAGE_WIDTH = 'algoliasearch_images/image/width'; public const XML_PATH_IMAGE_HEIGHT = 'algoliasearch_images/image/height'; public const XML_PATH_IMAGE_TYPE = 'algoliasearch_images/image/type'; @@ -156,7 +152,8 @@ public function __construct( protected GroupExcludedWebsiteRepositoryInterface $groupExcludedWebsiteRepository, protected CookieHelper $cookieHelper, protected AutocompleteHelper $autocompleteConfig, - protected InstantSearchHelper $instantSearchConfig + protected InstantSearchHelper $instantSearchConfig, + protected QueueHelper $queueHelper ) {} @@ -384,44 +381,6 @@ public function getNumberOfElementByPage($storeId = null) return (int)$this->configInterface->getValue(self::NUMBER_OF_ELEMENT_BY_PAGE, ScopeInterface::SCOPE_STORE, $storeId); } - /** - * @param $storeId - * @return mixed - */ - public function getNumberOfJobToRun($storeId = null) - { - $nbJobs = (int)$this->configInterface->getValue(self::NUMBER_OF_JOB_TO_RUN, ScopeInterface::SCOPE_STORE, $storeId); - - return max($nbJobs, 1); - } - - /** - * @param $storeId - * @return int - */ - public function getRetryLimit($storeId = null) - { - return (int)$this->configInterface->getValue(self::RETRY_LIMIT, ScopeInterface::SCOPE_STORE, $storeId); - } - - /** - * @param $storeId - * @return bool - */ - public function isQueueActive($storeId = null) - { - return $this->configInterface->isSetFlag(self::IS_ACTIVE, ScopeInterface::SCOPE_STORE, $storeId); - } - - /** - * @param $storeId - * @return bool - */ - public function useBuiltInCron($storeId = null) - { - return $this->configInterface->isSetFlag(self::USE_BUILT_IN_CRON, ScopeInterface::SCOPE_STORE, $storeId); - } - /** * @param $storeId * @return bool @@ -1715,6 +1674,32 @@ public function isQueueIndexerEnabled(): bool */ public const LEGACY_USE_VIRTUAL_REPLICA_ENABLED = 'algoliasearch_instant/instant/use_virtual_replica'; + // --- Indexing Queue --- // + + /** + * @deprecated This constant has been moved to a domain specific config helper and will be removed in a future release + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::IS_ACTIVE + */ + public const IS_ACTIVE = QueueHelper::IS_ACTIVE; + + /** + * @deprecated This constant has been moved to a domain specific config helper and will be removed in a future release + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::USE_BUILT_IN_CRON + */ + public const USE_BUILT_IN_CRON = QueueHelper::USE_BUILT_IN_CRON; + + /** + * @deprecated This constant has been moved to a domain specific config helper and will be removed in a future release + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::NUMBER_OF_JOB_TO_RUN + */ + public const NUMBER_OF_JOB_TO_RUN = QueueHelper::NUMBER_OF_JOB_TO_RUN; + + /** + * @deprecated This constant has been moved to a domain specific config helper and will be removed in a future release + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::RETRY_LIMIT + */ + public const RETRY_LIMIT = QueueHelper::RETRY_LIMIT; + // --- Indexing Manager --- // /** @@ -2045,6 +2030,52 @@ public function hidePaginationInInstantSearchPage($storeId = null) return $this->instantSearchConfig->shouldHidePagination($storeId); } + // --- Indexing Queue --- // + + /** + * @param $storeId + * @return bool + * @deprecated This method has been moved to the Queue config helper and will be removed in a future version + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::isQueueActive() + */ + public function isQueueActive($storeId = null) + { + return $this->queueHelper->isQueueActive($storeId); + } + + /** + * @param $storeId + * @return bool + * @deprecated This method has been moved to the Queue config helper and will be removed in a future version + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::useBuiltInCron() + */ + public function useBuiltInCron($storeId = null) + { + return $this->queueHelper->useBuiltInCron($storeId); + } + + /** + * @param $storeId + * @return bool + * @deprecated This method has been moved to the Queue config helper and will be removed in a future version + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::getNumberOfJobToRun() + */ + public function getNumberOfJobToRun($storeId = null) + { + return $this->queueHelper->getNumberOfJobToRun($storeId); + } + + /** + * @param $storeId + * @return bool + * @deprecated This method has been moved to the Queue config helper and will be removed in a future version + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::getRetryLimit() + */ + public function getRetryLimit($storeId = null) + { + return $this->queueHelper->getRetryLimit($storeId); + } + // --- Indexing Manager --- // /** diff --git a/Helper/Configuration/QueueHelper.php b/Helper/Configuration/QueueHelper.php new file mode 100644 index 000000000..792feb0c5 --- /dev/null +++ b/Helper/Configuration/QueueHelper.php @@ -0,0 +1,56 @@ +configInterface->isSetFlag(self::IS_ACTIVE, ScopeInterface::SCOPE_STORE, $storeId); + } + + /** + * @param $storeId + * @return bool + */ + public function useBuiltInCron($storeId = null) + { + return $this->configInterface->isSetFlag(self::USE_BUILT_IN_CRON, ScopeInterface::SCOPE_STORE, $storeId); + } + + /** + * @param $storeId + * @return mixed + */ + public function getNumberOfJobToRun($storeId = null) + { + $nbJobs = (int)$this->configInterface->getValue(self::NUMBER_OF_JOB_TO_RUN, ScopeInterface::SCOPE_STORE, $storeId); + + return max($nbJobs, 1); + } + + /** + * @param $storeId + * @return int + */ + public function getRetryLimit($storeId = null) + { + return (int)$this->configInterface->getValue(self::RETRY_LIMIT, ScopeInterface::SCOPE_STORE, $storeId); + } +} diff --git a/Service/Product/IndexBuilder.php b/Service/Product/IndexBuilder.php index f980791f3..3bb495aee 100644 --- a/Service/Product/IndexBuilder.php +++ b/Service/Product/IndexBuilder.php @@ -102,7 +102,7 @@ public function buildIndex(int $storeId, ?array $entityIds, ?array $options): vo $options['page'] ?? 1, $options['pageSize'] ?? $this->configHelper->getNumberOfElementByPage($storeId), $entityIds, - $options['useTmpIndex'] + $options['useTmpIndex'] ?? false ); $this->stopEmulation(); diff --git a/etc/adminhtml/system.xml b/etc/adminhtml/system.xml index 3391f7bfe..ac08e9a97 100755 --- a/etc/adminhtml/system.xml +++ b/etc/adminhtml/system.xml @@ -1038,6 +1038,23 @@ validate-digits Algolia\AlgoliaSearch\Model\Source\RetryValues + + + + + 1 + + + + + Magento\Config\Model\Config\Source\Yesno + + _tmp will be used for products full reindex. If you set it to "No", product updates will be sent directly to your production index. + ]]> + 1 diff --git a/etc/config.xml b/etc/config.xml index d11902020..46ecd94d5 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -66,6 +66,7 @@ 0 */5 * * * * + 1 From f690735f55d5214d5c847094e6336d641f6d700f Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 3 Jul 2025 15:20:51 +0200 Subject: [PATCH 020/119] MAGE-1138: add useTmpIndex logic to the product index builder --- Helper/Configuration/QueueHelper.php | 11 +++++++++++ Service/Product/BatchQueueProcessor.php | 4 +++- Ui/Component/Listing/Column/Data.php | 4 +++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Helper/Configuration/QueueHelper.php b/Helper/Configuration/QueueHelper.php index 792feb0c5..a7b09b3fa 100644 --- a/Helper/Configuration/QueueHelper.php +++ b/Helper/Configuration/QueueHelper.php @@ -11,6 +11,7 @@ class QueueHelper public const USE_BUILT_IN_CRON = 'algoliasearch_queue/queue/use_built_in_cron'; public const NUMBER_OF_JOB_TO_RUN = 'algoliasearch_queue/queue/number_of_job_to_run'; public const RETRY_LIMIT = 'algoliasearch_queue/queue/number_of_retries'; + public const USE_TMP_INDEX = 'algoliasearch_queue/queue/use_tmp_index'; public function __construct( protected ScopeConfigInterface $configInterface, @@ -53,4 +54,14 @@ public function getRetryLimit($storeId = null) { return (int)$this->configInterface->getValue(self::RETRY_LIMIT, ScopeInterface::SCOPE_STORE, $storeId); } + + /** + * @param $storeId + * @return bool + */ + public function useTmpIndex($storeId = null) + { + return $this->isQueueActive($storeId) && + $this->configInterface->isSetFlag(self::USE_TMP_INDEX, ScopeInterface::SCOPE_STORE, $storeId); + } } diff --git a/Service/Product/BatchQueueProcessor.php b/Service/Product/BatchQueueProcessor.php index 680192a62..a41456e62 100644 --- a/Service/Product/BatchQueueProcessor.php +++ b/Service/Product/BatchQueueProcessor.php @@ -6,6 +6,7 @@ use Algolia\AlgoliaSearch\Exception\DiagnosticsException; use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; use Algolia\AlgoliaSearch\Helper\ConfigHelper; +use Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper; use Algolia\AlgoliaSearch\Helper\Data; use Algolia\AlgoliaSearch\Helper\Entity\ProductHelper; use Algolia\AlgoliaSearch\Logger\DiagnosticsLogger; @@ -24,6 +25,7 @@ public function __construct( protected Data $dataHelper, protected ConfigHelper $configHelper, protected ProductHelper $productHelper, + protected QueueHelper $queueHelper, protected Queue $queue, protected DiagnosticsLogger $diag, protected AlgoliaCredentialsManager $algoliaCredentialsManager, @@ -56,7 +58,7 @@ public function processBatch(int $storeId, ?array $entityIds = null): void return; } - $useTmpIndex = $this->configHelper->isQueueActive($storeId); + $useTmpIndex = $this->queueHelper->useTmpIndex($storeId); $this->syncAlgoliaSettings($storeId, $useTmpIndex); $this->handleFullIndex($storeId, $productsPerPage, $useTmpIndex); diff --git a/Ui/Component/Listing/Column/Data.php b/Ui/Component/Listing/Column/Data.php index 2b9a5e3ba..3ae71e929 100644 --- a/Ui/Component/Listing/Column/Data.php +++ b/Ui/Component/Listing/Column/Data.php @@ -53,7 +53,9 @@ protected function formatData(array $data, $depth = 0): string continue; } - + if (is_bool($value)) { + $value = $value ? 'Yes' : 'No'; + } $formattedData .= str_repeat('   ', $depth ) . $stringKey . ' : ' . $value . '
'; } From aa365988d2fc1f01cbb8ddc7c2640ac186be3a47 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 3 Jul 2025 16:34:45 +0200 Subject: [PATCH 021/119] MAGE-1138: add integration test --- Setup/Patch/Schema/ConfigPatch.php | 1 + Test/Integration/Indexing/Queue/QueueTest.php | 66 +++++++++++++++---- 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/Setup/Patch/Schema/ConfigPatch.php b/Setup/Patch/Schema/ConfigPatch.php index 77309f665..920bb640e 100644 --- a/Setup/Patch/Schema/ConfigPatch.php +++ b/Setup/Patch/Schema/ConfigPatch.php @@ -68,6 +68,7 @@ class ConfigPatch implements SchemaPatchInterface 'algoliasearch_queue/queue/active' => '0', 'algoliasearch_queue/queue/number_of_job_to_run' => '5', 'algoliasearch_queue/queue/number_of_retries' => '3', + 'algoliasearch_queue/queue/use_tmp_index' => '1', 'algoliasearch_cc_analytics/cc_analytics_group/enable' => '0', 'algoliasearch_cc_analytics/cc_analytics_group/is_selector' => '.ais-Hits-item a.result, .ais-InfiniteHits-item a.result', diff --git a/Test/Integration/Indexing/Queue/QueueTest.php b/Test/Integration/Indexing/Queue/QueueTest.php index c851b05cf..ec7534636 100644 --- a/Test/Integration/Indexing/Queue/QueueTest.php +++ b/Test/Integration/Indexing/Queue/QueueTest.php @@ -3,6 +3,7 @@ namespace Algolia\AlgoliaSearch\Test\Integration\Indexing\Queue; use Algolia\AlgoliaSearch\Helper\ConfigHelper; +use Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper; use Algolia\AlgoliaSearch\Model\Indexer\QueueRunner; use Algolia\AlgoliaSearch\Model\IndicesConfigurator; use Algolia\AlgoliaSearch\Model\Job; @@ -44,11 +45,11 @@ protected function setUp(): void public function testFill() { $this->resetConfigs([ - ConfigHelper::NUMBER_OF_JOB_TO_RUN, + QueueHelper::NUMBER_OF_JOB_TO_RUN, ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, ]); - $this->setConfig(ConfigHelper::IS_ACTIVE, '1'); + $this->setConfig(QueueHelper::IS_ACTIVE, '1'); $this->connection->query('TRUNCATE TABLE algoliasearch_queue'); $productBatchQueueProcessor = $this->objectManager->get(ProductBatchQueueProcessor::class); @@ -84,7 +85,7 @@ public function testFill() public function testExecute() { - $this->setConfig(ConfigHelper::IS_ACTIVE, '1'); + $this->setConfig(QueueHelper::IS_ACTIVE, '1'); $this->connection->query('TRUNCATE TABLE algoliasearch_queue'); $productBatchQueueProcessor = $this->objectManager->get(ProductBatchQueueProcessor::class); @@ -136,16 +137,57 @@ public function testExecute() $this->assertEquals(0, count($rows)); } + public function testTmpIndexConfig() + { + $this->setConfig(QueueHelper::IS_ACTIVE, '1'); + // Setting "Use a temporary index for full products reindex" configuration to "No" + $this->setConfig(QueueHelper::USE_TMP_INDEX, '0'); + $this->connection->query('TRUNCATE TABLE algoliasearch_queue'); + + $productBatchQueueProcessor = $this->objectManager->get(ProductBatchQueueProcessor::class); + $productBatchQueueProcessor->processBatch(1); + + $rows = $this->connection->query('SELECT * FROM algoliasearch_queue')->fetchAll(); + // Without temporary index enabled, there are only 2 jobs (saveConfigurationToAlgolia and buildIndexFull) + $this->assertEquals(2, count($rows)); + + $indexingJob = $rows[1]; + $jobData = json_decode($indexingJob['data'], true); + + $this->assertEquals('buildIndexFull', $indexingJob['method']); + $this->assertFalse($jobData['options']['useTmpIndex']); + + /** @var Queue $queue */ + $queue = $this->objectManager->get(Queue::class); + + // Run the first job (saveSettings) + $queue->runCron(1, true); + + $this->algoliaConnector->waitLastTask(); + + $indices = $this->algoliaConnector->listIndexes(); + + $existsDefaultTmpIndex = false; + foreach ($indices['items'] as $index) { + if ($index['name'] === $this->indexPrefix . 'default_products_tmp') { + $existsDefaultTmpIndex = true; + } + } + // Checking if the temporary index hasn't been created + $this->assertFalse($existsDefaultTmpIndex, 'Temporary index exists and it should not'); + } + public function testSettings() { $this->resetConfigs([ - ConfigHelper::NUMBER_OF_JOB_TO_RUN, + QueueHelper::NUMBER_OF_JOB_TO_RUN, ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, ConfigHelper::FACETS, + QueueHelper::USE_TMP_INDEX, ConfigHelper::PRODUCT_ATTRIBUTES ]); - $this->setConfig(ConfigHelper::IS_ACTIVE, '1'); + $this->setConfig(QueueHelper::IS_ACTIVE, '1'); $this->connection->query('DELETE FROM algoliasearch_queue'); @@ -178,8 +220,8 @@ public function testSettings() public function testMergeSettings() { - $this->setConfig(ConfigHelper::IS_ACTIVE, '1'); - $this->setConfig(ConfigHelper::NUMBER_OF_JOB_TO_RUN, 1); + $this->setConfig(QueueHelper::IS_ACTIVE, '1'); + $this->setConfig(QueueHelper::NUMBER_OF_JOB_TO_RUN, 1); $this->setConfig(ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, 300); $this->connection->query('DELETE FROM algoliasearch_queue'); @@ -791,7 +833,7 @@ public function testGetJobs() public function testHugeJob() { // Default value - maxBatchSize = 1000 - $this->setConfig(ConfigHelper::NUMBER_OF_JOB_TO_RUN, 10); + $this->setConfig(QueueHelper::NUMBER_OF_JOB_TO_RUN, 10); $this->setConfig(ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, 100); $productIds = range(1, 5000); @@ -829,7 +871,7 @@ public function testHugeJob() public function testMaxSingleJobSize() { // Default value - maxBatchSize = 1000 - $this->setConfig(ConfigHelper::NUMBER_OF_JOB_TO_RUN, 10); + $this->setConfig(QueueHelper::NUMBER_OF_JOB_TO_RUN, 10); $this->setConfig(ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, 100); $productIds = range(1, 99); @@ -870,13 +912,13 @@ public function testMaxSingleJobSize() public function testMaxSingleJobsSizeOnProductReindex() { $this->resetConfigs([ - ConfigHelper::NUMBER_OF_JOB_TO_RUN, + QueueHelper::NUMBER_OF_JOB_TO_RUN, ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, ]); - $this->setConfig(ConfigHelper::IS_ACTIVE, '1'); + $this->setConfig(QueueHelper::IS_ACTIVE, '1'); - $this->setConfig(ConfigHelper::NUMBER_OF_JOB_TO_RUN, 10); + $this->setConfig(QueueHelper::NUMBER_OF_JOB_TO_RUN, 10); $this->setConfig(ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, 100); $this->connection->query('TRUNCATE TABLE algoliasearch_queue'); From 361043f39d3a0b9a69b863ec2e7018560cf4ce05 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 9 Jul 2025 14:19:52 +0200 Subject: [PATCH 022/119] MAGE-1112: make price attribute optional on product records --- Service/Product/RecordBuilder.php | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Service/Product/RecordBuilder.php b/Service/Product/RecordBuilder.php index 7f9fff32f..16213a185 100644 --- a/Service/Product/RecordBuilder.php +++ b/Service/Product/RecordBuilder.php @@ -123,7 +123,11 @@ public function buildRecord(DataObject $entity): array } $subProducts = $this->getSubProducts($product); $customData = $this->addAdditionalAttributes($customData, $additionalAttributes, $product, $subProducts); - $customData = $this->priceManager->addPriceDataByProductType($customData, $product, $subProducts); + + if ($this->isPriceIndexingEnabled($additionalAttributes)) { + $customData = $this->priceManager->addPriceDataByProductType($customData, $product, $subProducts); + } + $transport = new DataObject($customData); $this->eventManager->dispatch( 'algolia_subproducts_index', @@ -195,6 +199,15 @@ public function isAttributeEnabled($additionalAttributes, $attributeName): bool return false; } + /** + * @param array $additionalAttributes + * @return bool + */ + protected function isPriceIndexingEnabled(array $additionalAttributes): bool + { + return $this->isAttributeEnabled($additionalAttributes, 'price'); + } + /** * @param array $algoliaData Data for product object to be serialized to Algolia index * @param Product $product @@ -816,3 +829,4 @@ public function productIsInStock($product, $storeId): bool return $product->isSaleable() && $stockItem->getIsInStock(); } } + From 8e773951e791ed35616b37e98ca3dd82814177f1 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 9 Jul 2025 16:00:58 +0200 Subject: [PATCH 023/119] MAGE-1112: add price attribute notice --- Helper/ConfigHelper.php | 31 ++++++++++++++++++++++--- Helper/Configuration/NoticeHelper.php | 33 +++++++++++++++++++++++++-- Service/Product/RecordBuilder.php | 10 ++------ 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/Helper/ConfigHelper.php b/Helper/ConfigHelper.php index ceb7db730..8cf4e2bb9 100755 --- a/Helper/ConfigHelper.php +++ b/Helper/ConfigHelper.php @@ -1068,17 +1068,42 @@ public function getAttributesToRetrieve($groupId) return ['attributesToRetrieve' => $attributes]; } + /** + * @param $attributes + * @param $attributeName + * @return bool + */ + public function isAttributeInList($attributes, $attributeName): bool + { + foreach ($attributes as $attr) { + if ($attr['attribute'] === $attributeName) { + return true; + } + } + + return false; + } + /** * @param $storeId - * @return array + * @return mixed */ - public function getProductAdditionalAttributes($storeId = null) + public function getProductAttributesList($storeId = null) { - $attributes = $this->serializer->unserialize($this->configInterface->getValue( + return $this->serializer->unserialize($this->configInterface->getValue( self::PRODUCT_ATTRIBUTES, ScopeInterface::SCOPE_STORE, $storeId )); + } + + /** + * @param $storeId + * @return array + */ + public function getProductAdditionalAttributes($storeId = null) + { + $attributes = $this->getProductAttributesList($storeId); $facets = $this->serializer->unserialize($this->configInterface->getValue( self::FACETS, diff --git a/Helper/Configuration/NoticeHelper.php b/Helper/Configuration/NoticeHelper.php index 03326e9ec..34e0e21fd 100644 --- a/Helper/Configuration/NoticeHelper.php +++ b/Helper/Configuration/NoticeHelper.php @@ -23,6 +23,7 @@ class NoticeHelper extends \Magento\Framework\App\Helper\AbstractHelper 'getRecommendNotice', 'getCookieConfigurationNotice', 'getMultiApplicationIDsNotice', + 'getPriceIndexingNotice', ]; /** @var array[] */ @@ -189,8 +190,8 @@ protected function getClickAnalyticsNotice() ]; } - protected function getCookieConfigurationNotice() - { + protected function getCookieConfigurationNotice() + { $noticeContent = ''; $selector = ''; $method = 'after'; @@ -363,4 +364,32 @@ protected function getMultiApplicationIDsNotice(): void 'message' => $this->formatNotice($noticeTitle, $noticeContent), ]; } + + + /** + * This notice serves as a warning when user removes the price attribute from the attributes list but it's still present either in the sortings or in the facets + * @return void + */ + protected function getPriceIndexingNotice(): void + { + $attributesToIndex = $this->configHelper->getProductAdditionalAttributes(); + $attributesList = $this->configHelper->getProductAttributesList(); + + // we want to display the warning only if price is not present in the attribute list but is present somewhere else + if (!($this->configHelper->isAttributeInList($attributesToIndex, 'price') + && !$this->configHelper->isAttributeInList($attributesList, 'price')) + ) { + return; + } + + $noticeTitle = 'Price attribute indexing'; + $noticeContent = '

Price attribute has been removed from the product attributes list but is still present in the facets, sortings or custom rankings lists.

+

If you want to remove prices from the product records, you need to remove them from those lists as well.

'; + + $this->notices[] = [ + 'selector' => '.entry-edit', + 'method' => 'before', + 'message' => $this->formatNotice($noticeTitle, $noticeContent), + ]; + } } diff --git a/Service/Product/RecordBuilder.php b/Service/Product/RecordBuilder.php index 16213a185..3b0e6363c 100644 --- a/Service/Product/RecordBuilder.php +++ b/Service/Product/RecordBuilder.php @@ -190,13 +190,7 @@ protected function addAttribute($attribute, $defaultData, $customData, $addition */ public function isAttributeEnabled($additionalAttributes, $attributeName): bool { - foreach ($additionalAttributes as $attr) { - if ($attr['attribute'] === $attributeName) { - return true; - } - } - - return false; + return $this->configHelper->isAttributeInList($additionalAttributes, $attributeName); } /** @@ -205,7 +199,7 @@ public function isAttributeEnabled($additionalAttributes, $attributeName): bool */ protected function isPriceIndexingEnabled(array $additionalAttributes): bool { - return $this->isAttributeEnabled($additionalAttributes, 'price'); + return $this->configHelper->isAttributeInList($additionalAttributes, 'price'); } /** From 67cb7aedbc79a6c8bdaad7b15654bd0b4933bc01 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 9 Jul 2025 16:15:55 +0200 Subject: [PATCH 024/119] MAGE-1112: updated product indexing test --- Test/Integration/Indexing/Product/ProductsIndexingTest.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Test/Integration/Indexing/Product/ProductsIndexingTest.php b/Test/Integration/Indexing/Product/ProductsIndexingTest.php index 28c5e5ca5..fbebbb5b9 100644 --- a/Test/Integration/Indexing/Product/ProductsIndexingTest.php +++ b/Test/Integration/Indexing/Product/ProductsIndexingTest.php @@ -70,7 +70,7 @@ public function testDefaultIndexableAttributes() 'thumbnail_url', 'image_url', 'in_stock', - 'price', + //'price', since version 3.17.0, the price attribute is not mandatory if it's not present in any attributes list 'type_id', 'algoliaLastUpdateAtCET', 'categoryIds', @@ -85,6 +85,8 @@ public function testDefaultIndexableAttributes() unset($hit[$attribute]); } + $this->assertArrayNotHasKey('price', $hit, 'Record has a price attribute but it should not'); + $extraAttributes = implode(', ', array_keys($hit)); $this->assertEmpty($hit, 'Extra products attributes (' . $extraAttributes . ') are indexed and should not be.'); } From 8789a32e19c3536d59a7f7736abd2a2a965603fc Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 10 Jul 2025 15:48:40 +0200 Subject: [PATCH 025/119] MAGE-1112: added additional sentence to the notice --- Helper/Configuration/NoticeHelper.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Helper/Configuration/NoticeHelper.php b/Helper/Configuration/NoticeHelper.php index 34e0e21fd..4a3e12f18 100644 --- a/Helper/Configuration/NoticeHelper.php +++ b/Helper/Configuration/NoticeHelper.php @@ -384,7 +384,8 @@ protected function getPriceIndexingNotice(): void $noticeTitle = 'Price attribute indexing'; $noticeContent = '

Price attribute has been removed from the product attributes list but is still present in the facets, sortings or custom rankings lists.

-

If you want to remove prices from the product records, you need to remove them from those lists as well.

'; +

If you want to remove prices from the product records, you need to remove them from those lists as well.

+

If you want the prices to be included in the product records, you need to add it in the product attributes list in the "Products" section of the configuration.

'; $this->notices[] = [ 'selector' => '.entry-edit', From d8b96317610dfd825d09a1987ff5d21730cfae2c Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 10 Jul 2025 15:49:47 +0200 Subject: [PATCH 026/119] MAGE-1112: added additional sentence to the notice --- Helper/Configuration/NoticeHelper.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Helper/Configuration/NoticeHelper.php b/Helper/Configuration/NoticeHelper.php index 4a3e12f18..2198b28b1 100644 --- a/Helper/Configuration/NoticeHelper.php +++ b/Helper/Configuration/NoticeHelper.php @@ -385,7 +385,7 @@ protected function getPriceIndexingNotice(): void $noticeTitle = 'Price attribute indexing'; $noticeContent = '

Price attribute has been removed from the product attributes list but is still present in the facets, sortings or custom rankings lists.

If you want to remove prices from the product records, you need to remove them from those lists as well.

-

If you want the prices to be included in the product records, you need to add it in the product attributes list in the "Products" section of the configuration.

'; +

If you want the prices to be included in the product records, you need to add the price attribute in the product attributes list in the "Products" section of the configuration.

'; $this->notices[] = [ 'selector' => '.entry-edit', From fb74bdb7829d9ef8e49765ae6f4307b17056e87a Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Mon, 28 Jul 2025 15:45:26 -0400 Subject: [PATCH 027/119] MAGE-1374 Restore changes introduced on PR #1779 --- Helper/ConfigHelper.php | 123 ++++++++++++++++++++++++---------------- 1 file changed, 75 insertions(+), 48 deletions(-) diff --git a/Helper/ConfigHelper.php b/Helper/ConfigHelper.php index cd62e9a84..5e945e1e4 100755 --- a/Helper/ConfigHelper.php +++ b/Helper/ConfigHelper.php @@ -5,6 +5,7 @@ use Algolia\AlgoliaSearch\Api\Product\ReplicaManagerInterface; use Algolia\AlgoliaSearch\Helper\Configuration\AutocompleteHelper; use Algolia\AlgoliaSearch\Helper\Configuration\InstantSearchHelper; +use Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper; use Algolia\AlgoliaSearch\Service\AlgoliaConnector; use Algolia\AlgoliaSearch\Service\Serializer; use Magento\Cookie\Helper\Cookie as CookieHelper; @@ -84,13 +85,6 @@ class ConfigHelper public const XML_PATH_IMAGE_HEIGHT = 'algoliasearch_images/image/height'; public const XML_PATH_IMAGE_TYPE = 'algoliasearch_images/image/type'; - // --- Indexing Queue / Cron --- // - - public const IS_ACTIVE = 'algoliasearch_queue/queue/active'; - public const USE_BUILT_IN_CRON = 'algoliasearch_queue/queue/use_built_in_cron'; - public const NUMBER_OF_JOB_TO_RUN = 'algoliasearch_queue/queue/number_of_job_to_run'; - public const RETRY_LIMIT = 'algoliasearch_queue/queue/number_of_retries'; - // --- Indexing Manager --- // public const ENABLE_INDEXING = 'algoliasearch_indexing_manager/algolia_indexing/enable_indexing'; @@ -177,7 +171,8 @@ public function __construct( protected GroupExcludedWebsiteRepositoryInterface $groupExcludedWebsiteRepository, protected CookieHelper $cookieHelper, protected AutocompleteHelper $autocompleteConfig, - protected InstantSearchHelper $instantSearchConfig + protected InstantSearchHelper $instantSearchConfig, + protected QueueHelper $queueHelper ) {} @@ -943,46 +938,6 @@ public function getImageType($storeId = null) return $this->configInterface->getValue(self::XML_PATH_IMAGE_TYPE, ScopeInterface::SCOPE_STORE, $storeId); } - // --- Indexing Queue / Cron --- // - - /** - * @param $storeId - * @return mixed - */ - public function getNumberOfJobToRun($storeId = null): int - { - $nbJobs = (int) $this->configInterface->getValue(self::NUMBER_OF_JOB_TO_RUN, ScopeInterface::SCOPE_STORE, $storeId); - - return (int) max($nbJobs, 1); - } - - /** - * @param $storeId - * @return int - */ - public function getRetryLimit($storeId = null): int - { - return (int) $this->configInterface->getValue(self::RETRY_LIMIT, ScopeInterface::SCOPE_STORE, $storeId); - } - - /** - * @param $storeId - * @return bool - */ - public function isQueueActive($storeId = null): bool - { - return $this->configInterface->isSetFlag(self::IS_ACTIVE, ScopeInterface::SCOPE_STORE, $storeId); - } - - /** - * @param $storeId - * @return bool - */ - public function useBuiltInCron($storeId = null): bool - { - return $this->configInterface->isSetFlag(self::USE_BUILT_IN_CRON, ScopeInterface::SCOPE_STORE, $storeId); - } - // --- Indexing Manager --- // /** @@ -1740,6 +1695,32 @@ public function getCacheTime($storeId = null) */ public const LEGACY_USE_VIRTUAL_REPLICA_ENABLED = 'algoliasearch_instant/instant/use_virtual_replica'; + // --- Indexing Queue / Cron --- // + + /** + * @deprecated This constant has been moved to a domain specific config helper and will be removed in a future release + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::IS_ACTIVE + */ + public const IS_ACTIVE = QueueHelper::IS_ACTIVE; + + /** + * @deprecated This constant has been moved to a domain specific config helper and will be removed in a future release + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::USE_BUILT_IN_CRON + */ + public const USE_BUILT_IN_CRON = QueueHelper::USE_BUILT_IN_CRON; + + /** + * @deprecated This constant has been moved to a domain specific config helper and will be removed in a future release + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::NUMBER_OF_JOB_TO_RUN + */ + public const NUMBER_OF_JOB_TO_RUN = QueueHelper::NUMBER_OF_JOB_TO_RUN; + + /** + * @deprecated This constant has been moved to a domain specific config helper and will be removed in a future release + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::RETRY_LIMIT + */ + public const RETRY_LIMIT = QueueHelper::RETRY_LIMIT; + // --- Indexing Manager --- // /** @@ -2088,6 +2069,52 @@ public function hidePaginationInInstantSearchPage($storeId = null) return $this->instantSearchConfig->shouldHidePagination($storeId); } + // --- Indexing Queue / Cron --- // + + /** + * @param $storeId + * @return bool + * @deprecated This method has been moved to the Queue config helper and will be removed in a future version + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::isQueueActive() + */ + public function isQueueActive($storeId = null) + { + return $this->queueHelper->isQueueActive($storeId); + } + + /** + * @param $storeId + * @return bool + * @deprecated This method has been moved to the Queue config helper and will be removed in a future version + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::useBuiltInCron() + */ + public function useBuiltInCron($storeId = null) + { + return $this->queueHelper->useBuiltInCron($storeId); + } + + /** + * @param $storeId + * @return bool + * @deprecated This method has been moved to the Queue config helper and will be removed in a future version + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::getNumberOfJobToRun() + */ + public function getNumberOfJobToRun($storeId = null) + { + return $this->queueHelper->getNumberOfJobToRun($storeId); + } + + /** + * @param $storeId + * @return bool + * @deprecated This method has been moved to the Queue config helper and will be removed in a future version + * @see \Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper::getRetryLimit() + */ + public function getRetryLimit($storeId = null) + { + return $this->queueHelper->getRetryLimit($storeId); + } + // --- Indexing Manager --- // /** From 411094212f3647386af72db1573e117988dab8e7 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Mon, 28 Jul 2025 15:57:17 -0400 Subject: [PATCH 028/119] MAGE-1374 Restore changes introduced on PR #1781 --- Helper/ConfigHelper.php | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/Helper/ConfigHelper.php b/Helper/ConfigHelper.php index 5e945e1e4..1c1c23308 100755 --- a/Helper/ConfigHelper.php +++ b/Helper/ConfigHelper.php @@ -310,11 +310,7 @@ public function getAlgoliaCookieDuration($storeId = null) */ public function getProductAdditionalAttributes($storeId = null) { - $attributes = $this->serializer->unserialize($this->configInterface->getValue( - self::PRODUCT_ATTRIBUTES, - ScopeInterface::SCOPE_STORE, - $storeId - )); + $attributes = $this->getProductAttributesList($storeId); $facets = $this->serializer->unserialize($this->configInterface->getValue( self::FACETS, @@ -520,6 +516,35 @@ public function useVirtualReplica(?int $storeId = null): bool )); } + /** + * @param $attributes + * @param $attributeName + * @return bool + */ + public function isAttributeInList($attributes, $attributeName): bool + { + foreach ($attributes as $attr) { + if ($attr['attribute'] === $attributeName) { + return true; + } + } + + return false; + } + + /** + * @param $storeId + * @return mixed + */ + public function getProductAttributesList($storeId = null) + { + return $this->serializer->unserialize($this->configInterface->getValue( + self::PRODUCT_ATTRIBUTES, + ScopeInterface::SCOPE_STORE, + $storeId + )); + } + // --- Categories --- // /** * @param $storeId From 6bb77c24d129dfa5c4767d3ced05e41a04a3652c Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 6 Aug 2025 16:06:03 -0400 Subject: [PATCH 029/119] MAGE-1374 Repair unit tests for 3.17 post 3.16 port --- Test/Unit/Helper/ConfigHelperTest.php | 6 +++++- Test/Unit/Service/Product/BatchQueueProcessorTest.php | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Test/Unit/Helper/ConfigHelperTest.php b/Test/Unit/Helper/ConfigHelperTest.php index 3e51bacbe..78071ca34 100644 --- a/Test/Unit/Helper/ConfigHelperTest.php +++ b/Test/Unit/Helper/ConfigHelperTest.php @@ -2,6 +2,7 @@ namespace Algolia\AlgoliaSearch\Test\Unit\Helper; +use Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper; use Algolia\AlgoliaSearch\Service\Serializer; use Algolia\AlgoliaSearch\Helper\Configuration\AutocompleteHelper; use Algolia\AlgoliaSearch\Helper\Configuration\InstantSearchHelper; @@ -38,6 +39,7 @@ class ConfigHelperTest extends TestCase protected ?CookieHelper $cookieHelper; protected ?AutocompleteHelper $autocompleteHelper; protected ?InstantSearchHelper $instantSearchHelper; + protected ?QueueHelper $queueHelper; protected function setUp(): void { @@ -56,6 +58,7 @@ protected function setUp(): void $this->cookieHelper = $this->createMock(CookieHelper::class); $this->autocompleteHelper = $this->createMock(AutocompleteHelper::class); $this->instantSearchHelper = $this->createMock(InstantSearchHelper::class); + $this->queueHelper = $this->createMock(QueueHelper::class); $this->configHelper = new ConfigHelperTestable( $this->configInterface, @@ -72,7 +75,8 @@ protected function setUp(): void $this->groupExcludedWebsiteRepository, $this->cookieHelper, $this->autocompleteHelper, - $this->instantSearchHelper + $this->instantSearchHelper, + $this->queueHelper ); } diff --git a/Test/Unit/Service/Product/BatchQueueProcessorTest.php b/Test/Unit/Service/Product/BatchQueueProcessorTest.php index 8eaf8534f..a2e041de4 100644 --- a/Test/Unit/Service/Product/BatchQueueProcessorTest.php +++ b/Test/Unit/Service/Product/BatchQueueProcessorTest.php @@ -4,6 +4,7 @@ use Algolia\AlgoliaSearch\Exception\DiagnosticsException; use Algolia\AlgoliaSearch\Helper\ConfigHelper; +use Algolia\AlgoliaSearch\Helper\Configuration\QueueHelper; use Algolia\AlgoliaSearch\Helper\Data; use Algolia\AlgoliaSearch\Helper\Entity\ProductHelper; use Algolia\AlgoliaSearch\Logger\DiagnosticsLogger; @@ -23,6 +24,7 @@ class BatchQueueProcessorTest extends TestCase protected ?Data $dataHelper; protected ?ConfigHelper $configHelper; protected ?ProductHelper $productHelper; + protected ?QueueHelper $queueHelper; protected ?Queue $queue; protected ?DiagnosticsLogger $diag; protected ?AlgoliaCredentialsManager $algoliaCredentialsManager; @@ -40,11 +42,13 @@ protected function setUp(): void $this->algoliaCredentialsManager = $this->createMock(AlgoliaCredentialsManager::class); $this->indexBuilder = $this->createMock(IndexBuilder::class); $this->indexCollectionSizeCache = $this->createMock(IndexCollectionSize::class); + $this->queueHelper = $this->createMock(QueueHelper::class); $this->processor = new BatchQueueProcessor( $this->dataHelper, $this->configHelper, $this->productHelper, + $this->queueHelper, $this->queue, $this->diag, $this->algoliaCredentialsManager, @@ -136,6 +140,7 @@ public function testProcessBatchHandlesFullIndexing() $this->configHelper->method('isQueueActive')->willReturn(false); $this->indexCollectionSizeCache->expects($this->once())->method('get')->willReturn(10); $this->productHelper->method('getProductCollectionQuery')->willReturn($this->getMockCollection()); + $this->queueHelper->method('useTmpIndex')->willReturn(false); $invocations = $this->exactly(2); $this->queue->expects($invocations) @@ -176,6 +181,7 @@ public function testProcessBatchFullIndexingWithNoCache() $this->configHelper->method('isQueueActive')->willReturn(false); $this->indexCollectionSizeCache->expects($this->once())->method('get')->willReturn(IndexCollectionSize::NOT_FOUND); $this->productHelper->method('getProductCollectionQuery')->willReturn($this->getMockCollection(10, 1)); + $this->queueHelper->method('useTmpIndex')->willReturn(false); $this->queue->expects($this->exactly(2)) ->method('addToQueue'); @@ -194,6 +200,7 @@ public function testProcessBatchHandlesFullIndexingPaged() $this->configHelper->method('isQueueActive')->willReturn(false); $this->indexCollectionSizeCache->expects($this->once())->method('get')->willReturn(50); $this->productHelper->method('getProductCollectionQuery')->willReturn($this->getMockCollection()); + $this->queueHelper->method('useTmpIndex')->willReturn(false); $invocations = $this->exactly(6); $this->queue->expects($invocations) @@ -240,6 +247,8 @@ public function testProcessBatchMovesTempIndexIfQueueActive() $this->productHelper->method('getTempIndexName')->willReturn('tmp_index'); $this->productHelper->method('getIndexName')->willReturn('main_index'); + $this->queueHelper->method('useTmpIndex')->willReturn(true); + $invocations = $this->exactly(3); $this->queue->expects($invocations) ->method('addToQueue') From ae7a89c32d1c9242dd38c602fa5e1ef2be863733 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 6 Aug 2025 16:23:29 -0400 Subject: [PATCH 030/119] MAGE-1374 Add placeholder changelog entries --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3464ba3c1..fa5da7555 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # CHANGE LOG +## 3.17.0 + +### Features + +(This is a WIP until release) + +- Added an Algolia indexing cache for storing metadata to prevent extra queries. Large collections that run periodic full indexes can benefit from this cache. +- Price indexing is now optional. If the price attribute is not present in the product attributes, sorts, facets or custom ranking then that information will not be indexed in the product record. +- Creating a temporary index while performing a full index is now optional and can be enabled or disabled in the Magento admin. + ## 3.16.0 ### Features From b25428a65808004a558b74a047ea1cbccb9ec304 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 7 Aug 2025 18:16:25 +0200 Subject: [PATCH 031/119] MAGE-1109: Add Batching Optimizer CLI --- Console/Command/BatchingOptimizerCommand.php | 245 +++++++++++++++++++ etc/di.xml | 8 + 2 files changed, 253 insertions(+) create mode 100644 Console/Command/BatchingOptimizerCommand.php diff --git a/Console/Command/BatchingOptimizerCommand.php b/Console/Command/BatchingOptimizerCommand.php new file mode 100644 index 000000000..70ce8257f --- /dev/null +++ b/Console/Command/BatchingOptimizerCommand.php @@ -0,0 +1,245 @@ + Algolia Search > Advanced > Indexing Queue > Maximum number of records processed per indexing job\" according to various configurations."; + } + + protected function getStoreArgumentDescription(): string + { + return 'ID(s) for store(s) to optimize (optional), if no store is specified, all stores will be taken into account.'; + } + + protected function getAdditionalDefinition(): array + { + return []; + } + + /** + * @throws NoSuchEntityException|LocalizedException + */ + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->input = $input; + $this->output = $output; + $this->setAreaCode(); + + $storeIds = $this->getStoreIds($input); + + try { + $this->optimizeBatchingConfiguration($storeIds); + } catch (\Exception $e) { + $this->output->writeln('' . $e->getMessage() . ''); + return CLI::RETURN_FAILURE; + } + + return Cli::RETURN_SUCCESS; + } + + /** + * @param array $storeIds + * @return void + */ + protected function optimizeBatchingConfiguration(array $storeIds = []): void + { + if (count($storeIds)) { + foreach ($storeIds as $storeId) { + $this->optimizeBatchingForStore($storeId); + } + } else { + $this->optimizeBatchingForAllStores(); + } + } + + /** + * @return void + */ + protected function optimizeBatchingForAllStores(): void + { + $storeIds = array_keys($this->storeManager->getStores()); + + foreach ($storeIds as $storeId) { + $this->optimizeBatchingForStore($storeId); + } + } + + /** + * @param int $storeId + * @return void + * @throws AlgoliaException + * @throws NoSuchEntityException + */ + protected function optimizeBatchingForStore(int $storeId): void + { + $indexOptions = $this->indexOptionsBuilder->buildEntityIndexOptions($storeId); + $indexData = $this->getIndexData($indexOptions); + $configurablePercentile = $this->getConfigurablePercentile($indexData['entries'], $storeId); + + $this->output->writeln(' ====== ' . $this->storeNameFetcher->getStoreName($storeId) . ' ====== '); + $this->output->writeln('Index: ' . $indexOptions->getIndexName()); + $this->output->writeln('Number of records: ' . $indexData['entries'] + . ' (' . round($configurablePercentile) . '% of configurable products)'); + $this->output->writeln('Index data size: ' . $indexData['dataSize'] . 'B'); + + $averageRecordSize = (int)($indexData['dataSize']/$indexData['entries']); + $this->output->writeln('Average record size: ' . $averageRecordSize . 'B'); + + $maxBatchCount = (int)(self::MAX_BATCH_SIZE / $averageRecordSize); + $this->output->writeln(' ============ '); + $this->output->writeln('Estimated max batch count: ' . $maxBatchCount . ' objects'); + + $recommendedBatchCount = $this->getRecommendedBatchCount($maxBatchCount, $configurablePercentile); + $this->output->writeln('Recommended max batch count: ' . $recommendedBatchCount . ' objects'); + + // @todo : add a prompt to change the value in the Magento configuration (Maximum number of records processed per indexing job) + } + + /** + * Returns percentile of configurable products contained in the index + * + * @param int $nbProducts + * @param int $storeId + * @return float + */ + protected function getConfigurablePercentile(int $nbProducts, int $storeId): float + { + if (! isset($this->configurablePercentile[$storeId])) { + $collection = $this->productCollectionFactory->create(); + $collection->addStoreFilter($storeId); + $collection->addAttributeToFilter('type_id', ['eq' => 'configurable']); + + $this->configurablePercentile[$storeId] = $collection->count() * 100 / $nbProducts; + } + + return $this->configurablePercentile[$storeId]; + } + + /** + * Fetches index data from the Algolia Dashboard + * + * @param IndexOptions $indexOptions + * @return array + * @throws AlgoliaException + */ + protected function getIndexData(IndexOptions $indexOptions): array + { + if (is_null($this->indices)) { + $this->indices = $this->algoliaConnector->listIndexes(); + } + + foreach ($this->indices['items'] as $index) { + if ($index['name'] === $indexOptions->getIndexName()) { + return $index; + } + } + + throw new AlgoliaException('Index does not exist'); + } + + /** + * Calculates the recommended batch count according to: + * - the average record size + * - the max batch count + * - the percentile of configurable products (<10% and >90% are considered as "steady" so the margin is lower) + * + * @param int $maxBatchCount + * @param float $configurablePercentile + * @return int + */ + protected function getRecommendedBatchCount(int $maxBatchCount, float $configurablePercentile): int + { + $margin = $configurablePercentile > self::CONFIGURABLE_PERCENTILE_UPPER_BOUNDARY + || $configurablePercentile < self::CONFIGURABLE_PERCENTILE_LOWER_BOUNDARY ? + self::DEFAULT_MARGIN : + self::INCREASED_MARGIN; + + $recommendedBatchCount = (int) ($maxBatchCount * (1 - ($margin / 100))); + + if ($recommendedBatchCount >= 1000) { + $recommendedBatchCount = floor($recommendedBatchCount / 1000) * 1000; + } else { + $length = strlen(floor($recommendedBatchCount)); + $times = str_pad('1', $length, "0"); + $recommendedBatchCount = floor($recommendedBatchCount / $times) * $times; + } + + return $recommendedBatchCount; + } +} diff --git a/etc/di.xml b/etc/di.xml index efafc7619..a549363de 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -152,6 +152,7 @@ Algolia\AlgoliaSearch\Console\Command\ReplicaRebuildCommand Algolia\AlgoliaSearch\Console\Command\ReplicaDisableVirtualCommand Algolia\AlgoliaSearch\Console\Command\SynonymDeduplicateCommand + Algolia\AlgoliaSearch\Console\Command\BatchingOptimizerCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexProductsCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexCategoriesCommand @@ -200,6 +201,13 @@ + + + Algolia\AlgoliaSearch\Service\AlgoliaConnector\Proxy + Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder\Proxy + + + From 013ec7b91329e75e751870e1d39804b94e36755f Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 7 Aug 2025 18:31:33 +0200 Subject: [PATCH 032/119] MAGE-1109: Codacy bullying --- Console/Command/BatchingOptimizerCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Console/Command/BatchingOptimizerCommand.php b/Console/Command/BatchingOptimizerCommand.php index 70ce8257f..e61e1592f 100644 --- a/Console/Command/BatchingOptimizerCommand.php +++ b/Console/Command/BatchingOptimizerCommand.php @@ -200,7 +200,7 @@ protected function getConfigurablePercentile(int $nbProducts, int $storeId): flo */ protected function getIndexData(IndexOptions $indexOptions): array { - if (is_null($this->indices)) { + if ($this->indices === null) { $this->indices = $this->algoliaConnector->listIndexes(); } From 678c7ee614f08e65a0878c8bcb4021918034f624 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Mon, 11 Aug 2025 11:41:22 +0200 Subject: [PATCH 033/119] MAGE-1109: review feedback + prompt --- Console/Command/BatchingOptimizerCommand.php | 22 +++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Console/Command/BatchingOptimizerCommand.php b/Console/Command/BatchingOptimizerCommand.php index e61e1592f..b4ba6a3d8 100644 --- a/Console/Command/BatchingOptimizerCommand.php +++ b/Console/Command/BatchingOptimizerCommand.php @@ -3,11 +3,14 @@ namespace Algolia\AlgoliaSearch\Console\Command; use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; -use Algolia\AlgoliaSearch\Model\IndexOptions; +use Algolia\AlgoliaSearch\Api\Data\IndexOptionsInterface; +use Algolia\AlgoliaSearch\Helper\ConfigHelper; use Algolia\AlgoliaSearch\Service\AlgoliaConnector; use Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder; use Algolia\AlgoliaSearch\Service\StoreNameFetcher; use Magento\Catalog\Model\ResourceModel\Product\CollectionFactory; +use Magento\Framework\App\Config\ScopeConfigInterface; +use Magento\Framework\App\Config\Storage\WriterInterface; use Magento\Framework\App\State; use Magento\Framework\Console\Cli; use Magento\Framework\Exception\LocalizedException; @@ -15,6 +18,7 @@ use Magento\Store\Model\StoreManagerInterface; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Question\ConfirmationQuestion; class BatchingOptimizerCommand extends AbstractStoreCommand { @@ -32,7 +36,7 @@ class BatchingOptimizerCommand extends AbstractStoreCommand * Recommended Max batch size * https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/how-to/sending-records-in-batches/ */ - const int MAX_BATCH_SIZE = 10_000_000; + const int MAX_BATCH_SIZE = 10000000; //10MB /** * Arbitrary default margin to ensure not to exceed recommended batch size @@ -62,6 +66,7 @@ public function __construct( protected StoreManagerInterface $storeManager, protected IndexOptionsBuilder $indexOptionsBuilder, protected CollectionFactory $productCollectionFactory, + protected WriterInterface $configWriter, ?string $name = null ) { parent::__construct($state, $storeNameFetcher, $name); @@ -168,7 +173,14 @@ protected function optimizeBatchingForStore(int $storeId): void $recommendedBatchCount = $this->getRecommendedBatchCount($maxBatchCount, $configurablePercentile); $this->output->writeln('Recommended max batch count: ' . $recommendedBatchCount . ' objects'); - // @todo : add a prompt to change the value in the Magento configuration (Maximum number of records processed per indexing job) + if ($this->confirmOperation()) { + $this->configWriter->save( + ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, + $recommendedBatchCount, + ScopeConfigInterface::SCOPE_TYPE_DEFAULT, + $storeId + ); + } } /** @@ -194,11 +206,11 @@ protected function getConfigurablePercentile(int $nbProducts, int $storeId): flo /** * Fetches index data from the Algolia Dashboard * - * @param IndexOptions $indexOptions + * @param IndexOptionsInterface $indexOptions * @return array * @throws AlgoliaException */ - protected function getIndexData(IndexOptions $indexOptions): array + protected function getIndexData(IndexOptionsInterface $indexOptions): array { if ($this->indices === null) { $this->indices = $this->algoliaConnector->listIndexes(); From 01e64e4a5e5f5ed4b15aa4e7bc22c15104f22047 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Mon, 11 Aug 2025 15:18:32 +0200 Subject: [PATCH 034/119] MAGE-1109: fix value save --- Console/Command/BatchingOptimizerCommand.php | 26 +++++++++----------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/Console/Command/BatchingOptimizerCommand.php b/Console/Command/BatchingOptimizerCommand.php index b4ba6a3d8..17f6a8dba 100644 --- a/Console/Command/BatchingOptimizerCommand.php +++ b/Console/Command/BatchingOptimizerCommand.php @@ -9,7 +9,6 @@ use Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder; use Algolia\AlgoliaSearch\Service\StoreNameFetcher; use Magento\Catalog\Model\ResourceModel\Product\CollectionFactory; -use Magento\Framework\App\Config\ScopeConfigInterface; use Magento\Framework\App\Config\Storage\WriterInterface; use Magento\Framework\App\State; use Magento\Framework\Console\Cli; @@ -18,14 +17,13 @@ use Magento\Store\Model\StoreManagerInterface; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -use Symfony\Component\Console\Question\ConfirmationQuestion; class BatchingOptimizerCommand extends AbstractStoreCommand { /** * @var array|null */ - protected ?array $indices = null; + protected ?array $indices = []; /** * @var array|null @@ -36,28 +34,28 @@ class BatchingOptimizerCommand extends AbstractStoreCommand * Recommended Max batch size * https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/how-to/sending-records-in-batches/ */ - const int MAX_BATCH_SIZE = 10000000; //10MB + const MAX_BATCH_SIZE = 10000000; //10MB /** * Arbitrary default margin to ensure not to exceed recommended batch size */ - const int DEFAULT_MARGIN = 25; + const DEFAULT_MARGIN = 25; /** * Arbitrary increased margin to ensure not to exceed recommended batch size when catalog is a mix between configurables and other product types * (i.e. with a lot of record sizes variations) */ - const int INCREASED_MARGIN = 50; + const INCREASED_MARGIN = 50; /** * Arbitrary lower boundary where percentile of configurable products is considered "low enough" */ - const int CONFIGURABLE_PERCENTILE_LOWER_BOUNDARY = 10; + const CONFIGURABLE_PERCENTILE_LOWER_BOUNDARY = 10; /** * Arbitrary upper boundary where percentile of configurable products is considered "high enough" */ - const int CONFIGURABLE_PERCENTILE_UPPER_BOUNDARY = 90; + const CONFIGURABLE_PERCENTILE_UPPER_BOUNDARY = 90; public function __construct( protected AlgoliaConnector $algoliaConnector, @@ -154,7 +152,7 @@ protected function optimizeBatchingForAllStores(): void protected function optimizeBatchingForStore(int $storeId): void { $indexOptions = $this->indexOptionsBuilder->buildEntityIndexOptions($storeId); - $indexData = $this->getIndexData($indexOptions); + $indexData = $this->getIndexData($indexOptions, $storeId); $configurablePercentile = $this->getConfigurablePercentile($indexData['entries'], $storeId); $this->output->writeln(' ====== ' . $this->storeNameFetcher->getStoreName($storeId) . ' ====== '); @@ -177,7 +175,7 @@ protected function optimizeBatchingForStore(int $storeId): void $this->configWriter->save( ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, $recommendedBatchCount, - ScopeConfigInterface::SCOPE_TYPE_DEFAULT, + 'stores', $storeId ); } @@ -210,13 +208,13 @@ protected function getConfigurablePercentile(int $nbProducts, int $storeId): flo * @return array * @throws AlgoliaException */ - protected function getIndexData(IndexOptionsInterface $indexOptions): array + protected function getIndexData(IndexOptionsInterface $indexOptions, int $storeId): array { - if ($this->indices === null) { - $this->indices = $this->algoliaConnector->listIndexes(); + if (!isset($this->indices[$storeId])) { + $this->indices[$storeId] = $this->algoliaConnector->listIndexes($storeId); } - foreach ($this->indices['items'] as $index) { + foreach ($this->indices[$storeId]['items'] as $index) { if ($index['name'] === $indexOptions->getIndexName()) { return $index; } From 9b0f8f488c642651c1a890951cef04920cc89523 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Mon, 11 Aug 2025 21:59:18 -0400 Subject: [PATCH 035/119] MAGE-1385 Remove implicit nullable and modify unit tests to support both PHPUnit 9 and 10 --- Plugin/Cache/CacheCleanProductPlugin.php | 2 +- .../Product/BatchQueueProcessorTest.php | 39 ++++++++++--------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/Plugin/Cache/CacheCleanProductPlugin.php b/Plugin/Cache/CacheCleanProductPlugin.php index 03dab9832..6e5b77777 100644 --- a/Plugin/Cache/CacheCleanProductPlugin.php +++ b/Plugin/Cache/CacheCleanProductPlugin.php @@ -77,7 +77,7 @@ protected function hasEnablementChanged(array $orig, array $new): bool return $orig[$key] !== $new[$key]; } - protected function hasVisibilityChanged(array $orig, array $new, int $storeId = null): bool + protected function hasVisibilityChanged(array $orig, array $new, ?int $storeId = null): bool { if ($this->configHelper->includeNonVisibleProductsInIndex($storeId)) { return false; diff --git a/Test/Unit/Service/Product/BatchQueueProcessorTest.php b/Test/Unit/Service/Product/BatchQueueProcessorTest.php index a2e041de4..e1fb67d82 100644 --- a/Test/Unit/Service/Product/BatchQueueProcessorTest.php +++ b/Test/Unit/Service/Product/BatchQueueProcessorTest.php @@ -112,18 +112,19 @@ public function testProcessBatchHandlesDeltaIndexingPaged() $this->setupBasicIndexingConfig($pageSize); $this->productHelper->method('getParentProductIds')->willReturn([]); - $invocations = $this->exactly(5); - $this->queue->expects($invocations) + $invocationCount = 0; + $this->queue->expects($this->exactly(5)) ->method('addToQueue') ->with( IndexBuilder::class, 'buildIndexList', - $this->callback(function(array $arg) use ($invocations, $pageSize) { + $this->callback(function(array $arg) use (&$invocationCount, $pageSize) { + $invocationCount++; return array_key_exists('storeId', $arg) && array_key_exists('entityIds', $arg) && array_key_exists('options', $arg) && $arg['options']['pageSize'] === $pageSize - && $arg['options']['page'] === $invocations->getInvocationCount(); + && $arg['options']['page'] === $invocationCount; }) ); @@ -142,8 +143,8 @@ public function testProcessBatchHandlesFullIndexing() $this->productHelper->method('getProductCollectionQuery')->willReturn($this->getMockCollection()); $this->queueHelper->method('useTmpIndex')->willReturn(false); - $invocations = $this->exactly(2); - $this->queue->expects($invocations) + $invocationCount = 0; + $this->queue->expects($this->exactly(2)) ->method('addToQueue') ->willReturnCallback( function( @@ -152,8 +153,9 @@ function( array $data, int $dataSize, bool $isFullReindex) - use ($invocations) { - switch ($invocations->getInvocationCount()) { + use (&$invocationCount) { + $invocationCount++; + switch ($invocationCount) { case 1: $this->assertEquals(IndicesConfigurator::class, $className); $this->assertEquals('saveConfigurationToAlgolia', $method); @@ -202,8 +204,8 @@ public function testProcessBatchHandlesFullIndexingPaged() $this->productHelper->method('getProductCollectionQuery')->willReturn($this->getMockCollection()); $this->queueHelper->method('useTmpIndex')->willReturn(false); - $invocations = $this->exactly(6); - $this->queue->expects($invocations) + $invocationCount = 0; + $this->queue->expects($this->exactly(6)) ->method('addToQueue') ->willReturnCallback( function( @@ -212,9 +214,9 @@ function( array $data, int $dataSize, bool $isFullReindex) - use ($invocations, $pageSize) { - $invocation = $invocations->getInvocationCount(); - switch ($invocation) { + use (&$invocationCount, $pageSize) { + $invocationCount++; + switch ($invocationCount) { case 1: $this->assertEquals(IndicesConfigurator::class, $className); $this->assertEquals('saveConfigurationToAlgolia', $method); @@ -224,7 +226,7 @@ function( $this->assertEquals('buildIndexFull', $method); $this->assertArrayHasKey('options', $data); $this->assertEquals($pageSize, $data['options']['pageSize']); - $this->assertEquals($invocation - 1, $data['options']['page']); + $this->assertEquals($invocationCount - 1, $data['options']['page']); break; } } @@ -249,8 +251,8 @@ public function testProcessBatchMovesTempIndexIfQueueActive() $this->queueHelper->method('useTmpIndex')->willReturn(true); - $invocations = $this->exactly(3); - $this->queue->expects($invocations) + $invocationCount = 0; + $this->queue->expects($this->exactly(3)) ->method('addToQueue') ->willReturnCallback( function( @@ -259,8 +261,9 @@ function( array $data, int $dataSize, bool $isFullReindex) - use ($invocations) { - if ($invocations->getInvocationCount() === 3) { + use (&$invocationCount) { + $invocationCount++; + if ($invocationCount === 3) { $this->assertEquals(IndexMover::class, $className); $this->assertEquals('moveIndexWithSetSettings', $method); $this->assertArrayHasKey('tmpIndexName', $data); From eed8bd963e7f978c604aebc82fb8dad7d40e1a04 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Tue, 12 Aug 2025 15:32:06 +0200 Subject: [PATCH 036/119] MAGE-1109: added batching scan CLI command --- Console/Command/BatchingCommandTrait.php | 54 ++++ Console/Command/BatchingOptimizerCommand.php | 67 ++--- Console/Command/BatchingScanCommand.php | 297 +++++++++++++++++++ etc/di.xml | 15 + 4 files changed, 392 insertions(+), 41 deletions(-) create mode 100644 Console/Command/BatchingCommandTrait.php create mode 100644 Console/Command/BatchingScanCommand.php diff --git a/Console/Command/BatchingCommandTrait.php b/Console/Command/BatchingCommandTrait.php new file mode 100644 index 000000000..c1778a637 --- /dev/null +++ b/Console/Command/BatchingCommandTrait.php @@ -0,0 +1,54 @@ +configHelper->includeNonVisibleProductsInIndex(); + $collection = $this->productHelper->getProductCollectionQuery($storeId, null, $onlyVisible); + if (count($productTypes) > 0) { + $collection->addAttributeToFilter('type_id', ['in' => $productTypes]); + } + + return $collection; + } +} diff --git a/Console/Command/BatchingOptimizerCommand.php b/Console/Command/BatchingOptimizerCommand.php index 17f6a8dba..102ded257 100644 --- a/Console/Command/BatchingOptimizerCommand.php +++ b/Console/Command/BatchingOptimizerCommand.php @@ -5,10 +5,10 @@ use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; use Algolia\AlgoliaSearch\Api\Data\IndexOptionsInterface; use Algolia\AlgoliaSearch\Helper\ConfigHelper; +use Algolia\AlgoliaSearch\Helper\Entity\ProductHelper; use Algolia\AlgoliaSearch\Service\AlgoliaConnector; use Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder; use Algolia\AlgoliaSearch\Service\StoreNameFetcher; -use Magento\Catalog\Model\ResourceModel\Product\CollectionFactory; use Magento\Framework\App\Config\Storage\WriterInterface; use Magento\Framework\App\State; use Magento\Framework\Console\Cli; @@ -20,6 +20,7 @@ class BatchingOptimizerCommand extends AbstractStoreCommand { + use BatchingCommandTrait; /** * @var array|null */ @@ -28,34 +29,17 @@ class BatchingOptimizerCommand extends AbstractStoreCommand /** * @var array|null */ - protected ?array $configurablePercentile = []; + protected ?array $complexPercentile = []; /** - * Recommended Max batch size - * https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/how-to/sending-records-in-batches/ + * Arbitrary lower boundary where percentile of complex products is considered "low enough" */ - const MAX_BATCH_SIZE = 10000000; //10MB + const COMPLEX_PERCENTILE_LOWER_BOUNDARY = 10; /** - * Arbitrary default margin to ensure not to exceed recommended batch size + * Arbitrary upper boundary where percentile of complex products is considered "high enough" */ - const DEFAULT_MARGIN = 25; - - /** - * Arbitrary increased margin to ensure not to exceed recommended batch size when catalog is a mix between configurables and other product types - * (i.e. with a lot of record sizes variations) - */ - const INCREASED_MARGIN = 50; - - /** - * Arbitrary lower boundary where percentile of configurable products is considered "low enough" - */ - const CONFIGURABLE_PERCENTILE_LOWER_BOUNDARY = 10; - - /** - * Arbitrary upper boundary where percentile of configurable products is considered "high enough" - */ - const CONFIGURABLE_PERCENTILE_UPPER_BOUNDARY = 90; + const COMPLEX_PERCENTILE_UPPER_BOUNDARY = 90; public function __construct( protected AlgoliaConnector $algoliaConnector, @@ -63,7 +47,8 @@ public function __construct( protected StoreNameFetcher $storeNameFetcher, protected StoreManagerInterface $storeManager, protected IndexOptionsBuilder $indexOptionsBuilder, - protected CollectionFactory $productCollectionFactory, + protected ProductHelper $productHelper, + protected ConfigHelper $configHelper, protected WriterInterface $configWriter, ?string $name = null ) { @@ -153,12 +138,12 @@ protected function optimizeBatchingForStore(int $storeId): void { $indexOptions = $this->indexOptionsBuilder->buildEntityIndexOptions($storeId); $indexData = $this->getIndexData($indexOptions, $storeId); - $configurablePercentile = $this->getConfigurablePercentile($indexData['entries'], $storeId); + $complexPercentile = $this->getComplexPercentile($indexData['entries'], $storeId); $this->output->writeln(' ====== ' . $this->storeNameFetcher->getStoreName($storeId) . ' ====== '); $this->output->writeln('Index: ' . $indexOptions->getIndexName()); $this->output->writeln('Number of records: ' . $indexData['entries'] - . ' (' . round($configurablePercentile) . '% of configurable products)'); + . ' (' . round($complexPercentile) . '% of complex products)'); $this->output->writeln('Index data size: ' . $indexData['dataSize'] . 'B'); $averageRecordSize = (int)($indexData['dataSize']/$indexData['entries']); @@ -168,7 +153,7 @@ protected function optimizeBatchingForStore(int $storeId): void $this->output->writeln(' ============ '); $this->output->writeln('Estimated max batch count: ' . $maxBatchCount . ' objects'); - $recommendedBatchCount = $this->getRecommendedBatchCount($maxBatchCount, $configurablePercentile); + $recommendedBatchCount = $this->getRecommendedBatchCount($maxBatchCount, $complexPercentile); $this->output->writeln('Recommended max batch count: ' . $recommendedBatchCount . ' objects'); if ($this->confirmOperation()) { @@ -182,23 +167,23 @@ protected function optimizeBatchingForStore(int $storeId): void } /** - * Returns percentile of configurable products contained in the index + * Returns percentile of complex products (configurable, bundle, grouped) contained in the index * * @param int $nbProducts * @param int $storeId * @return float */ - protected function getConfigurablePercentile(int $nbProducts, int $storeId): float + protected function getComplexPercentile(int $nbProducts, int $storeId): float { - if (! isset($this->configurablePercentile[$storeId])) { - $collection = $this->productCollectionFactory->create(); - $collection->addStoreFilter($storeId); - $collection->addAttributeToFilter('type_id', ['eq' => 'configurable']); - - $this->configurablePercentile[$storeId] = $collection->count() * 100 / $nbProducts; + if (! isset($this->complexPercentile[$storeId])) { + $this->complexPercentile[$storeId] = + $this->getProductsCollectionForStore( + $storeId, + self::PRODUCTS_COMPLEX_TYPES) + ->count() * 100 / $nbProducts; } - return $this->configurablePercentile[$storeId]; + return $this->complexPercentile[$storeId]; } /** @@ -227,16 +212,16 @@ protected function getIndexData(IndexOptionsInterface $indexOptions, int $storeI * Calculates the recommended batch count according to: * - the average record size * - the max batch count - * - the percentile of configurable products (<10% and >90% are considered as "steady" so the margin is lower) + * - the percentile of complex products (<10% and >90% are considered as "steady" so the margin is lower) * * @param int $maxBatchCount - * @param float $configurablePercentile + * @param float $complexPercentile * @return int */ - protected function getRecommendedBatchCount(int $maxBatchCount, float $configurablePercentile): int + protected function getRecommendedBatchCount(int $maxBatchCount, float $complexPercentile): int { - $margin = $configurablePercentile > self::CONFIGURABLE_PERCENTILE_UPPER_BOUNDARY - || $configurablePercentile < self::CONFIGURABLE_PERCENTILE_LOWER_BOUNDARY ? + $margin = $complexPercentile > self::COMPLEX_PERCENTILE_UPPER_BOUNDARY + || $complexPercentile < self::COMPLEX_PERCENTILE_LOWER_BOUNDARY ? self::DEFAULT_MARGIN : self::INCREASED_MARGIN; diff --git a/Console/Command/BatchingScanCommand.php b/Console/Command/BatchingScanCommand.php new file mode 100644 index 000000000..d2a37d093 --- /dev/null +++ b/Console/Command/BatchingScanCommand.php @@ -0,0 +1,297 @@ +input = $input; + $this->output = $output; + $this->setAreaCode(); + + $storeIds = $this->getStoreIds($input); + + try { + $this->scanProductRecords($storeIds); + } catch (\Exception $e) { + $this->output->writeln('' . $e->getMessage() . ''); + return CLI::RETURN_FAILURE; + } + + return Cli::RETURN_SUCCESS; + } + + /** + * @param array $storeIds + * @return void + */ + protected function scanProductRecords(array $storeIds = []): void + { + if (count($storeIds)) { + foreach ($storeIds as $storeId) { + $this->scanProductRecordsForStore($storeId); + } + } else { + $this->scanProductRecordsForAllStores(); + } + } + + /** + * @return void + */ + protected function scanProductRecordsForAllStores(): void + { + $storeIds = array_keys($this->storeManager->getStores()); + + foreach ($storeIds as $storeId) { + $this->scanProductRecordsForStore($storeId); + } + } + + /** + * @param int $storeId + * @return void + */ + protected function scanProductRecordsForStore(int $storeId): void + { + if (!isset($this->storeCounts[$storeId])) { + $this->setStoreCounts($storeId); + } + + $this->output->writeln(' '); + $storeName = $this->storeNameFetcher->getStoreName($storeId); + $this->output->writeln(' ====== Products for store ' . $storeName . ' ====== '); + $this->output->writeln('Simple Products: ' . $this->storeCounts[$storeId]['simple'] . ' (' . round($this->storeCounts[$storeId]['simple_percentage'], 2) . '% of total)'); + $this->output->writeln('Complex Products: ' . $this->storeCounts[$storeId]['complex'] . ' (' . round($this->storeCounts[$storeId]['complex_percentage'], 2) . '% of total)'); + + $this->output->writeln(' ============ '); + $this->output->writeln('Total: ' . $this->storeCounts[$storeId]['total'] . ' products'); + + $this->output->writeln(' ============ '); + + $sample = $this->storeCounts[$storeId]['sample']; + + if (count($sample) > 0) { + $this->output->writeln('Sample (' . count($sample) . ' products):'); + foreach ($sample as $sku => $size) { + $this->output->writeln(' - ' . $size . 'B (sku: ' . $sku . ')'); + } + } + + $this->output->writeln(' ============ '); + $sizeAverage = $this->getSizeAverage($sample); + $this->output->writeln('Average record size : ' . $sizeAverage . 'B'); + + $estimatedBatchCount = $this->getEstimatedMaxBatchCount($sizeAverage); + $this->output->writeln('Estimated Max batch count : ' . $estimatedBatchCount . ' records'); + + $standardDeviation = $this->getStandardDeviation($sample, $sizeAverage); + $this->output->writeln('Standard Deviation : ' . $standardDeviation); + + $recommendedBatchCountLow = $this->getRecommendedBatchCount($sizeAverage, $standardDeviation, self::INCREASED_MARGIN); + $recommendedBatchCountHigh = $this->getRecommendedBatchCount($sizeAverage, $standardDeviation); + $this->output->writeln(' ============ '); + $this->output->writeln('Recommended batch count (low) : ' . $recommendedBatchCountLow . ' records'); + $this->output->writeln('Recommended batch count (high) : ' . $recommendedBatchCountHigh . ' records'); + $this->output->writeln(' '); + $this->output->writeln( + 'This will override your "Maximum number of records processed per indexing job" configuration to ' . $recommendedBatchCountLow . ' for store "' . $storeName . '".'); + + if ($this->confirmOperation()) { + $this->configWriter->save( + ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, + $recommendedBatchCountLow, + 'stores', + $storeId + ); + } + } + + /** + * @param int $storeId + * @return void + * @throws AlgoliaException + * @throws DiagnosticsException + * @throws LocalizedException + * @throws NoSuchEntityException + */ + protected function setStoreCounts(int $storeId): void + { + $simpleProducts = $this->getProductsCollectionForStore($storeId, self::PRODUCTS_SIMPLE_TYPES); + $complexProducts = $this->getProductsCollectionForStore($storeId, self::PRODUCTS_COMPLEX_TYPES); + + $this->storeCounts[$storeId] = [ + 'simple' => $simpleProducts->count(), + 'complex' => $complexProducts->count() + ]; + + $this->storeCounts[$storeId]['total'] = + (int) $this->storeCounts[$storeId]['simple'] + (int) $this->storeCounts[$storeId]['complex']; + + $this->storeCounts[$storeId]['simple_percentage'] = + ($this->storeCounts[$storeId]['simple'] * 100) / $this->storeCounts[$storeId]['total']; + + $this->storeCounts[$storeId]['complex_percentage'] = + ($this->storeCounts[$storeId]['complex'] * 100) / $this->storeCounts[$storeId]['total']; + + $simpleSampleSize = (int)round(self::DEFAULT_SAMPLE_SIZE * ($this->storeCounts[$storeId]['simple_percentage'] / 100)); + $complexSampleSize = (int)round(self::DEFAULT_SAMPLE_SIZE * ($this->storeCounts[$storeId]['complex_percentage'] / 100)); + + $this->storeCounts[$storeId]['simple_sample_size'] = $simpleSampleSize; + $this->storeCounts[$storeId]['complex_sample_size'] = $complexSampleSize; + + $this->storeCounts[$storeId]['sample'] = array_merge( + $this->getProductsSizes($simpleProducts, $simpleSampleSize), + $this->getProductsSizes($complexProducts, $complexSampleSize) + ); + } + + /** + * @param Collection $products + * @param int $sampleSize + * @return array + * @throws LocalizedException + * @throws NoSuchEntityException + * @throws DiagnosticsException + * @throws AlgoliaException + */ + protected function getProductsSizes(Collection $products, int $sampleSize): array + { + $stats = []; + $limit = 0; + + foreach ($products as $product) { + if ($limit >= $sampleSize) { + break; + } + + $serializedRecord = json_encode($this->recordBuilder->buildRecord($product)); + + if (function_exists('mb_strlen')) { + $size = mb_strlen($serializedRecord, '8bit'); + } else { + $size = strlen($serializedRecord); + } + + $stats[$product->getSku()] = $size; + $limit++; + } + + return $stats; + } + + /** + * @param array $sizes + * @return int + */ + protected function getSizeAverage(array $sizes): int + { + return (int) round(array_sum(array_values($sizes)) / count($sizes)); + } + + /** + * @param int $averageSize + * @return int + */ + protected function getEstimatedMaxBatchCount(int $averageSize): int + { + return (int) round(self::MAX_BATCH_SIZE / $averageSize); + } + + /** + * @param array $sizes + * @param int $averageSize + * @return float + */ + protected function getStandardDeviation(array $sizes, int $averageSize): float + { + $sum = 0; + foreach ($sizes as $size) { + $sum += ($size - abs($averageSize)) * ($size - abs($averageSize)); + } + + return round(sqrt($sum / count($sizes)), 2); + } + + /** + * @param int $averageSize + * @param float $standardDeviation + * @param int $margin + * @return int + */ + protected function getRecommendedBatchCount(int $averageSize, float $standardDeviation, int $margin = self::DEFAULT_MARGIN): int + { + return (int) (self::MAX_BATCH_SIZE / ($averageSize + ($margin/100) * $standardDeviation)); + } +} diff --git a/etc/di.xml b/etc/di.xml index a549363de..70e59dc18 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -153,6 +153,7 @@ Algolia\AlgoliaSearch\Console\Command\ReplicaDisableVirtualCommand Algolia\AlgoliaSearch\Console\Command\SynonymDeduplicateCommand Algolia\AlgoliaSearch\Console\Command\BatchingOptimizerCommand + Algolia\AlgoliaSearch\Console\Command\BatchingScanCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexProductsCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexCategoriesCommand @@ -204,7 +205,21 @@ Algolia\AlgoliaSearch\Service\AlgoliaConnector\Proxy + Algolia\AlgoliaSearch\Service\StoreNameFetcher\Proxy + Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder\Proxy + Algolia\AlgoliaSearch\Helper\Entity\ProductHelper\Proxy + Algolia\AlgoliaSearch\Helper\ConfigHelper\Proxy + + + + + + Algolia\AlgoliaSearch\Service\AlgoliaConnector\Proxy + Algolia\AlgoliaSearch\Service\StoreNameFetcher\Proxy Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder\Proxy + Algolia\AlgoliaSearch\Helper\Entity\ProductHelper\Proxy + Algolia\AlgoliaSearch\Service\Product\RecordBuilder\Proxy + Algolia\AlgoliaSearch\Helper\ConfigHelper\Proxy From 6df50cd2b6efb914c45a532a698a8d382e85c755 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 13 Aug 2025 11:23:10 +0200 Subject: [PATCH 037/119] MAGE-1109: move trait in right directory --- Console/Command/BatchingOptimizerCommand.php | 3 ++- Console/Command/BatchingScanCommand.php | 1 + Console/{Command => Traits}/BatchingCommandTrait.php | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) rename Console/{Command => Traits}/BatchingCommandTrait.php (96%) diff --git a/Console/Command/BatchingOptimizerCommand.php b/Console/Command/BatchingOptimizerCommand.php index 102ded257..0eb3b53e0 100644 --- a/Console/Command/BatchingOptimizerCommand.php +++ b/Console/Command/BatchingOptimizerCommand.php @@ -2,8 +2,9 @@ namespace Algolia\AlgoliaSearch\Console\Command; -use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; use Algolia\AlgoliaSearch\Api\Data\IndexOptionsInterface; +use Algolia\AlgoliaSearch\Console\Traits\BatchingCommandTrait; +use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; use Algolia\AlgoliaSearch\Helper\ConfigHelper; use Algolia\AlgoliaSearch\Helper\Entity\ProductHelper; use Algolia\AlgoliaSearch\Service\AlgoliaConnector; diff --git a/Console/Command/BatchingScanCommand.php b/Console/Command/BatchingScanCommand.php index d2a37d093..1be4d2180 100644 --- a/Console/Command/BatchingScanCommand.php +++ b/Console/Command/BatchingScanCommand.php @@ -2,6 +2,7 @@ namespace Algolia\AlgoliaSearch\Console\Command; +use Algolia\AlgoliaSearch\Console\Traits\BatchingCommandTrait; use Algolia\AlgoliaSearch\Exception\DiagnosticsException; use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; use Algolia\AlgoliaSearch\Helper\ConfigHelper; diff --git a/Console/Command/BatchingCommandTrait.php b/Console/Traits/BatchingCommandTrait.php similarity index 96% rename from Console/Command/BatchingCommandTrait.php rename to Console/Traits/BatchingCommandTrait.php index c1778a637..38975202a 100644 --- a/Console/Command/BatchingCommandTrait.php +++ b/Console/Traits/BatchingCommandTrait.php @@ -1,6 +1,6 @@ Date: Wed, 13 Aug 2025 14:02:52 +0200 Subject: [PATCH 038/119] MAGE-1109: add large sample size option --- Console/Command/BatchingScanCommand.php | 33 +++++++++++++++++++++---- Console/Traits/BatchingCommandTrait.php | 3 +++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/Console/Command/BatchingScanCommand.php b/Console/Command/BatchingScanCommand.php index 1be4d2180..bf13b5b96 100644 --- a/Console/Command/BatchingScanCommand.php +++ b/Console/Command/BatchingScanCommand.php @@ -18,14 +18,21 @@ use Magento\Framework\Exception\LocalizedException; use Magento\Framework\Exception\NoSuchEntityException; use Magento\Store\Model\StoreManagerInterface; +use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; class BatchingScanCommand extends AbstractStoreCommand { use BatchingCommandTrait; - const DEFAULT_SAMPLE_SIZE = 10; + const DEFAULT_SAMPLE_SIZE = 20; + const LARGE_SAMPLE_SIZE = 100; + + protected const LARGE_SAMPLE_OPTION = 'l-sample'; + + protected const LARGE_SAMPLE_OPTION_SHORTCUT = 'l'; /** * @var array|null @@ -69,7 +76,14 @@ protected function getStoreArgumentDescription(): string protected function getAdditionalDefinition(): array { - return []; + return [ + new InputOption( + self::LARGE_SAMPLE_OPTION, + '-' . self::LARGE_SAMPLE_OPTION_SHORTCUT, + InputOption::VALUE_NONE, + 'Use a large sample of products (100)' + ) + ]; } /** @@ -126,12 +140,18 @@ protected function scanProductRecordsForAllStores(): void */ protected function scanProductRecordsForStore(int $storeId): void { + $storeName = $this->storeNameFetcher->getStoreName($storeId); + + if (!$this->configHelper->isIndexingEnabled($storeId)) { + $this->output->writeln('Indexing is disabled for store ' . $storeName . ''); + return; + } + if (!isset($this->storeCounts[$storeId])) { $this->setStoreCounts($storeId); } $this->output->writeln(' '); - $storeName = $this->storeNameFetcher->getStoreName($storeId); $this->output->writeln(' ====== Products for store ' . $storeName . ' ====== '); $this->output->writeln('Simple Products: ' . $this->storeCounts[$storeId]['simple'] . ' (' . round($this->storeCounts[$storeId]['simple_percentage'], 2) . '% of total)'); $this->output->writeln('Complex Products: ' . $this->storeCounts[$storeId]['complex'] . ' (' . round($this->storeCounts[$storeId]['complex_percentage'], 2) . '% of total)'); @@ -206,8 +226,11 @@ protected function setStoreCounts(int $storeId): void $this->storeCounts[$storeId]['complex_percentage'] = ($this->storeCounts[$storeId]['complex'] * 100) / $this->storeCounts[$storeId]['total']; - $simpleSampleSize = (int)round(self::DEFAULT_SAMPLE_SIZE * ($this->storeCounts[$storeId]['simple_percentage'] / 100)); - $complexSampleSize = (int)round(self::DEFAULT_SAMPLE_SIZE * ($this->storeCounts[$storeId]['complex_percentage'] / 100)); + $sampleSize = $this->input->getOption(self::LARGE_SAMPLE_OPTION) ? + self::LARGE_SAMPLE_SIZE : + self::DEFAULT_SAMPLE_SIZE; + $simpleSampleSize = (int)round($sampleSize * ($this->storeCounts[$storeId]['simple_percentage'] / 100)); + $complexSampleSize = (int)round($sampleSize * ($this->storeCounts[$storeId]['complex_percentage'] / 100)); $this->storeCounts[$storeId]['simple_sample_size'] = $simpleSampleSize; $this->storeCounts[$storeId]['complex_sample_size'] = $complexSampleSize; diff --git a/Console/Traits/BatchingCommandTrait.php b/Console/Traits/BatchingCommandTrait.php index 38975202a..cee392e80 100644 --- a/Console/Traits/BatchingCommandTrait.php +++ b/Console/Traits/BatchingCommandTrait.php @@ -49,6 +49,9 @@ protected function getProductsCollectionForStore(int $storeId, array $productTyp $collection->addAttributeToFilter('type_id', ['in' => $productTypes]); } + // Randomize the results to get a more "diverse" sample + $collection->getSelect()->orderRand(); + return $collection; } } From 9d80d0165dcf0bfc080a0c0e28cb6a681c898ae1 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 13 Aug 2025 14:07:29 +0200 Subject: [PATCH 039/119] MAGE-1109: remove old class --- ...ommand.php => BatchingOptimizeCommand.php} | 4 +- Console/Command/BatchingOptimizerCommand.php | 241 ------------------ etc/di.xml | 15 +- 3 files changed, 4 insertions(+), 256 deletions(-) rename Console/Command/{BatchingScanCommand.php => BatchingOptimizeCommand.php} (99%) delete mode 100644 Console/Command/BatchingOptimizerCommand.php diff --git a/Console/Command/BatchingScanCommand.php b/Console/Command/BatchingOptimizeCommand.php similarity index 99% rename from Console/Command/BatchingScanCommand.php rename to Console/Command/BatchingOptimizeCommand.php index bf13b5b96..9edd2a14f 100644 --- a/Console/Command/BatchingScanCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -23,7 +23,7 @@ use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; -class BatchingScanCommand extends AbstractStoreCommand +class BatchingOptimizeCommand extends AbstractStoreCommand { use BatchingCommandTrait; @@ -61,7 +61,7 @@ protected function getCommandPrefix(): string protected function getCommandName(): string { - return 'scan'; + return 'optimize'; } protected function getCommandDescription(): string diff --git a/Console/Command/BatchingOptimizerCommand.php b/Console/Command/BatchingOptimizerCommand.php deleted file mode 100644 index 0eb3b53e0..000000000 --- a/Console/Command/BatchingOptimizerCommand.php +++ /dev/null @@ -1,241 +0,0 @@ - Algolia Search > Advanced > Indexing Queue > Maximum number of records processed per indexing job\" according to various configurations."; - } - - protected function getStoreArgumentDescription(): string - { - return 'ID(s) for store(s) to optimize (optional), if no store is specified, all stores will be taken into account.'; - } - - protected function getAdditionalDefinition(): array - { - return []; - } - - /** - * @throws NoSuchEntityException|LocalizedException - */ - protected function execute(InputInterface $input, OutputInterface $output): int - { - $this->input = $input; - $this->output = $output; - $this->setAreaCode(); - - $storeIds = $this->getStoreIds($input); - - try { - $this->optimizeBatchingConfiguration($storeIds); - } catch (\Exception $e) { - $this->output->writeln('' . $e->getMessage() . ''); - return CLI::RETURN_FAILURE; - } - - return Cli::RETURN_SUCCESS; - } - - /** - * @param array $storeIds - * @return void - */ - protected function optimizeBatchingConfiguration(array $storeIds = []): void - { - if (count($storeIds)) { - foreach ($storeIds as $storeId) { - $this->optimizeBatchingForStore($storeId); - } - } else { - $this->optimizeBatchingForAllStores(); - } - } - - /** - * @return void - */ - protected function optimizeBatchingForAllStores(): void - { - $storeIds = array_keys($this->storeManager->getStores()); - - foreach ($storeIds as $storeId) { - $this->optimizeBatchingForStore($storeId); - } - } - - /** - * @param int $storeId - * @return void - * @throws AlgoliaException - * @throws NoSuchEntityException - */ - protected function optimizeBatchingForStore(int $storeId): void - { - $indexOptions = $this->indexOptionsBuilder->buildEntityIndexOptions($storeId); - $indexData = $this->getIndexData($indexOptions, $storeId); - $complexPercentile = $this->getComplexPercentile($indexData['entries'], $storeId); - - $this->output->writeln(' ====== ' . $this->storeNameFetcher->getStoreName($storeId) . ' ====== '); - $this->output->writeln('Index: ' . $indexOptions->getIndexName()); - $this->output->writeln('Number of records: ' . $indexData['entries'] - . ' (' . round($complexPercentile) . '% of complex products)'); - $this->output->writeln('Index data size: ' . $indexData['dataSize'] . 'B'); - - $averageRecordSize = (int)($indexData['dataSize']/$indexData['entries']); - $this->output->writeln('Average record size: ' . $averageRecordSize . 'B'); - - $maxBatchCount = (int)(self::MAX_BATCH_SIZE / $averageRecordSize); - $this->output->writeln(' ============ '); - $this->output->writeln('Estimated max batch count: ' . $maxBatchCount . ' objects'); - - $recommendedBatchCount = $this->getRecommendedBatchCount($maxBatchCount, $complexPercentile); - $this->output->writeln('Recommended max batch count: ' . $recommendedBatchCount . ' objects'); - - if ($this->confirmOperation()) { - $this->configWriter->save( - ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, - $recommendedBatchCount, - 'stores', - $storeId - ); - } - } - - /** - * Returns percentile of complex products (configurable, bundle, grouped) contained in the index - * - * @param int $nbProducts - * @param int $storeId - * @return float - */ - protected function getComplexPercentile(int $nbProducts, int $storeId): float - { - if (! isset($this->complexPercentile[$storeId])) { - $this->complexPercentile[$storeId] = - $this->getProductsCollectionForStore( - $storeId, - self::PRODUCTS_COMPLEX_TYPES) - ->count() * 100 / $nbProducts; - } - - return $this->complexPercentile[$storeId]; - } - - /** - * Fetches index data from the Algolia Dashboard - * - * @param IndexOptionsInterface $indexOptions - * @return array - * @throws AlgoliaException - */ - protected function getIndexData(IndexOptionsInterface $indexOptions, int $storeId): array - { - if (!isset($this->indices[$storeId])) { - $this->indices[$storeId] = $this->algoliaConnector->listIndexes($storeId); - } - - foreach ($this->indices[$storeId]['items'] as $index) { - if ($index['name'] === $indexOptions->getIndexName()) { - return $index; - } - } - - throw new AlgoliaException('Index does not exist'); - } - - /** - * Calculates the recommended batch count according to: - * - the average record size - * - the max batch count - * - the percentile of complex products (<10% and >90% are considered as "steady" so the margin is lower) - * - * @param int $maxBatchCount - * @param float $complexPercentile - * @return int - */ - protected function getRecommendedBatchCount(int $maxBatchCount, float $complexPercentile): int - { - $margin = $complexPercentile > self::COMPLEX_PERCENTILE_UPPER_BOUNDARY - || $complexPercentile < self::COMPLEX_PERCENTILE_LOWER_BOUNDARY ? - self::DEFAULT_MARGIN : - self::INCREASED_MARGIN; - - $recommendedBatchCount = (int) ($maxBatchCount * (1 - ($margin / 100))); - - if ($recommendedBatchCount >= 1000) { - $recommendedBatchCount = floor($recommendedBatchCount / 1000) * 1000; - } else { - $length = strlen(floor($recommendedBatchCount)); - $times = str_pad('1', $length, "0"); - $recommendedBatchCount = floor($recommendedBatchCount / $times) * $times; - } - - return $recommendedBatchCount; - } -} diff --git a/etc/di.xml b/etc/di.xml index 70e59dc18..54a985f5e 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -152,8 +152,7 @@ Algolia\AlgoliaSearch\Console\Command\ReplicaRebuildCommand Algolia\AlgoliaSearch\Console\Command\ReplicaDisableVirtualCommand Algolia\AlgoliaSearch\Console\Command\SynonymDeduplicateCommand - Algolia\AlgoliaSearch\Console\Command\BatchingOptimizerCommand - Algolia\AlgoliaSearch\Console\Command\BatchingScanCommand + Algolia\AlgoliaSearch\Console\Command\BatchingOptimizeCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexProductsCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexCategoriesCommand @@ -202,17 +201,7 @@ - - - Algolia\AlgoliaSearch\Service\AlgoliaConnector\Proxy - Algolia\AlgoliaSearch\Service\StoreNameFetcher\Proxy - Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder\Proxy - Algolia\AlgoliaSearch\Helper\Entity\ProductHelper\Proxy - Algolia\AlgoliaSearch\Helper\ConfigHelper\Proxy - - - - + Algolia\AlgoliaSearch\Service\AlgoliaConnector\Proxy Algolia\AlgoliaSearch\Service\StoreNameFetcher\Proxy From f8fc6d2c8c7fc2d35074ebd9c88a60b30d78cce4 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 13 Aug 2025 14:10:58 +0200 Subject: [PATCH 040/119] MAGE-1109: remove unnecessary trait --- Console/Command/BatchingOptimizeCommand.php | 51 ++++++++++++++++-- Console/Traits/BatchingCommandTrait.php | 57 --------------------- 2 files changed, 48 insertions(+), 60 deletions(-) delete mode 100644 Console/Traits/BatchingCommandTrait.php diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index 9edd2a14f..95c5b53c2 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -2,7 +2,6 @@ namespace Algolia\AlgoliaSearch\Console\Command; -use Algolia\AlgoliaSearch\Console\Traits\BatchingCommandTrait; use Algolia\AlgoliaSearch\Exception\DiagnosticsException; use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; use Algolia\AlgoliaSearch\Helper\ConfigHelper; @@ -18,14 +17,28 @@ use Magento\Framework\Exception\LocalizedException; use Magento\Framework\Exception\NoSuchEntityException; use Magento\Store\Model\StoreManagerInterface; -use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; class BatchingOptimizeCommand extends AbstractStoreCommand { - use BatchingCommandTrait; + /** + * Recommended Max batch size + * https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/how-to/sending-records-in-batches/ + */ + const MAX_BATCH_SIZE = 10000000; //10MB + + /** + * Arbitrary default margin to ensure not to exceed recommended batch size + */ + const DEFAULT_MARGIN = 25; + + /** + * Arbitrary increased margin to ensure not to exceed recommended batch size when catalog is a mix between complex and other product types + * (i.e. with a lot of record sizes variations) + */ + const INCREASED_MARGIN = 50; const DEFAULT_SAMPLE_SIZE = 20; const LARGE_SAMPLE_SIZE = 100; @@ -34,6 +47,19 @@ class BatchingOptimizeCommand extends AbstractStoreCommand protected const LARGE_SAMPLE_OPTION_SHORTCUT = 'l'; + const PRODUCTS_SIMPLE_TYPES = [ + 'simple', + 'downloadable', + 'virtual', + 'giftcard' + ]; + + const PRODUCTS_COMPLEX_TYPES = [ + 'configurable', + 'grouped', + 'bundle' + ]; + /** * @var array|null */ @@ -241,6 +267,25 @@ protected function setStoreCounts(int $storeId): void ); } + /** + * @param int $storeId + * @param array $productTypes + * @return Collection + */ + protected function getProductsCollectionForStore(int $storeId, array $productTypes = []): Collection + { + $onlyVisible = !$this->configHelper->includeNonVisibleProductsInIndex(); + $collection = $this->productHelper->getProductCollectionQuery($storeId, null, $onlyVisible); + if (count($productTypes) > 0) { + $collection->addAttributeToFilter('type_id', ['in' => $productTypes]); + } + + // Randomize the results to get a more "diverse" sample + $collection->getSelect()->orderRand(); + + return $collection; + } + /** * @param Collection $products * @param int $sampleSize diff --git a/Console/Traits/BatchingCommandTrait.php b/Console/Traits/BatchingCommandTrait.php deleted file mode 100644 index cee392e80..000000000 --- a/Console/Traits/BatchingCommandTrait.php +++ /dev/null @@ -1,57 +0,0 @@ -configHelper->includeNonVisibleProductsInIndex(); - $collection = $this->productHelper->getProductCollectionQuery($storeId, null, $onlyVisible); - if (count($productTypes) > 0) { - $collection->addAttributeToFilter('type_id', ['in' => $productTypes]); - } - - // Randomize the results to get a more "diverse" sample - $collection->getSelect()->orderRand(); - - return $collection; - } -} From 2a5ab0a766031da598ba8b4161030613b4c24b3b Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 13 Aug 2025 16:30:26 +0200 Subject: [PATCH 041/119] MAGE-1109: address feedback --- Console/Command/BatchingOptimizeCommand.php | 45 ++++++++++++++++----- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index 95c5b53c2..2cbdf33fa 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -27,7 +27,7 @@ class BatchingOptimizeCommand extends AbstractStoreCommand * Recommended Max batch size * https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/how-to/sending-records-in-batches/ */ - const MAX_BATCH_SIZE = 10000000; //10MB + const MAX_BATCH_SIZE_IN_BYTES = 10_000_000; //10MB /** * Arbitrary default margin to ensure not to exceed recommended batch size @@ -136,6 +136,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int /** * @param array $storeIds * @return void + * @throws AlgoliaException + * @throws DiagnosticsException + * @throws LocalizedException + * @throws NoSuchEntityException */ protected function scanProductRecords(array $storeIds = []): void { @@ -150,6 +154,10 @@ protected function scanProductRecords(array $storeIds = []): void /** * @return void + * @throws AlgoliaException + * @throws DiagnosticsException + * @throws LocalizedException + * @throws NoSuchEntityException */ protected function scanProductRecordsForAllStores(): void { @@ -163,6 +171,10 @@ protected function scanProductRecordsForAllStores(): void /** * @param int $storeId * @return void + * @throws AlgoliaException + * @throws DiagnosticsException + * @throws LocalizedException + * @throws NoSuchEntityException */ protected function scanProductRecordsForStore(int $storeId): void { @@ -198,6 +210,8 @@ protected function scanProductRecordsForStore(int $storeId): void $this->output->writeln(' ============ '); $sizeAverage = $this->getSizeAverage($sample); + $this->output->writeln('Min record size : ' . $this->storeCounts[$storeId]['sample_min'] . 'B'); + $this->output->writeln('Max record size : ' . $this->storeCounts[$storeId]['sample_max'] . 'B'); $this->output->writeln('Average record size : ' . $sizeAverage . 'B'); $estimatedBatchCount = $this->getEstimatedMaxBatchCount($sizeAverage); @@ -246,11 +260,13 @@ protected function setStoreCounts(int $storeId): void $this->storeCounts[$storeId]['total'] = (int) $this->storeCounts[$storeId]['simple'] + (int) $this->storeCounts[$storeId]['complex']; - $this->storeCounts[$storeId]['simple_percentage'] = - ($this->storeCounts[$storeId]['simple'] * 100) / $this->storeCounts[$storeId]['total']; + $this->storeCounts[$storeId]['simple_percentage'] = $this->storeCounts[$storeId]['total'] > 0 ? + ($this->storeCounts[$storeId]['simple'] * 100) / $this->storeCounts[$storeId]['total'] : + 0; - $this->storeCounts[$storeId]['complex_percentage'] = - ($this->storeCounts[$storeId]['complex'] * 100) / $this->storeCounts[$storeId]['total']; + $this->storeCounts[$storeId]['complex_percentage'] = $this->storeCounts[$storeId]['total'] > 0 ? + ($this->storeCounts[$storeId]['complex'] * 100) / $this->storeCounts[$storeId]['total']: + 0; $sampleSize = $this->input->getOption(self::LARGE_SAMPLE_OPTION) ? self::LARGE_SAMPLE_SIZE : @@ -265,6 +281,9 @@ protected function setStoreCounts(int $storeId): void $this->getProductsSizes($simpleProducts, $simpleSampleSize), $this->getProductsSizes($complexProducts, $complexSampleSize) ); + + $this->storeCounts[$storeId]['sample_min'] = min($this->storeCounts[$storeId]['sample']); + $this->storeCounts[$storeId]['sample_max'] = max($this->storeCounts[$storeId]['sample']); } /** @@ -326,6 +345,10 @@ protected function getProductsSizes(Collection $products, int $sampleSize): arra */ protected function getSizeAverage(array $sizes): int { + if (count($sizes) <= 1) { + return 0.0; + } + return (int) round(array_sum(array_values($sizes)) / count($sizes)); } @@ -335,7 +358,7 @@ protected function getSizeAverage(array $sizes): int */ protected function getEstimatedMaxBatchCount(int $averageSize): int { - return (int) round(self::MAX_BATCH_SIZE / $averageSize); + return (int) round(self::MAX_BATCH_SIZE_IN_BYTES / $averageSize); } /** @@ -345,12 +368,16 @@ protected function getEstimatedMaxBatchCount(int $averageSize): int */ protected function getStandardDeviation(array $sizes, int $averageSize): float { + if (count($sizes) <= 1) { + return 0.0; + } + $sum = 0; foreach ($sizes as $size) { - $sum += ($size - abs($averageSize)) * ($size - abs($averageSize)); + $sum += pow($size - $averageSize, 2); } - return round(sqrt($sum / count($sizes)), 2); + return round(sqrt($sum / (count($sizes) - 1)), 2); } /** @@ -361,6 +388,6 @@ protected function getStandardDeviation(array $sizes, int $averageSize): float */ protected function getRecommendedBatchCount(int $averageSize, float $standardDeviation, int $margin = self::DEFAULT_MARGIN): int { - return (int) (self::MAX_BATCH_SIZE / ($averageSize + ($margin/100) * $standardDeviation)); + return (int) (self::MAX_BATCH_SIZE_IN_BYTES / ($averageSize + ($margin/100) * $standardDeviation)); } } From 4097119cf9af8670c0c5f3d6f50278dfb4e553fd Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 13 Aug 2025 16:47:46 +0200 Subject: [PATCH 042/119] MAGE-1109: added warning --- Console/Command/BatchingOptimizeCommand.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index 2cbdf33fa..9a9815f18 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -226,8 +226,11 @@ protected function scanProductRecordsForStore(int $storeId): void $this->output->writeln('Recommended batch count (low) : ' . $recommendedBatchCountLow . ' records'); $this->output->writeln('Recommended batch count (high) : ' . $recommendedBatchCountHigh . ' records'); $this->output->writeln(' '); + $this->output->writeln('Important: Those numbers are estimates only. Indexing activity should be monitored after making changes to ensure batches are not exceeding the recommended size of 10 MB.'); + $this->output->writeln(' ============ '); $this->output->writeln( 'This will override your "Maximum number of records processed per indexing job" configuration to ' . $recommendedBatchCountLow . ' for store "' . $storeName . '".'); + $this->output->writeln(' '); if ($this->confirmOperation()) { $this->configWriter->save( From 4d06c8c2efd9ee6b093c44f558bc550bcfa7cc49 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 13 Aug 2025 11:41:11 -0400 Subject: [PATCH 043/119] MAGE-1109 Add sample size as configurable option --- Console/Command/BatchingOptimizeCommand.php | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index 9a9815f18..bb7d8b2c0 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -41,11 +41,9 @@ class BatchingOptimizeCommand extends AbstractStoreCommand const INCREASED_MARGIN = 50; const DEFAULT_SAMPLE_SIZE = 20; - const LARGE_SAMPLE_SIZE = 100; - protected const LARGE_SAMPLE_OPTION = 'l-sample'; - - protected const LARGE_SAMPLE_OPTION_SHORTCUT = 'l'; + protected const OPTION_SAMPLE_SIZE = 'sample-size'; + protected const OPTION_SAMPLE_SIZE_SHORTCUT = 's'; const PRODUCTS_SIMPLE_TYPES = [ 'simple', @@ -104,10 +102,10 @@ protected function getAdditionalDefinition(): array { return [ new InputOption( - self::LARGE_SAMPLE_OPTION, - '-' . self::LARGE_SAMPLE_OPTION_SHORTCUT, - InputOption::VALUE_NONE, - 'Use a large sample of products (100)' + self::OPTION_SAMPLE_SIZE, + '-' . self::OPTION_SAMPLE_SIZE_SHORTCUT, + InputOption::VALUE_REQUIRED, + 'Sample size (number of products) - DEFAULT: ' . static::DEFAULT_SAMPLE_SIZE, ) ]; } @@ -271,9 +269,8 @@ protected function setStoreCounts(int $storeId): void ($this->storeCounts[$storeId]['complex'] * 100) / $this->storeCounts[$storeId]['total']: 0; - $sampleSize = $this->input->getOption(self::LARGE_SAMPLE_OPTION) ? - self::LARGE_SAMPLE_SIZE : - self::DEFAULT_SAMPLE_SIZE; + + $sampleSize = $this->input->getOption(self::OPTION_SAMPLE_SIZE) ?? self::DEFAULT_SAMPLE_SIZE; $simpleSampleSize = (int)round($sampleSize * ($this->storeCounts[$storeId]['simple_percentage'] / 100)); $complexSampleSize = (int)round($sampleSize * ($this->storeCounts[$storeId]['complex_percentage'] / 100)); From 31d4cb350c20cf8456a111460633eb97bb6582fc Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 14 Aug 2025 11:03:39 +0200 Subject: [PATCH 044/119] MAGE-1109: added margin and sample size options --- Console/Command/BatchingOptimizeCommand.php | 74 +++++++++++++++------ 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index bb7d8b2c0..c77e210d9 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -27,32 +27,33 @@ class BatchingOptimizeCommand extends AbstractStoreCommand * Recommended Max batch size * https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/how-to/sending-records-in-batches/ */ - const MAX_BATCH_SIZE_IN_BYTES = 10_000_000; //10MB + protected const MAX_BATCH_SIZE_IN_BYTES = 10_000_000; //10MB /** - * Arbitrary default margin to ensure not to exceed recommended batch size - */ - const DEFAULT_MARGIN = 25; - - /** - * Arbitrary increased margin to ensure not to exceed recommended batch size when catalog is a mix between complex and other product types + * Margin to ensure not to exceed recommended batch size when catalog is a mix between various product types * (i.e. with a lot of record sizes variations) */ - const INCREASED_MARGIN = 50; + protected const DEFAULT_MARGIN = 1; + protected const MAX_MARGIN = 10; - const DEFAULT_SAMPLE_SIZE = 20; + protected const DEFAULT_SAMPLE_SIZE = 20; + + protected const MAX_SAMPLE_SIZE = 1000; protected const OPTION_SAMPLE_SIZE = 'sample-size'; protected const OPTION_SAMPLE_SIZE_SHORTCUT = 's'; - const PRODUCTS_SIMPLE_TYPES = [ + protected const OPTION_MARGIN = 'margin'; + protected const OPTION_MARGIN_SHORTCUT = 'm'; + + protected const PRODUCTS_SIMPLE_TYPES = [ 'simple', 'downloadable', 'virtual', 'giftcard' ]; - const PRODUCTS_COMPLEX_TYPES = [ + protected const PRODUCTS_COMPLEX_TYPES = [ 'configurable', 'grouped', 'bundle' @@ -105,7 +106,13 @@ protected function getAdditionalDefinition(): array self::OPTION_SAMPLE_SIZE, '-' . self::OPTION_SAMPLE_SIZE_SHORTCUT, InputOption::VALUE_REQUIRED, - 'Sample size (number of products) - DEFAULT: ' . static::DEFAULT_SAMPLE_SIZE, + 'Sample size (number of products) - DEFAULT: ' . self::DEFAULT_SAMPLE_SIZE . ' - MAXIMUM: ' . self::MAX_SAMPLE_SIZE, + ), + new InputOption( + self::OPTION_MARGIN, + '-' . self::OPTION_MARGIN_SHORTCUT, + InputOption::VALUE_REQUIRED, + 'Safety margin - DEFAULT: ' . self::DEFAULT_MARGIN . ' - FROM 0 TO ' . self::MAX_MARGIN, ) ]; } @@ -122,6 +129,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $storeIds = $this->getStoreIds($input); try { + $this->validateOptions(); $this->scanProductRecords($storeIds); } catch (\Exception $e) { $this->output->writeln('' . $e->getMessage() . ''); @@ -131,6 +139,33 @@ protected function execute(InputInterface $input, OutputInterface $output): int return Cli::RETURN_SUCCESS; } + /** + * @return void + * @throws AlgoliaException + */ + protected function validateOptions(): void + { + if ( + $this->input->getOption(self::OPTION_SAMPLE_SIZE) + && ( + !ctype_digit((string) $this->input->getOption(self::OPTION_SAMPLE_SIZE)) + || (int) $this->input->getOption(self::OPTION_SAMPLE_SIZE) > self::MAX_SAMPLE_SIZE + ) + ) { + throw new AlgoliaException("Sample size option should be an integer (maximum 1000)" ); + } + + if ( + $this->input->getOption(self::OPTION_MARGIN) + && ( + !ctype_digit((string) $this->input->getOption(self::OPTION_MARGIN)) + || (int) $this->input->getOption(self::OPTION_MARGIN) > self::MAX_MARGIN + ) + ) { + throw new AlgoliaException("Margin option should be an integer (maximum 10)" ); + } + } + /** * @param array $storeIds * @return void @@ -218,22 +253,23 @@ protected function scanProductRecordsForStore(int $storeId): void $standardDeviation = $this->getStandardDeviation($sample, $sizeAverage); $this->output->writeln('Standard Deviation : ' . $standardDeviation); - $recommendedBatchCountLow = $this->getRecommendedBatchCount($sizeAverage, $standardDeviation, self::INCREASED_MARGIN); - $recommendedBatchCountHigh = $this->getRecommendedBatchCount($sizeAverage, $standardDeviation); + $margin = $this->input->getOption(self::OPTION_MARGIN) ?? self::DEFAULT_MARGIN; + $this->output->writeln('Safety margin : ' . $margin); + + $recommendedBatchCount = $this->getRecommendedBatchCount($sizeAverage, $standardDeviation, $margin); $this->output->writeln(' ============ '); - $this->output->writeln('Recommended batch count (low) : ' . $recommendedBatchCountLow . ' records'); - $this->output->writeln('Recommended batch count (high) : ' . $recommendedBatchCountHigh . ' records'); + $this->output->writeln('Recommended batch count : ' . $recommendedBatchCount . ' records'); $this->output->writeln(' '); $this->output->writeln('Important: Those numbers are estimates only. Indexing activity should be monitored after making changes to ensure batches are not exceeding the recommended size of 10 MB.'); $this->output->writeln(' ============ '); $this->output->writeln( - 'This will override your "Maximum number of records processed per indexing job" configuration to ' . $recommendedBatchCountLow . ' for store "' . $storeName . '".'); + 'This will override your "Maximum number of records processed per indexing job" configuration to ' . $recommendedBatchCount . ' for store "' . $storeName . '".'); $this->output->writeln(' '); if ($this->confirmOperation()) { $this->configWriter->save( ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, - $recommendedBatchCountLow, + $recommendedBatchCount, 'stores', $storeId ); @@ -388,6 +424,6 @@ protected function getStandardDeviation(array $sizes, int $averageSize): float */ protected function getRecommendedBatchCount(int $averageSize, float $standardDeviation, int $margin = self::DEFAULT_MARGIN): int { - return (int) (self::MAX_BATCH_SIZE_IN_BYTES / ($averageSize + ($margin/100) * $standardDeviation)); + return (int) (self::MAX_BATCH_SIZE_IN_BYTES / ($averageSize + $margin * $standardDeviation)); } } From 5688e3cfc9544a53d5fd869aaadfbe5c787db23b Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 14 Aug 2025 11:08:14 +0200 Subject: [PATCH 045/119] MAGE-1109: address feedback --- Console/Command/BatchingOptimizeCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index c77e210d9..4f6268eed 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -381,7 +381,7 @@ protected function getProductsSizes(Collection $products, int $sampleSize): arra */ protected function getSizeAverage(array $sizes): int { - if (count($sizes) <= 1) { + if (empty($sizes)) { return 0.0; } From d064761fd416e196c3698715d89dc0e05bda08cc Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 14 Aug 2025 11:47:44 +0200 Subject: [PATCH 046/119] MAGE-1109: added math helper --- Console/Command/BatchingOptimizeCommand.php | 75 +++++++++++---------- Helper/MathHelper.php | 38 +++++++++++ 2 files changed, 76 insertions(+), 37 deletions(-) create mode 100644 Helper/MathHelper.php diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index 4f6268eed..28727e8f5 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -6,6 +6,7 @@ use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; use Algolia\AlgoliaSearch\Helper\ConfigHelper; use Algolia\AlgoliaSearch\Helper\Entity\ProductHelper; +use Algolia\AlgoliaSearch\Helper\MathHelper; use Algolia\AlgoliaSearch\Service\AlgoliaConnector; use Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder; use Algolia\AlgoliaSearch\Service\Product\RecordBuilder; @@ -30,14 +31,28 @@ class BatchingOptimizeCommand extends AbstractStoreCommand protected const MAX_BATCH_SIZE_IN_BYTES = 10_000_000; //10MB /** - * Margin to ensure not to exceed recommended batch size when catalog is a mix between various product types - * (i.e. with a lot of record sizes variations) + * Margin to ensure not to exceed maximum batch size when catalog is a mix between various product types + * (i.e. with a lot of record sizes variations) - can be updated by the --margin option (from 0 to 10) + * 0 => The recommended batch size will be almost equal to the strictly calculated maximum batch size + * [1 to 9] => The more this value is, the more the recommended batch size will differ from the calculated maximum batch size + * 10 => Highest possible value, the recommended batch size will be greatly lower than the calculated maximum batch size */ protected const DEFAULT_MARGIN = 1; + + /** + * Max value for safety margin + */ protected const MAX_MARGIN = 10; + /** + * The sample size if the amount of products fetched to determine the recommended batch size + * Can be updated by the --sample-size option + */ protected const DEFAULT_SAMPLE_SIZE = 20; + /** + * Max Sample size + */ protected const MAX_SAMPLE_SIZE = 1000; protected const OPTION_SAMPLE_SIZE = 'sample-size'; @@ -46,6 +61,9 @@ class BatchingOptimizeCommand extends AbstractStoreCommand protected const OPTION_MARGIN = 'margin'; protected const OPTION_MARGIN_SHORTCUT = 'm'; + /** + * Simple product types (should generate smaller product records) + */ protected const PRODUCTS_SIMPLE_TYPES = [ 'simple', 'downloadable', @@ -53,6 +71,9 @@ class BatchingOptimizeCommand extends AbstractStoreCommand 'giftcard' ]; + /** + * Complex product types (should generate bigger product records) + */ protected const PRODUCTS_COMPLEX_TYPES = [ 'configurable', 'grouped', @@ -91,7 +112,7 @@ protected function getCommandName(): string protected function getCommandDescription(): string { - return "Scans some products to determine the average product record size."; + return "Performs catalog analysis and provides recommendation regarding optimal batching size for product indexing."; } protected function getStoreArgumentDescription(): string @@ -140,6 +161,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int } /** + * Ensures sample size and margin options are valid + * * @return void * @throws AlgoliaException */ @@ -242,7 +265,7 @@ protected function scanProductRecordsForStore(int $storeId): void } $this->output->writeln(' ============ '); - $sizeAverage = $this->getSizeAverage($sample); + $sizeAverage = MathHelper::getRoundedAverage($sample); $this->output->writeln('Min record size : ' . $this->storeCounts[$storeId]['sample_min'] . 'B'); $this->output->writeln('Max record size : ' . $this->storeCounts[$storeId]['sample_max'] . 'B'); $this->output->writeln('Average record size : ' . $sizeAverage . 'B'); @@ -250,7 +273,7 @@ protected function scanProductRecordsForStore(int $storeId): void $estimatedBatchCount = $this->getEstimatedMaxBatchCount($sizeAverage); $this->output->writeln('Estimated Max batch count : ' . $estimatedBatchCount . ' records'); - $standardDeviation = $this->getStandardDeviation($sample, $sizeAverage); + $standardDeviation = MathHelper::getSampleStandardDeviation($sample, $sizeAverage); $this->output->writeln('Standard Deviation : ' . $standardDeviation); $margin = $this->input->getOption(self::OPTION_MARGIN) ?? self::DEFAULT_MARGIN; @@ -323,6 +346,8 @@ protected function setStoreCounts(int $storeId): void } /** + * Generates a product collection with the same helper as the product indexer to get the exact amount of expected products in the Algolia index + * * @param int $storeId * @param array $productTypes * @return Collection @@ -376,19 +401,8 @@ protected function getProductsSizes(Collection $products, int $sampleSize): arra } /** - * @param array $sizes - * @return int - */ - protected function getSizeAverage(array $sizes): int - { - if (empty($sizes)) { - return 0.0; - } - - return (int) round(array_sum(array_values($sizes)) / count($sizes)); - } - - /** + * Determines the maximum estimated batch count which will be considered as the upper boundary + * * @param int $averageSize * @return int */ @@ -398,25 +412,12 @@ protected function getEstimatedMaxBatchCount(int $averageSize): int } /** - * @param array $sizes - * @param int $averageSize - * @return float - */ - protected function getStandardDeviation(array $sizes, int $averageSize): float - { - if (count($sizes) <= 1) { - return 0.0; - } - - $sum = 0; - foreach ($sizes as $size) { - $sum += pow($size - $averageSize, 2); - } - - return round(sqrt($sum / (count($sizes) - 1)), 2); - } - - /** + * Provides a recommended batch count according to: + * - the average record size provided by the product sample + * - the standard deviation of the product sample + * - an arbitrary safety margin (1 to 10) to allow the user to alter the strictness of the recommendation + * (the lower the margin is, the closer it will be from the maximum batch count) + * * @param int $averageSize * @param float $standardDeviation * @param int $margin diff --git a/Helper/MathHelper.php b/Helper/MathHelper.php new file mode 100644 index 000000000..7c558fee5 --- /dev/null +++ b/Helper/MathHelper.php @@ -0,0 +1,38 @@ + Date: Thu, 14 Aug 2025 12:52:59 +0200 Subject: [PATCH 047/119] MAGE-1109: added helper tests --- Console/Command/BatchingOptimizeCommand.php | 4 +- Helper/MathHelper.php | 13 ++--- Test/Unit/Helper/MathHelperTest.php | 58 +++++++++++++++++++++ 3 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 Test/Unit/Helper/MathHelperTest.php diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index 28727e8f5..b9221ce8c 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -265,7 +265,7 @@ protected function scanProductRecordsForStore(int $storeId): void } $this->output->writeln(' ============ '); - $sizeAverage = MathHelper::getRoundedAverage($sample); + $sizeAverage = (int) round(MathHelper::getAverage($sample)); $this->output->writeln('Min record size : ' . $this->storeCounts[$storeId]['sample_min'] . 'B'); $this->output->writeln('Max record size : ' . $this->storeCounts[$storeId]['sample_max'] . 'B'); $this->output->writeln('Average record size : ' . $sizeAverage . 'B'); @@ -273,7 +273,7 @@ protected function scanProductRecordsForStore(int $storeId): void $estimatedBatchCount = $this->getEstimatedMaxBatchCount($sizeAverage); $this->output->writeln('Estimated Max batch count : ' . $estimatedBatchCount . ' records'); - $standardDeviation = MathHelper::getSampleStandardDeviation($sample, $sizeAverage); + $standardDeviation = MathHelper::getSampleStandardDeviation($sample); $this->output->writeln('Standard Deviation : ' . $standardDeviation); $margin = $this->input->getOption(self::OPTION_MARGIN) ?? self::DEFAULT_MARGIN; diff --git a/Helper/MathHelper.php b/Helper/MathHelper.php index 7c558fee5..a38f3faca 100644 --- a/Helper/MathHelper.php +++ b/Helper/MathHelper.php @@ -6,28 +6,29 @@ class MathHelper { /** * @param array $values - * @return int + * @return float */ - static public function getRoundedAverage(array $values): int + static public function getAverage(array $values): float { if (empty($values)) { - return 0.0; + return 0.00; } - return (int) round(array_sum(array_values($values)) / count($values)); + return round(array_sum(array_values($values)) / count($values), 2); } /** * @param array $values - * @param int $average * @return float */ - static public function getSampleStandardDeviation(array $values, int $average): float + static public function getSampleStandardDeviation(array $values): float { if (count($values) <= 1) { return 0.0; } + $average = self::getAverage($values); + $sum = 0; foreach ($values as $value) { $sum += pow($value - $average, 2); diff --git a/Test/Unit/Helper/MathHelperTest.php b/Test/Unit/Helper/MathHelperTest.php new file mode 100644 index 000000000..ad0845ed1 --- /dev/null +++ b/Test/Unit/Helper/MathHelperTest.php @@ -0,0 +1,58 @@ +assertEquals($expectedResult, MathHelper::getAverage($values)); + } + + /** + * @dataProvider standardDeviationProvider + */ + public function testStandardDeviation($values, $expectedResult) + { + $this->assertEquals($expectedResult, MathHelper::getSampleStandardDeviation($values)); + } + + public static function averageProvider(): array + { + /** Tested with https://www.calculator.net/average-calculator.html */ + return [ + ['values' => [], 'expectedResult' => 0], + ['values' => [1, 3], 'expectedResult' => 2], + ['values' => ['foo' => 1, 'bar' => 3], 'expectedResult' => 2], + ['values' => [1, 9], 'expectedResult' => 5], + ['values' => [1, 2], 'expectedResult' => 1.5], + ['values' => [1, 2, 3], 'expectedResult' => 2], + ['values' => [1, 2, 4], 'expectedResult' => 2.33], + ['values' => [11253, 10025, 9521, 13250], 'expectedResult' => 11012.25], + ['values' => [10, 12, 23, 23, 16, 23, 21, 16], 'expectedResult' => 18], + ]; + } + + public static function standardDeviationProvider(): array + { + /** Tested with https://www.calculator.net/standard-deviation-calculator.html */ + return [ + ['values' => [], 'expectedResult' => 0.0], + ['values' => [1], 'expectedResult' => 0.0], + ['values' => [1, 1], 'expectedResult' => 0.0], + ['values' => [1, 3], 'expectedResult' => 1.41], + ['values' => [1, 4, 12], 'expectedResult' => 5.69], + ['values' => [3, 4, 6], 'expectedResult' => 1.53], + ['values' => [3, 4, 6, 8, 7, 11], 'expectedResult' => 2.88], + ['values' => [11253, 10025, 9521, 13250], 'expectedResult' => 1659.72], + ['values' => [10, 12, 23, 23, 16, 23, 21, 16], 'expectedResult' => 5.24], + ]; + } +} From 97f7f54ff6b8d953383024d82315a805b5865306 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 14 Aug 2025 17:19:33 +0200 Subject: [PATCH 048/119] MAGE-1109: rework safety margin --- Console/Command/BatchingOptimizeCommand.php | 30 ++++++++++++--------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index b9221ce8c..c257357ab 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -32,17 +32,22 @@ class BatchingOptimizeCommand extends AbstractStoreCommand /** * Margin to ensure not to exceed maximum batch size when catalog is a mix between various product types - * (i.e. with a lot of record sizes variations) - can be updated by the --margin option (from 0 to 10) - * 0 => The recommended batch size will be almost equal to the strictly calculated maximum batch size - * [1 to 9] => The more this value is, the more the recommended batch size will differ from the calculated maximum batch size - * 10 => Highest possible value, the recommended batch size will be greatly lower than the calculated maximum batch size + * (i.e. with a lot of record sizes variations) - can be updated by the --margin option (from 0.25 to 3.00) + * 0.00 => Lowest possible value (0.00 * standard deviation = 0), the recommended batch size will be almost equal to the strictly calculated maximum batch size + * 0.25 => Default value (0.25 * standard deviation), the recommended batch size will be close to the strictly calculated maximum batch size + * 3.00 => Highest possible value (3 * standard deviation), the recommended batch size will be greatly lower than the calculated maximum batch size */ - protected const DEFAULT_MARGIN = 1; + protected const DEFAULT_MARGIN = 0.25; + + /** + * Min value for safety margin + */ + protected const MIN_MARGIN = 0; /** * Max value for safety margin */ - protected const MAX_MARGIN = 10; + protected const MAX_MARGIN = 3; /** * The sample size if the amount of products fetched to determine the recommended batch size @@ -133,7 +138,7 @@ protected function getAdditionalDefinition(): array self::OPTION_MARGIN, '-' . self::OPTION_MARGIN_SHORTCUT, InputOption::VALUE_REQUIRED, - 'Safety margin - DEFAULT: ' . self::DEFAULT_MARGIN . ' - FROM 0 TO ' . self::MAX_MARGIN, + 'Safety margin - DEFAULT: ' . self::DEFAULT_MARGIN . ' - FROM ' . self::MIN_MARGIN . ' TO ' . self::MAX_MARGIN, ) ]; } @@ -181,11 +186,12 @@ protected function validateOptions(): void if ( $this->input->getOption(self::OPTION_MARGIN) && ( - !ctype_digit((string) $this->input->getOption(self::OPTION_MARGIN)) - || (int) $this->input->getOption(self::OPTION_MARGIN) > self::MAX_MARGIN + !is_numeric($this->input->getOption(self::OPTION_MARGIN)) + || (float) $this->input->getOption(self::OPTION_MARGIN) > self::MAX_MARGIN + || (float) $this->input->getOption(self::OPTION_MARGIN) < self::MIN_MARGIN ) ) { - throw new AlgoliaException("Margin option should be an integer (maximum 10)" ); + throw new AlgoliaException("Margin option should be a decimal value (between 0 and 3)" ); } } @@ -420,10 +426,10 @@ protected function getEstimatedMaxBatchCount(int $averageSize): int * * @param int $averageSize * @param float $standardDeviation - * @param int $margin + * @param float $margin * @return int */ - protected function getRecommendedBatchCount(int $averageSize, float $standardDeviation, int $margin = self::DEFAULT_MARGIN): int + protected function getRecommendedBatchCount(int $averageSize, float $standardDeviation, float $margin = self::DEFAULT_MARGIN): int { return (int) (self::MAX_BATCH_SIZE_IN_BYTES / ($averageSize + $margin * $standardDeviation)); } From 2afbfaef039d62cc6196082135b443b8109283cf Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Mon, 18 Aug 2025 19:42:02 -0400 Subject: [PATCH 049/119] MAGE-1394 Implement boilerplate command + truncate op --- Console/Command/Queue/ClearQueueCommand.php | 159 ++++++++++++++++++ .../ProcessQueueCommand.php | 7 +- etc/di.xml | 9 +- 3 files changed, 170 insertions(+), 5 deletions(-) create mode 100644 Console/Command/Queue/ClearQueueCommand.php rename Console/Command/{Indexer => Queue}/ProcessQueueCommand.php (89%) diff --git a/Console/Command/Queue/ClearQueueCommand.php b/Console/Command/Queue/ClearQueueCommand.php new file mode 100644 index 000000000..c540ae98b --- /dev/null +++ b/Console/Command/Queue/ClearQueueCommand.php @@ -0,0 +1,159 @@ +bin/magento algolia:queue:clear 1 2 3'; + } + + protected function getAdditionalDefinition(): array + { + return []; + } + + /** + * @throws NoSuchEntityException + * @throws LocalizedException + */ + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->input = $input; + $this->output = $output; + $this->setAreaCode(); + + if (!$this->confirmClearOperation()) { + return Cli::RETURN_SUCCESS; + } + + $storeIds = $this->getStoreIds($input); + + $output->writeln($this->decorateOperationAnnouncementMessage('Clearing indexing queue for {{target}}', $storeIds)); + + try { + $this->clearIndexingQueue($storeIds); + } catch (\Exception $e) { + $this->output->writeln('' . $e->getMessage() . ''); + return Cli::RETURN_FAILURE; + } + + $output->writeln('Indexing queue cleared successfully!'); + return Cli::RETURN_SUCCESS; + } + + /** + * Confirm the clear operation as it's destructive + * + * @return bool + */ + protected function confirmClearOperation(): bool + { + $this->output->writeln('WARNING: This will clear all pending indexing jobs from the queue for the specified store(s). This action cannot be undone!'); + return $this->confirmOperation( + 'Indexing queue clear operation confirmed', + 'Indexing queue clear operation cancelled' + ); + } + + /** + * Clear indexing queue for specified stores or all stores + * + * @param array $storeIds + * @throws NoSuchEntityException + */ + public function clearIndexingQueue(array $storeIds = []): void + { + if (count($storeIds)) { + foreach ($storeIds as $storeId) { + $this->clearIndexingQueueForStore($storeId); + } + } else { + $this->clearIndexingQueueForAllStores(); + } + } + + /** + * Clear indexing queue for a specific store + * + * @param int $storeId + * @throws NoSuchEntityException + */ + public function clearIndexingQueueForStore(int $storeId): void + { + $storeName = $this->storeNameFetcher->getStoreName($storeId); + $this->output->writeln('Clearing indexing queue for ' . $storeName . '...'); + + try { + // Clear the indexing queue for this store + // Note: You'll need to implement the actual queue clearing logic based on your queue implementation + $this->clearQueueForStore($storeId); + + $this->output->writeln('✓ Indexing queue cleared for ' . $storeName . ''); + } catch (\Exception $e) { + $this->output->writeln('✗ Failed to clear indexing queue for ' . $storeName . ': ' . $e->getMessage() . ''); + } + } + + /** + * Clear indexing queue for all stores + * + * @throws NoSuchEntityException + * @throws LocalizedException + */ + public function clearIndexingQueueForAllStores(): void + { + $connection = $this->jobResourceModel->getConnection(); + $connection->truncateTable($this->jobResourceModel->getMainTable()); + } + + /** + * Clear the actual queue for a specific store + * This method should be implemented based on your specific queue implementation + * + * @param int $storeId + * @throws \Exception + */ + protected function clearQueueForStore(int $storeId): void + { + throw new \Exception('Queue clearing method not implemented by store'); + } +} diff --git a/Console/Command/Indexer/ProcessQueueCommand.php b/Console/Command/Queue/ProcessQueueCommand.php similarity index 89% rename from Console/Command/Indexer/ProcessQueueCommand.php rename to Console/Command/Queue/ProcessQueueCommand.php index 13db37e56..43d6292c9 100644 --- a/Console/Command/Indexer/ProcessQueueCommand.php +++ b/Console/Command/Queue/ProcessQueueCommand.php @@ -1,6 +1,6 @@ setName($this->getCommandName()) - ->setDescription($this->getCommandDescription()); + ->setDescription($this->getCommandDescription()) + ->setAliases(['algolia:reindex:process_queue']); parent::configure(); } diff --git a/etc/di.xml b/etc/di.xml index 54a985f5e..960b977e5 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -147,21 +147,26 @@ + Algolia\AlgoliaSearch\Console\Command\ReplicaSyncCommand Algolia\AlgoliaSearch\Console\Command\ReplicaDeleteCommand Algolia\AlgoliaSearch\Console\Command\ReplicaRebuildCommand Algolia\AlgoliaSearch\Console\Command\ReplicaDisableVirtualCommand Algolia\AlgoliaSearch\Console\Command\SynonymDeduplicateCommand Algolia\AlgoliaSearch\Console\Command\BatchingOptimizeCommand - + + Algolia\AlgoliaSearch\Console\Command\Indexer\IndexProductsCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexCategoriesCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexPagesCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexSuggestionsCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexAdditionalSectionsCommand Algolia\AlgoliaSearch\Console\Command\Indexer\DeleteProductsCommand - Algolia\AlgoliaSearch\Console\Command\Indexer\ProcessQueueCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexAllCommand + + + Algolia\AlgoliaSearch\Console\Command\Queue\ProcessQueueCommand + Algolia\AlgoliaSearch\Console\Command\Queue\ClearQueueCommand From e86712504f6feeadefd45d46247fa18623a406f6 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Mon, 18 Aug 2025 21:48:07 -0400 Subject: [PATCH 050/119] MAGE-1394 Implement selective store queue clearing operation --- Console/Command/Queue/ClearQueueCommand.php | 88 +++++++++++++++++++-- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/Console/Command/Queue/ClearQueueCommand.php b/Console/Command/Queue/ClearQueueCommand.php index c540ae98b..c474b57da 100644 --- a/Console/Command/Queue/ClearQueueCommand.php +++ b/Console/Command/Queue/ClearQueueCommand.php @@ -4,7 +4,6 @@ use Algolia\AlgoliaSearch\Console\Command\AbstractStoreCommand; use Algolia\AlgoliaSearch\Model\ResourceModel\Job as JobResourceModel; -use Algolia\AlgoliaSearch\Service\AlgoliaConnector; use Algolia\AlgoliaSearch\Service\StoreNameFetcher; use Magento\Framework\App\State; use Magento\Framework\Console\Cli; @@ -38,7 +37,7 @@ protected function getCommandName(): string protected function getCommandDescription(): string { - return "Clear the indexing queue for specified store(s) or all stores. This will remove all pending indexing tasks from the queue."; + return "Clear the indexing queue for specified store(s) or all stores. This will remove all pending indexing jobs from the queue."; } protected function getStoreArgumentDescription(): string @@ -98,7 +97,7 @@ protected function confirmClearOperation(): bool * Clear indexing queue for specified stores or all stores * * @param array $storeIds - * @throws NoSuchEntityException + * @throws NoSuchEntityException|LocalizedException */ public function clearIndexingQueue(array $storeIds = []): void { @@ -123,8 +122,6 @@ public function clearIndexingQueueForStore(int $storeId): void $this->output->writeln('Clearing indexing queue for ' . $storeName . '...'); try { - // Clear the indexing queue for this store - // Note: You'll need to implement the actual queue clearing logic based on your queue implementation $this->clearQueueForStore($storeId); $this->output->writeln('✓ Indexing queue cleared for ' . $storeName . ''); @@ -146,14 +143,89 @@ public function clearIndexingQueueForAllStores(): void } /** - * Clear the actual queue for a specific store - * This method should be implemented based on your specific queue implementation + * Clear the queue for a specific store (2 different approaches) + * Filters jobs by store_id in the JSON data field and deletes them * * @param int $storeId * @throws \Exception */ protected function clearQueueForStore(int $storeId): void { - throw new \Exception('Queue clearing method not implemented by store'); + try { + // Use JSON_EXTRACT to filter by store_id in the data field + // This assumes MySQL 5.7+ or MariaDB 10.2+ for JSON support + $connection = $this->jobResourceModel->getConnection(); + $mainTable = $this->jobResourceModel->getMainTable(); + + $select = $connection->select() + ->from($mainTable, ['job_id']) + ->where('JSON_EXTRACT(data, "$.storeId") = ?', $storeId); + + $jobIds = $connection->fetchCol($select); + + if (empty($jobIds)) { + $this->output->writeln('No jobs found for store ID ' . $storeId . ''); + return; + } + + // Delete the filtered jobs + $deletedCount = $connection->delete( + $mainTable, + ['job_id IN (?)' => $jobIds] + ); + + $this->output->writeln('Deleted ' . $deletedCount . ' jobs for store ID ' . $storeId . ''); + + } catch (\Exception $e) { + // Fallback method if JSON_EXTRACT is not supported + $this->output->writeln('JSON filtering not supported by database, using fallback method...'); + $this->clearQueueForStoreFallback($storeId); + } + } + + /** + * Fallback method for clearing queue by store when JSON filtering is not supported + * Loads all jobs and filters them in PHP + * + * @param int $storeId + * @throws \Exception + */ + protected function clearQueueForStoreFallback(int $storeId): void + { + try { + $connection = $this->jobResourceModel->getConnection(); + $mainTable = $this->jobResourceModel->getMainTable(); + + // Get all jobs and filter by store_id in PHP + $select = $connection->select() + ->from($mainTable, ['job_id', 'data']) + ->where('data IS NOT NULL'); + + $jobs = $connection->fetchAll($select); + $jobsToDelete = []; + + foreach ($jobs as $job) { + $data = json_decode($job['data'], true); + if (isset($data['storeId']) && $data['storeId'] == $storeId) { + $jobsToDelete[] = $job['job_id']; + } + } + + if (empty($jobsToDelete)) { + $this->output->writeln('No jobs found for store ID ' . $storeId . ' (fallback method)'); + return; + } + + // Delete the filtered jobs + $deletedCount = $connection->delete( + $mainTable, + ['job_id IN (?)' => $jobsToDelete] + ); + + $this->output->writeln('Deleted ' . $deletedCount . ' jobs for store ID ' . $storeId . ' (fallback method)'); + + } catch (\Exception $e) { + throw new \Exception('Failed to clear queue for store ' . $storeId . ': ' . $e->getMessage()); + } } } From 85dcaa002df97b7e7555fa0f9d08c90ee9d0e928 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Mon, 18 Aug 2025 22:37:24 -0400 Subject: [PATCH 051/119] MAGE-1394 Update DI --- etc/di.xml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/etc/di.xml b/etc/di.xml index 960b977e5..26203b472 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -254,19 +254,26 @@ Algolia\AlgoliaSearch\Service\StoreNameFetcher\Proxy - + + + Algolia\AlgoliaSearch\Service\StoreNameFetcher\Proxy + + + + + + Algolia\AlgoliaSearch\Helper\ConfigHelper\Proxy Algolia\AlgoliaSearch\Model\Queue\Proxy Algolia\AlgoliaSearch\Service\AlgoliaCredentialsManager\Proxy - + Algolia\AlgoliaSearch\Service\StoreNameFetcher\Proxy - From 023c8cbf21783526daa9e06f4181576b214c3c9f Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Mon, 18 Aug 2025 22:54:17 -0400 Subject: [PATCH 052/119] MAGE-1394 Allow default override for non-interactive mode --- Console/Command/AbstractStoreCommand.php | 4 ++-- Console/Command/Queue/ClearQueueCommand.php | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Console/Command/AbstractStoreCommand.php b/Console/Command/AbstractStoreCommand.php index f6481f9c8..92b94c194 100644 --- a/Console/Command/AbstractStoreCommand.php +++ b/Console/Command/AbstractStoreCommand.php @@ -127,10 +127,10 @@ protected function decorateOperationAnnouncementMessage(string $msg, array $stor : "$msg"; } - protected function confirmOperation(string $okMessage = '', string $cancelMessage = 'Operation cancelled'): bool + protected function confirmOperation(string $okMessage = '', string $cancelMessage = 'Operation cancelled', bool $default = false): bool { $helper = $this->getHelper('question'); - $question = new ConfirmationQuestion('Are you sure wish to proceed? (y/n) ', false); + $question = new ConfirmationQuestion('Are you sure wish to proceed? (y/n) ', $default); if (!$helper->ask($this->input, $this->output, $question)) { if ($cancelMessage) { $this->output->writeln("$cancelMessage"); diff --git a/Console/Command/Queue/ClearQueueCommand.php b/Console/Command/Queue/ClearQueueCommand.php index c474b57da..a35011d25 100644 --- a/Console/Command/Queue/ClearQueueCommand.php +++ b/Console/Command/Queue/ClearQueueCommand.php @@ -89,7 +89,8 @@ protected function confirmClearOperation(): bool $this->output->writeln('WARNING: This will clear all pending indexing jobs from the queue for the specified store(s). This action cannot be undone!'); return $this->confirmOperation( 'Indexing queue clear operation confirmed', - 'Indexing queue clear operation cancelled' + 'Indexing queue clear operation cancelled', + true ); } From ee15986110190023f2d1e9b21de90dde0c00b92f Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Tue, 19 Aug 2025 13:47:43 +0200 Subject: [PATCH 053/119] MAGE-1110: Fix queue merging --- Model/Job.php | 111 ++++-------- Test/Integration/Indexing/Queue/QueueTest.php | 166 +++++++++--------- 2 files changed, 119 insertions(+), 158 deletions(-) diff --git a/Model/Job.php b/Model/Job.php index 645bef395..c1ac8e680 100644 --- a/Model/Job.php +++ b/Model/Job.php @@ -3,6 +3,12 @@ namespace Algolia\AlgoliaSearch\Model; use Algolia\AlgoliaSearch\Api\Data\JobInterface; +use Magento\Framework\Data\Collection\AbstractDb; +use Magento\Framework\Exception\AlreadyExistsException; +use Magento\Framework\Model\Context; +use Magento\Framework\Model\ResourceModel\AbstractResource; +use Magento\Framework\ObjectManagerInterface; +use Magento\Framework\Registry; /** * @api @@ -26,24 +32,24 @@ class Job extends \Magento\Framework\Model\AbstractModel implements JobInterface { protected $_eventPrefix = 'algoliasearch_queue_job'; - /** @var \Magento\Framework\ObjectManagerInterface */ - protected $objectManager; + /** @var ObjectManagerInterface */ + protected ObjectManagerInterface $objectManager; /** - * @param \Magento\Framework\Model\Context $context - * @param \Magento\Framework\Registry $registry - * @param \Magento\Framework\ObjectManagerInterface $objectManager - * @param \Magento\Framework\Model\ResourceModel\AbstractResource $resource - * @param \Magento\Framework\Data\Collection\AbstractDb $resourceCollection - * @param array $data + * @param Context $context + * @param Registry $registry + * @param ObjectManagerInterface $objectManager + * @param AbstractResource|null $resource + * @param AbstractDb|null $resourceCollection + * @param array $data */ public function __construct( - \Magento\Framework\Model\Context $context, - \Magento\Framework\Registry $registry, - \Magento\Framework\ObjectManagerInterface $objectManager, - ?\Magento\Framework\Model\ResourceModel\AbstractResource $resource = null, - ?\Magento\Framework\Data\Collection\AbstractDb $resourceCollection = null, - array $data = [] + Context $context, + Registry $registry, + ObjectManagerInterface $objectManager, + ?AbstractResource $resource = null, + ?AbstractDb $resourceCollection = null, + array $data = [] ) { parent::__construct($context, $registry, $resource, $resourceCollection, $data); @@ -57,13 +63,13 @@ public function __construct( */ protected function _construct() { - $this->_init(\Algolia\AlgoliaSearch\Model\ResourceModel\Job::class); + $this->_init(ResourceModel\Job::class); } /** - * @throws \Magento\Framework\Exception\AlreadyExistsException - * * @return $this + * @throws AlreadyExistsException|\Exception + * */ public function execute() { @@ -97,6 +103,10 @@ public function prepare() if (isset($decodedData['store_id'])) { $this->setStoreId($decodedData['store_id']); } + + if (isset($decodedData['storeId'])) { + $this->setStoreId($decodedData['storeId']); + } } return $this; @@ -124,44 +134,17 @@ public function canMerge(Job $job, $maxJobDataSize) $decodedData = $this->getDecodedData(); - // @todo Remove legacy checks on 3.16.0 - if ((!isset($decodedData['product_ids']) || count($decodedData['product_ids']) <= 0) - && (!isset($decodedData['category_ids']) || count($decodedData['category_ids']) < 0) - && (!isset($decodedData['entity_ids']) || count($decodedData['entity_ids']) < 0) - && (!isset($decodedData['page_ids']) || count($decodedData['page_ids']) < 0)) { + if (!isset($decodedData['entityIds']) || count($decodedData['entityIds']) <= 0) { return false; } $candidateDecodedData = $job->getDecodedData(); - // @todo Remove legacy checks on 3.16.0 - if ((!isset($candidateDecodedData['product_ids']) || count($candidateDecodedData['product_ids']) <= 0) - && (!isset($candidateDecodedData['category_ids']) || count($candidateDecodedData['category_ids']) < 0) - && (!isset($candidateDecodedData['entity_ids']) || count($candidateDecodedData['entity_ids']) < 0) - && (!isset($candidateDecodedData['page_ids']) || count($candidateDecodedData['page_ids']) < 0)) { - return false; - } - - // @todo Remove on 3.16.0 - if (isset($decodedData['product_ids']) - && count($decodedData['product_ids']) + count($candidateDecodedData['product_ids']) > $maxJobDataSize) { - return false; - } - - // @todo Remove on 3.16.0 - if (isset($decodedData['category_ids']) - && count($decodedData['category_ids']) + count($candidateDecodedData['category_ids']) > $maxJobDataSize) { + if (!isset($candidateDecodedData['entityIds']) || count($candidateDecodedData['entityIds']) <= 0) { return false; } - // @todo Remove on 3.16.0 - if (isset($decodedData['page_ids']) - && count($decodedData['page_ids']) + count($candidateDecodedData['page_ids']) > $maxJobDataSize) { - return false; - } - - if (isset($decodedData['entity_ids']) - && count($decodedData['entity_ids']) + count($candidateDecodedData['entity_ids']) > $maxJobDataSize) { + if (count($decodedData['entityIds']) + count($candidateDecodedData['entityIds']) > $maxJobDataSize) { return false; } @@ -185,35 +168,13 @@ public function merge(Job $mergedJob) $dataSize = $this->getDataSize(); - // @todo Remove useless code on 3.16.0 - if (isset($decodedData['product_ids'])) { - $decodedData['product_ids'] = array_unique(array_merge( - $decodedData['product_ids'], - $mergedJobDecodedData['product_ids'] - )); - - $dataSize = count($decodedData['product_ids']); - } elseif (isset($decodedData['category_ids'])) { - $decodedData['category_ids'] = array_unique(array_merge( - $decodedData['category_ids'], - $mergedJobDecodedData['category_ids'] - )); - - $dataSize = count($decodedData['category_ids']); - } elseif (isset($decodedData['page_ids'])) { - $decodedData['page_ids'] = array_unique(array_merge( - $decodedData['page_ids'], - $mergedJobDecodedData['page_ids'] - )); - - $dataSize = count($decodedData['page_ids']); - } elseif (isset($decodedData['entity_ids'])) { - $decodedData['entity_ids'] = array_unique(array_merge( - $decodedData['entity_ids'], - $mergedJobDecodedData['entity_ids'] + if (isset($decodedData['entityIds'])) { + $decodedData['entityIds'] = array_unique(array_merge( + $decodedData['entityIds'], + $mergedJobDecodedData['entityIds'] )); - $dataSize = count($decodedData['entity_ids']); + $dataSize = count($decodedData['entityIds']); } $this->setDecodedData($decodedData); @@ -253,7 +214,7 @@ public function getStatus() /** * @param \Exception $e * - * @throws \Magento\Framework\Exception\AlreadyExistsException + * @throws AlreadyExistsException * * @return Job */ diff --git a/Test/Integration/Indexing/Queue/QueueTest.php b/Test/Integration/Indexing/Queue/QueueTest.php index ec7534636..342cba814 100644 --- a/Test/Integration/Indexing/Queue/QueueTest.php +++ b/Test/Integration/Indexing/Queue/QueueTest.php @@ -269,7 +269,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"1","entity_ids":["9","22"]}', + 'data' => '{"storeId":"1","entityIds":["9","22"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -280,7 +280,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"2","entity_ids":["9","22"]}', + 'data' => '{"storeId":"2","entityIds":["9","22"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -291,7 +291,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"3","entity_ids":["9","22"]}', + 'data' => '{"storeId":"3","entityIds":["9","22"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -302,7 +302,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"1","entity_ids":["448"]}', + 'data' => '{"storeId":"1","entityIds":["448"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -313,7 +313,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"2","entity_ids":["448"]}', + 'data' => '{"storeId":"2","entityIds":["448"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -324,7 +324,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"3","entity_ids":["448"]}', + 'data' => '{"storeId":"3","entityIds":["448"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -335,7 +335,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"1","entity_ids":["40"]}', + 'data' => '{"storeId":"1","entityIds":["40"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -346,7 +346,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"2","entity_ids":["40"]}', + 'data' => '{"storeId":"2","entityIds":["40"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -357,7 +357,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"3","entity_ids":["40"]}', + 'data' => '{"storeId":"3","entityIds":["40"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -368,7 +368,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"1","entity_ids":["405"]}', + 'data' => '{"storeId":"1","entityIds":["405"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -379,7 +379,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"2","entity_ids":["405"]}', + 'data' => '{"storeId":"2","entityIds":["405"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -390,7 +390,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"3","entity_ids":["405"]}', + 'data' => '{"storeId":"3","entityIds":["405"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -412,7 +412,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"1","entity_ids":["9","22"]}', + 'data' => '{"storeId":"1","entityIds":["9","22"]}', 'max_retries' => '3', 'retries' => '0', 'error_log' => '', @@ -421,8 +421,8 @@ public function testMerging() 'store_id' => '1', 'is_full_reindex' => '0', 'decoded_data' => [ - 'store_id' => '1', - 'entity_ids' => [ + 'storeId' => '1', + 'entityIds' => [ 0 => '9', 1 => '22', 2 => '40', @@ -442,7 +442,7 @@ public function testMerging() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"1","entity_ids":["448"]}', + 'data' => '{"storeId":"1","entityIds":["448"]}', 'max_retries' => '3', 'retries' => '0', 'error_log' => '', @@ -451,8 +451,8 @@ public function testMerging() 'store_id' => '1', 'is_full_reindex' => '0', 'decoded_data' => [ - 'store_id' => '1', - 'entity_ids' => [ + 'storeId' => '1', + 'entityIds' => [ 0 => '448', 1 => '405', ], @@ -476,7 +476,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"1","entity_ids":["9","22"]}', + 'data' => '{"storeId":"1","entityIds":["9","22"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -486,7 +486,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"2","entity_ids":["9","22"]}', + 'data' => '{"storeId":"2","entityIds":["9","22"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -496,7 +496,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"3","entity_ids":["9","22"]}', + 'data' => '{"storeId":"3","entityIds":["9","22"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -506,7 +506,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, 'method' => 'deleteObjects', - 'data' => '{"store_id":"1","product_ids":["448"]}', + 'data' => '{"storeId":"1","product_ids":["448"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -516,7 +516,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"2","entity_ids":["448"]}', + 'data' => '{"storeId":"2","entityIds":["448"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -526,7 +526,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"3","entity_ids":["448"]}', + 'data' => '{"storeId":"3","entityIds":["448"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -536,7 +536,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => IndicesConfigurator::class, 'method' => 'saveConfigurationToAlgolia', - 'data' => '{"store_id":"1","entity_ids":["40"]}', + 'data' => '{"storeId":"1"}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -546,7 +546,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"2","entity_ids":["40"]}', + 'data' => '{"storeId":"2","entityIds":["40"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -556,7 +556,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Model\IndexMover::class, 'method' => 'moveIndexWithSetSettings', - 'data' => '{"store_id":"3","entity_ids":["40"]}', + 'data' => '{"storeId":"3"}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -566,7 +566,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"1","entity_ids":["405"]}', + 'data' => '{"storeId":"1","entityIds":["405"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -576,7 +576,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Model\IndexMover::class, 'method' => 'moveIndexWithSetSettings', - 'data' => '{"store_id":"2","entity_ids":["405"]}', + 'data' => '{"storeId":"2"}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -586,7 +586,7 @@ public function testMergingWithStaticMethods() 'pid' => null, 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, 'method' => 'buildIndexList', - 'data' => '{"store_id":"3","entity_ids":["405"]}', + 'data' => '{"storeId":"3","entityIds":["405"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -625,9 +625,9 @@ public function testGetJobs() 'job_id' => 1, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreCategoryIndex', - 'data' => '{"store_id":"1","category_ids":["9","22"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"1","entityIds":["9","22"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -636,9 +636,9 @@ public function testGetJobs() 'job_id' => 2, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreCategoryIndex', - 'data' => '{"store_id":"2","category_ids":["9","22"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"2","entityIds":["9","22"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -647,9 +647,9 @@ public function testGetJobs() 'job_id' => 3, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreCategoryIndex', - 'data' => '{"store_id":"3","category_ids":["9","22"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"3","entityIds":["9","22"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -658,9 +658,9 @@ public function testGetJobs() 'job_id' => 4, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreProductIndex', - 'data' => '{"store_id":"1","product_ids":["448"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"1","entityIds":["448"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -669,9 +669,9 @@ public function testGetJobs() 'job_id' => 5, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreProductIndex', - 'data' => '{"store_id":"2","product_ids":["448"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"2","entityIds":["448"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -680,9 +680,9 @@ public function testGetJobs() 'job_id' => 6, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreProductIndex', - 'data' => '{"store_id":"3","product_ids":["448"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"3","entityIds":["448"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -691,9 +691,9 @@ public function testGetJobs() 'job_id' => 7, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreCategoryIndex', - 'data' => '{"store_id":"1","category_ids":["40"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"1","entityIds":["40"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -702,9 +702,9 @@ public function testGetJobs() 'job_id' => 8, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreCategoryIndex', - 'data' => '{"store_id":"2","category_ids":["40"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"2","entityIds":["40"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -713,9 +713,9 @@ public function testGetJobs() 'job_id' => 9, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreCategoryIndex', - 'data' => '{"store_id":"3","category_ids":["40"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"3","entityIds":["40"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -724,9 +724,9 @@ public function testGetJobs() 'job_id' => 10, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreProductIndex', - 'data' => '{"store_id":"1","product_ids":["405"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"1","entityIds":["405"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -735,9 +735,9 @@ public function testGetJobs() 'job_id' => 11, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreProductIndex', - 'data' => '{"store_id":"2","product_ids":["405"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"2","entityIds":["405"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -746,9 +746,9 @@ public function testGetJobs() 'job_id' => 12, 'created' => '2017-09-01 12:00:00', 'pid' => null, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreProductIndex', - 'data' => '{"store_id":"3","product_ids":["405"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"3","entityIds":["405"]}', 'max_retries' => 3, 'retries' => 0, 'error_log' => '', @@ -765,14 +765,14 @@ public function testGetJobs() $expectedFirstJob = [ 'job_id' => '1', 'pid' => $pid, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreCategoryIndex', - 'data' => '{"store_id":"1","category_ids":["9","22"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Category\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"1","entityIds":["9","22"]}', 'merged_ids' => ['1', '7'], 'store_id' => '1', 'decoded_data' => [ - 'store_id' => '1', - 'category_ids' => [ + 'storeId' => '1', + 'entityIds' => [ 0 => '9', 1 => '22', 2 => '40', @@ -783,14 +783,14 @@ public function testGetJobs() $expectedLastJob = [ 'job_id' => '6', 'pid' => $pid, - 'class' => \Algolia\AlgoliaSearch\Helper\Data::class, - 'method' => 'rebuildStoreProductIndex', - 'data' => '{"store_id":"3","product_ids":["448"]}', + 'class' => \Algolia\AlgoliaSearch\Service\Product\IndexBuilder::class, + 'method' => 'buildIndexList', + 'data' => '{"storeId":"3","entityIds":["448"]}', 'merged_ids' => ['6', '12'], 'store_id' => '3', 'decoded_data' => [ - 'store_id' => '3', - 'product_ids' => [ + 'storeId' => '3', + 'entityIds' => [ 0 => '448', 1 => '405', ], @@ -841,8 +841,8 @@ public function testHugeJob() $this->connection->query('TRUNCATE TABLE algoliasearch_queue'); $this->connection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES - (1, NULL, \'class\', \'rebuildStoreProductIndex\', \'{"store_id":"1","product_ids":' . $jsonProductIds . '}\', 3, 0, \'\', 5000), - (2, NULL, \'class\', \'rebuildStoreProductIndex\', \'{"store_id":"2","product_ids":["9","22"]}\', 3, 0, \'\', 2);'); + (1, NULL, \'class\', \'buildIndexList\', \'{"storeId":"1","entityIds":' . $jsonProductIds . '}\', 3, 0, \'\', 5000), + (2, NULL, \'class\', \'buildIndexList\', \'{"storeId":"2","entityIds":["9","22"]}\', 3, 0, \'\', 2);'); $pid = getmypid(); /** @var Job[] $jobs */ @@ -852,7 +852,7 @@ public function testHugeJob() $job = reset($jobs); $this->assertEquals(5000, $job->getDataSize()); - $this->assertEquals(5000, count($job->getDecodedData()['product_ids'])); + $this->assertEquals(5000, count($job->getDecodedData()['entityIds'])); $dbJobs = $this->connection->query('SELECT * FROM algoliasearch_queue')->fetchAll(); @@ -879,8 +879,8 @@ public function testMaxSingleJobSize() $this->connection->query('TRUNCATE TABLE algoliasearch_queue'); $this->connection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES - (1, NULL, \'class\', \'rebuildStoreProductIndex\', \'{"store_id":"1","product_ids":' . $jsonProductIds . '}\', 3, 0, \'\', 99), - (2, NULL, \'class\', \'rebuildStoreProductIndex\', \'{"store_id":"2","product_ids":["9","22"]}\', 3, 0, \'\', 2);'); + (1, NULL, \'class\', \'buildIndexList\', \'{"storeId":"1","entityIds":' . $jsonProductIds . '}\', 3, 0, \'\', 99), + (2, NULL, \'class\', \'buildIndexList\', \'{"storeId":"2","entityIds":["9","22"]}\', 3, 0, \'\', 2);'); $pid = getmypid(); @@ -893,10 +893,10 @@ public function testMaxSingleJobSize() $lastJob = end($jobs); $this->assertEquals(99, $firstJob->getDataSize()); - $this->assertEquals(99, count($firstJob->getDecodedData()['product_ids'])); + $this->assertEquals(99, count($firstJob->getDecodedData()['entityIds'])); $this->assertEquals(2, $lastJob->getDataSize()); - $this->assertEquals(2, count($lastJob->getDecodedData()['product_ids'])); + $this->assertEquals(2, count($lastJob->getDecodedData()['entityIds'])); $dbJobs = $this->connection->query('SELECT * FROM algoliasearch_queue')->fetchAll(); From 5aacec491359c014bd9be0b6e4bd3c86d07de2b9 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Tue, 19 Aug 2025 10:01:08 -0400 Subject: [PATCH 054/119] MAGE-1394 Reorganize command classes --- .../{ => Replica}/AbstractReplicaCommand.php | 4 +- .../{ => Replica}/ReplicaDeleteCommand.php | 2 +- .../ReplicaDisableVirtualCommand.php | 2 +- .../{ => Replica}/ReplicaRebuildCommand.php | 2 +- .../{ => Replica}/ReplicaSyncCommand.php | 2 +- .../Product/MultiStoreReplicaTest.php | 4 +- .../Indexing/Product/ReplicaIndexingTest.php | 12 ++-- etc/di.xml | 62 ++++++++++--------- 8 files changed, 48 insertions(+), 42 deletions(-) rename Console/Command/{ => Replica}/AbstractReplicaCommand.php (61%) rename Console/Command/{ => Replica}/ReplicaDeleteCommand.php (98%) rename Console/Command/{ => Replica}/ReplicaDisableVirtualCommand.php (99%) rename Console/Command/{ => Replica}/ReplicaRebuildCommand.php (98%) rename Console/Command/{ => Replica}/ReplicaSyncCommand.php (98%) diff --git a/Console/Command/AbstractReplicaCommand.php b/Console/Command/Replica/AbstractReplicaCommand.php similarity index 61% rename from Console/Command/AbstractReplicaCommand.php rename to Console/Command/Replica/AbstractReplicaCommand.php index 41706d283..087cc2431 100644 --- a/Console/Command/AbstractReplicaCommand.php +++ b/Console/Command/Replica/AbstractReplicaCommand.php @@ -1,6 +1,8 @@ populateReplicas(1); - $rebuildCmd = $this->objectManager->get(\Algolia\AlgoliaSearch\Console\Command\ReplicaRebuildCommand::class); + $rebuildCmd = $this->objectManager->get(\Algolia\AlgoliaSearch\Console\Command\Replica\ReplicaRebuildCommand::class); $this->invokeMethod( $rebuildCmd, 'execute', @@ -410,7 +408,7 @@ protected function populateReplicas(int $storeId): array { $sorting = $this->objectManager->get(\Algolia\AlgoliaSearch\Service\Product\SortingTransformer::class)->getSortingIndices($storeId, null, null, true); - $cmd = $this->objectManager->get(\Algolia\AlgoliaSearch\Console\Command\ReplicaSyncCommand::class); + $cmd = $this->objectManager->get(\Algolia\AlgoliaSearch\Console\Command\Replica\ReplicaSyncCommand::class); $this->mockProperty($cmd, 'output', \Symfony\Component\Console\Output\OutputInterface::class); diff --git a/etc/di.xml b/etc/di.xml index 26203b472..f6e987b97 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -148,12 +148,10 @@ - Algolia\AlgoliaSearch\Console\Command\ReplicaSyncCommand - Algolia\AlgoliaSearch\Console\Command\ReplicaDeleteCommand - Algolia\AlgoliaSearch\Console\Command\ReplicaRebuildCommand - Algolia\AlgoliaSearch\Console\Command\ReplicaDisableVirtualCommand - Algolia\AlgoliaSearch\Console\Command\SynonymDeduplicateCommand - Algolia\AlgoliaSearch\Console\Command\BatchingOptimizeCommand + Algolia\AlgoliaSearch\Console\Command\Replica\ReplicaSyncCommand + Algolia\AlgoliaSearch\Console\Command\Replica\ReplicaDeleteCommand + Algolia\AlgoliaSearch\Console\Command\Replica\ReplicaRebuildCommand + Algolia\AlgoliaSearch\Console\Command\Replica\ReplicaDisableVirtualCommand Algolia\AlgoliaSearch\Console\Command\Indexer\IndexProductsCommand @@ -167,31 +165,37 @@ Algolia\AlgoliaSearch\Console\Command\Queue\ProcessQueueCommand Algolia\AlgoliaSearch\Console\Command\Queue\ClearQueueCommand + + + Algolia\AlgoliaSearch\Console\Command\SynonymDeduplicateCommand + Algolia\AlgoliaSearch\Console\Command\BatchingOptimizeCommand + - + + Algolia\AlgoliaSearch\Api\Product\ReplicaManagerInterface\Proxy Algolia\AlgoliaSearch\Helper\Entity\ProductHelper\Proxy - + Algolia\AlgoliaSearch\Api\Product\ReplicaManagerInterface\Proxy - + Algolia\AlgoliaSearch\Api\Product\ReplicaManagerInterface\Proxy Algolia\AlgoliaSearch\Helper\Entity\ProductHelper\Proxy - + Algolia\AlgoliaSearch\Helper\ConfigHelper\Proxy Algolia\AlgoliaSearch\Api\Product\ReplicaManagerInterface\Proxy @@ -199,24 +203,6 @@ - - - Algolia\AlgoliaSearch\Service\AlgoliaConnector\Proxy - Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder\Proxy - - - - - - Algolia\AlgoliaSearch\Service\AlgoliaConnector\Proxy - Algolia\AlgoliaSearch\Service\StoreNameFetcher\Proxy - Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder\Proxy - Algolia\AlgoliaSearch\Helper\Entity\ProductHelper\Proxy - Algolia\AlgoliaSearch\Service\Product\RecordBuilder\Proxy - Algolia\AlgoliaSearch\Helper\ConfigHelper\Proxy - - - @@ -275,6 +261,26 @@ + + + + Algolia\AlgoliaSearch\Service\AlgoliaConnector\Proxy + Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder\Proxy + + + + + + Algolia\AlgoliaSearch\Service\AlgoliaConnector\Proxy + Algolia\AlgoliaSearch\Service\StoreNameFetcher\Proxy + Algolia\AlgoliaSearch\Service\Product\IndexOptionsBuilder\Proxy + Algolia\AlgoliaSearch\Helper\Entity\ProductHelper\Proxy + Algolia\AlgoliaSearch\Service\Product\RecordBuilder\Proxy + Algolia\AlgoliaSearch\Helper\ConfigHelper\Proxy + + + + algolia From 024fe0d0f85866ca73593405f3e7cd3ca9dfe8ae Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 20 Aug 2025 16:37:33 +0200 Subject: [PATCH 055/119] MAGE-1396: fix fetchJobs method --- Model/Queue.php | 68 +++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/Model/Queue.php b/Model/Queue.php index ca8e54f82..697cd5d4f 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -114,7 +114,7 @@ public function __construct( * @param int $dataSize * @param bool $isFullReindex */ - public function addToQueue($className, $method, array $data, $dataSize = 1, $isFullReindex = false) + public function addToQueue(string $className, string $method, array $data, int $dataSize = 1, bool $isFullReindex = false): void { if (is_object($className)) { $className = $className::class; @@ -146,7 +146,7 @@ public function addToQueue($className, $method, array $data, $dataSize = 1, $isF * * @return float|null */ - public function getAverageProcessingTime() + public function getAverageProcessingTime(): ?float { $select = $this->db->select() ->from($this->logTable, ['number_of_runs' => 'COUNT(duration)', 'average_time' => 'AVG(duration)']) @@ -165,7 +165,7 @@ public function getAverageProcessingTime() * * @throws Exception */ - public function runCron($nbJobs = null, $force = false) + public function runCron(int $nbJobs = null, bool $force = false): void { if (!$this->configHelper->isQueueActive() && $force === false) { return; @@ -213,7 +213,8 @@ public function runCron($nbJobs = null, $force = false) * @param Job $job * @return string */ - protected function jobToWhereClause(Job $job): string { + protected function jobToWhereClause(Job $job): string + { return sprintf('job_id IN (%s)',implode(',', $job->getMergedIds())); } @@ -222,7 +223,8 @@ protected function jobToWhereClause(Job $job): string { * @return void * @throws \Exception */ - protected function processJob(Job $job): void { + protected function processJob(Job $job): void + { $job->execute(); $where = $this->jobToWhereClause($job); @@ -242,7 +244,8 @@ protected function processJob(Job $job): void { * @param Exception $e * @return void */ - protected function handleFailedJob(Job $job, Exception $e): void { + protected function handleFailedJob(Job $job, Exception $e): void + { $this->noOfFailedJobs++; // Log error information @@ -284,7 +287,7 @@ protected function handleFailedJob(Job $job, Exception $e): void { * * @throws Exception */ - public function run($maxJobs) + public function run(int $maxJobs): void { $this->clearOldFailingJobs(); @@ -375,12 +378,12 @@ protected function archiveSuccessfulJobs(string $whereClause): void { /** * @param int $maxJobs * - * @throws Exception - * * @return Job[] * + * @throws Exception + * */ - protected function getJobs($maxJobs) + protected function getJobs(int $maxJobs): array { $maxJobs = ($maxJobs === -1) ? $this->configHelper->getNumberOfJobToRun() : $maxJobs; @@ -432,14 +435,14 @@ protected function getJobs($maxJobs) * * @return Job[] */ - protected function fetchJobs($jobsLimit, $fetchFullReindexJobs = false, $lastJobId = null) + protected function fetchJobs(int $jobsLimit, bool $fetchFullReindexJobs = false, int $lastJobId = null): array { $jobs = []; $actualBatchSize = 0; - $maxBatchSize = $this->configHelper->getNumberOfElementByPage() * $jobsLimit; + $maxBatchSize = $this->configHelper->getNumberOfElementByPage(); - $limit = $maxJobs = $jobsLimit; + $limit = $jobsLimit; $offset = 0; $fetchFullReindexJobs = $fetchFullReindexJobs ? 1 : 0; @@ -470,27 +473,26 @@ protected function fetchJobs($jobsLimit, $fetchFullReindexJobs = false, $lastJob $rawJobsCount = count($rawJobs); $offset += $limit; - $limit = max(0, $maxJobs - $rawJobsCount); + $limit = max(0, $jobsLimit - $rawJobsCount); // $jobs will always be completely set from $rawJobs // Without resetting not-merged jobs would be stacked $jobs = []; - if (count($rawJobs) === $maxJobs) { + // At this point, if this condition is true, this means that no merge was possible in the last iteration + if (count($rawJobs) === $jobsLimit) { $jobs = $rawJobs; break; } + $jobSizes = []; + foreach ($rawJobs as $job) { $jobSize = (int) $job->getDataSize(); - - if ($actualBatchSize + $jobSize <= $maxBatchSize || !$jobs) { - $jobs[] = $job; - $actualBatchSize += $jobSize; - } else { - break 2; - } + $jobSizes[$job->getId()] = $jobSize; + $actualBatchSize = array_sum($jobSizes); + $jobs[] = $job; } } @@ -502,7 +504,7 @@ protected function fetchJobs($jobsLimit, $fetchFullReindexJobs = false, $lastJob * * @return Job[] */ - protected function mergeJobs(array $unmergedJobs) + protected function mergeJobs(array $unmergedJobs): array { $unmergedJobs = $this->sortJobs($unmergedJobs); @@ -539,7 +541,7 @@ protected function mergeJobs(array $unmergedJobs) * * @return Job[] */ - protected function sortJobs(array $jobs) + protected function sortJobs(array $jobs): array { $sortedJobs = []; @@ -571,7 +573,7 @@ protected function sortJobs(array $jobs) * * @return array */ - protected function stackSortedJobs(array $sortedJobs, array $tempSortableJobs, ?Job $job = null) + protected function stackSortedJobs(array $sortedJobs, array $tempSortableJobs, ?Job $job = null): array { if ($tempSortableJobs && $tempSortableJobs !== []) { $tempSortableJobs = $this->jobSort( @@ -599,7 +601,7 @@ protected function stackSortedJobs(array $sortedJobs, array $tempSortableJobs, ? /** * @return array */ - protected function jobSort() + protected function jobSort(): array { $args = func_get_args(); @@ -631,7 +633,7 @@ protected function jobSort() /** * @param Job[] $jobs */ - protected function lockJobs(array $jobs) + protected function lockJobs(array $jobs): void { $jobsIds = $this->getJobsIdsFromMergedJobs($jobs); @@ -657,7 +659,7 @@ protected function lockJobs(array $jobs) * * @return string[] */ - protected function getJobsIdsFromMergedJobs(array $mergedJobs) + protected function getJobsIdsFromMergedJobs(array $mergedJobs): array { $jobsIds = []; foreach ($mergedJobs as $job) { @@ -670,7 +672,7 @@ protected function getJobsIdsFromMergedJobs(array $mergedJobs) /** * @return void */ - protected function clearOldFailingJobs() + protected function clearOldFailingJobs(): void { // Enhanced archive will have already logged this failure if (!$this->configHelper->isEnhancedQueueArchiveEnabled()) { @@ -684,7 +686,7 @@ protected function clearOldFailingJobs() /** * @throws Zend_Db_Statement_Exception */ - protected function clearOldLogRecords() + protected function clearOldLogRecords(): void { $select = $this->db->select() ->from($this->logTable, ['id']) @@ -701,7 +703,7 @@ protected function clearOldLogRecords() /** * @return void */ - protected function clearOldArchiveRecords() + protected function clearOldArchiveRecords(): void { $archiveLogClearLimit = $this->configHelper->getArchiveLogClearLimit(); // Adding a fallback in case this configuration was not set in a consistent way @@ -718,7 +720,7 @@ protected function clearOldArchiveRecords() /** * @return void */ - protected function unlockStackedJobs() + protected function unlockStackedJobs(): void { $this->db->update($this->table, [ 'locked_at' => null, @@ -729,7 +731,7 @@ protected function unlockStackedJobs() /** * @return bool */ - protected function shouldEmptyQueue() + protected function shouldEmptyQueue(): bool { if (getenv('PROCESS_FULL_QUEUE') && getenv('PROCESS_FULL_QUEUE') === '1') { return true; From b29bed99c18faf99343ea444bfd893cbfbe45223 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 20 Aug 2025 16:37:50 +0200 Subject: [PATCH 056/119] MAGE-1396: fix fetchJobs method --- Model/Queue.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Model/Queue.php b/Model/Queue.php index 697cd5d4f..cd4eb978a 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -486,6 +486,8 @@ protected function fetchJobs(int $jobsLimit, bool $fetchFullReindexJobs = false, break; } + // Introduced an array of job sizes to determine the total batch size currently processed (sum of all jobs contained in the run) + // This will determine if we can continue to loop over the jobs $jobSizes = []; foreach ($rawJobs as $job) { From 2bb83c540e835aeed6df6423cb53836812375c46 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 20 Aug 2025 16:50:06 +0200 Subject: [PATCH 057/119] MAGE-1396: update queue notice --- Helper/Configuration/NoticeHelper.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Helper/Configuration/NoticeHelper.php b/Helper/Configuration/NoticeHelper.php index 0c7e7ef44..bebba33c3 100644 --- a/Helper/Configuration/NoticeHelper.php +++ b/Helper/Configuration/NoticeHelper.php @@ -100,6 +100,8 @@ protected function getQueueNotice() in approx. ' . $eta . '. You may want to clear the queue or configure indexing queue.

+ Depending on your configuration set on "Advanced > Maximum number of records processed per indexing job" and if the jobs can be merged into batches, you can expect higher performances. +

Find out more about Indexing Queue in documentation.'; } From c4f7f854657b0b39a6c3e14cddf0e75ca56be1de Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 20 Aug 2025 17:01:11 +0200 Subject: [PATCH 058/119] MAGE-1396: small fix --- Model/Queue.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Model/Queue.php b/Model/Queue.php index cd4eb978a..7209bdd6e 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -493,9 +493,10 @@ protected function fetchJobs(int $jobsLimit, bool $fetchFullReindexJobs = false, foreach ($rawJobs as $job) { $jobSize = (int) $job->getDataSize(); $jobSizes[$job->getId()] = $jobSize; - $actualBatchSize = array_sum($jobSizes); $jobs[] = $job; } + + $actualBatchSize = array_sum($jobSizes); } return $jobs; From e422e69860f0ae27e155bd0f92205666165a2407 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 20 Aug 2025 15:35:44 -0400 Subject: [PATCH 059/119] MAGE-1394 Add unit tests for new Command class plus base class to centralize reflection helper methods --- Console/Command/Queue/ClearQueueCommand.php | 20 +- Test/Integration/TestCase.php | 31 +- Test/TestCase.php | 60 +++ .../Command/Queue/ClearQueueCommandTest.php | 406 ++++++++++++++++++ etc/di.xml | 4 +- 5 files changed, 479 insertions(+), 42 deletions(-) create mode 100644 Test/TestCase.php create mode 100644 Test/Unit/Console/Command/Queue/ClearQueueCommandTest.php diff --git a/Console/Command/Queue/ClearQueueCommand.php b/Console/Command/Queue/ClearQueueCommand.php index a35011d25..529d3a466 100644 --- a/Console/Command/Queue/ClearQueueCommand.php +++ b/Console/Command/Queue/ClearQueueCommand.php @@ -69,7 +69,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $output->writeln($this->decorateOperationAnnouncementMessage('Clearing indexing queue for {{target}}', $storeIds)); try { - $this->clearIndexingQueue($storeIds); + $this->clearQueue($storeIds); } catch (\Exception $e) { $this->output->writeln('' . $e->getMessage() . ''); return Cli::RETURN_FAILURE; @@ -100,14 +100,14 @@ protected function confirmClearOperation(): bool * @param array $storeIds * @throws NoSuchEntityException|LocalizedException */ - public function clearIndexingQueue(array $storeIds = []): void + protected function clearQueue(array $storeIds = []): void { if (count($storeIds)) { foreach ($storeIds as $storeId) { - $this->clearIndexingQueueForStore($storeId); + $this->clearQueueForStore($storeId); } } else { - $this->clearIndexingQueueForAllStores(); + $this->clearQueueForAllStores(); } } @@ -117,13 +117,13 @@ public function clearIndexingQueue(array $storeIds = []): void * @param int $storeId * @throws NoSuchEntityException */ - public function clearIndexingQueueForStore(int $storeId): void + protected function clearQueueForStore(int $storeId): void { $storeName = $this->storeNameFetcher->getStoreName($storeId); $this->output->writeln('Clearing indexing queue for ' . $storeName . '...'); try { - $this->clearQueueForStore($storeId); + $this->clearQueueTableForStore($storeId); $this->output->writeln('✓ Indexing queue cleared for ' . $storeName . ''); } catch (\Exception $e) { @@ -137,7 +137,7 @@ public function clearIndexingQueueForStore(int $storeId): void * @throws NoSuchEntityException * @throws LocalizedException */ - public function clearIndexingQueueForAllStores(): void + protected function clearQueueForAllStores(): void { $connection = $this->jobResourceModel->getConnection(); $connection->truncateTable($this->jobResourceModel->getMainTable()); @@ -150,7 +150,7 @@ public function clearIndexingQueueForAllStores(): void * @param int $storeId * @throws \Exception */ - protected function clearQueueForStore(int $storeId): void + protected function clearQueueTableForStore(int $storeId): void { try { // Use JSON_EXTRACT to filter by store_id in the data field @@ -180,7 +180,7 @@ protected function clearQueueForStore(int $storeId): void } catch (\Exception $e) { // Fallback method if JSON_EXTRACT is not supported $this->output->writeln('JSON filtering not supported by database, using fallback method...'); - $this->clearQueueForStoreFallback($storeId); + $this->clearQueueTableForStoreFallback($storeId); } } @@ -191,7 +191,7 @@ protected function clearQueueForStore(int $storeId): void * @param int $storeId * @throws \Exception */ - protected function clearQueueForStoreFallback(int $storeId): void + protected function clearQueueTableForStoreFallback(int $storeId): void { try { $connection = $this->jobResourceModel->getConnection(); diff --git a/Test/Integration/TestCase.php b/Test/Integration/TestCase.php index 7129d1981..04153ed8a 100644 --- a/Test/Integration/TestCase.php +++ b/Test/Integration/TestCase.php @@ -19,7 +19,7 @@ use Magento\Store\Model\ScopeInterface; use Magento\TestFramework\Helper\Bootstrap; -abstract class TestCase extends \PHPUnit\Framework\TestCase +abstract class TestCase extends \Algolia\AlgoliaSearch\Test\TestCase { /** * @var ObjectManagerInterface @@ -230,35 +230,6 @@ private function bootstrap() $this->boostrapped = true; } - - /** - * @throws \ReflectionException - */ - protected function mockProperty(object $object, string $propertyName, string $propertyClass): void - { - $mock = $this->createMock($propertyClass); - $reflection = new \ReflectionClass($object); - $property = $reflection->getProperty($propertyName); - $property->setValue($object, $mock); - } - - /** - * Call protected/private method of a class. - * - * @param object $object instantiated object that we will run method on - * @param string $methodName Method name to call - * @param array $parameters array of parameters to pass into method - * - * @throws \ReflectionException - * - * @return mixed method return - */ - protected function invokeMethod(object $object, string $methodName, array $parameters = []) - { - $reflection = new \ReflectionClass($object::class); - return $reflection->getMethod($methodName)->invokeArgs($object, $parameters); - } - private function getMagentoVersion() { return $this->productMetadata->getVersion(); diff --git a/Test/TestCase.php b/Test/TestCase.php new file mode 100644 index 000000000..9a490b135 --- /dev/null +++ b/Test/TestCase.php @@ -0,0 +1,60 @@ +getMethod($methodName)->invokeArgs($object, $parameters); + } + + /** + * Set a private property of a class + * + * @param object $obj The object to set the property for + * @param string $prop The name of the property to set + * @param mixed $value The value to set the property to + * + * @throws \ReflectionException + */ + protected function setPrivateProperty(object $obj, string $prop, mixed $value): void + { + $ref = new \ReflectionClass($obj); + $p = $ref->getProperty($prop); + $p->setValue($obj, $value); + } + + /** + * Mock a private property of a class + * + * @param object $object The object to mock the property for + * @param string $propertyName The name of the property to mock + * @param string $propertyClass The class of the property to mock + * + * @throws \ReflectionException + */ + protected function mockProperty(object $object, string $propertyName, string $propertyClass): void + { + $mock = $this->createMock($propertyClass); + $reflection = new \ReflectionClass($object); + $property = $reflection->getProperty($propertyName); + $property->setValue($object, $mock); + } +} diff --git a/Test/Unit/Console/Command/Queue/ClearQueueCommandTest.php b/Test/Unit/Console/Command/Queue/ClearQueueCommandTest.php new file mode 100644 index 000000000..56e589a03 --- /dev/null +++ b/Test/Unit/Console/Command/Queue/ClearQueueCommandTest.php @@ -0,0 +1,406 @@ +state = $this->createMock(State::class); + $this->storeNameFetcher = $this->createMock(StoreNameFetcher::class); + $this->storeManager = $this->createMock(StoreManagerInterface::class); + $this->jobResourceModel = $this->createMock(JobResourceModel::class); + } + + /** + * Test that the command returns success when the user cancels the operation + * @throws \ReflectionException + */ + public function testExecuteReturnsSuccessWhenUserCancels(): void + { + $cmd = $this->makePartial(['setAreaCode', 'confirmOperation', 'getStoreIds', 'decorateOperationAnnouncementMessage', 'clearQueue']); + + $cmd->expects($this->once())->method('setAreaCode'); + $cmd->expects($this->once())->method('confirmOperation')->willReturn(false); + $cmd->expects($this->never())->method('getStoreIds'); + $cmd->expects($this->never())->method('clearQueue'); + + $input = new ArrayInput([]); + $output = $this->bufOut(); + + $code = $this->invokeExecute($cmd, $input, $output); + $this->assertSame(Cli::RETURN_SUCCESS, $code); + } + + /** + * Test that the command clears the indexing queue for the provided store IDs + * @throws \ReflectionException + */ + public function testExecuteClearsProvidedStoreIds(): void + { + $cmd = $this->makePartial(['setAreaCode', 'confirmOperation', 'getStoreIds', 'decorateOperationAnnouncementMessage', 'clearQueue']); + + $cmd->method('setAreaCode'); + $cmd->method('confirmOperation')->willReturn(true); + $cmd->method('getStoreIds')->willReturn([1, 2]); + + $msg = 'Clearing indexing queue for stores 1, 2'; + $cmd->method('decorateOperationAnnouncementMessage')->willReturn($msg); + + $cmd->expects($this->once()) + ->method('clearQueue') + ->with([1,2]); + + $input = new ArrayInput([]); + $output = $this->bufOut(); + + $code = $this->invokeExecute($cmd, $input, $output); + $this->assertSame(Cli::RETURN_SUCCESS, $code); + + $this->assertStringContainsString($msg, $output->fetch()); + } + + /** + * Test that the command returns failure when the clear operation fails + * + * @throws \ReflectionException + */ + public function testExecuteReturnsFailureOnClearException(): void + { + $cmd = $this->makePartial(['setAreaCode', 'confirmOperation', 'getStoreIds', 'decorateOperationAnnouncementMessage', 'clearQueue']); + + $cmd->method('setAreaCode'); + $cmd->method('confirmOperation')->willReturn(true); + $cmd->method('getStoreIds')->willReturn([]); + $cmd->method('decorateOperationAnnouncementMessage')->willReturn('Clearing indexing queue for all stores'); + + $errMsg = "Error encountered while attempting to clear queue."; + $cmd->method('clearQueue')->willThrowException(new \Exception($errMsg)); + + $input = new ArrayInput([]); + $output = $this->bufOut(); + + $code = $this->invokeExecute($cmd, $input, $output); + $this->assertSame(Cli::RETURN_FAILURE, $code); + $this->assertStringContainsString($errMsg, $output->fetch()); + } + + /** + * Test that the command calls the clearQueueForStore method for each store ID + * + * @throws \ReflectionException + */ + public function testClearQueueCallsPerStore(): void + { + $cmd = $this->makePartial(['clearQueueForStore', 'clearQueueForAllStores']); + + $expectedStoreIds = [1, 2]; + $callIndex = 0; + $cmd->expects($this->exactly(2)) + ->method('clearQueueForStore') + ->willReturnCallback(function($storeId) use (&$callIndex, $expectedStoreIds) { + $this->assertSame( + $expectedStoreIds[$callIndex], + $storeId, + "clearQueueForStore called with unexpected storeId at call $callIndex" + ); + $callIndex++; + }); + + $cmd->expects($this->never())->method('clearQueueForAllStores'); + + $this->invokeMethod($cmd, 'clearQueue', [[1, 2]]); + } + + /** + * Test that the command calls the clearQueueForAllStores method when no store IDs are provided + * + * @throws \ReflectionException + */ + public function testClearQueueEmptyCallsAllStores(): void + { + $cmd = $this->makePartial(['clearQueueForStore', 'clearQueueForAllStores']); + + $cmd->expects($this->never())->method('clearQueueForStore'); + $cmd->expects($this->once())->method('clearQueueForAllStores'); + + $this->invokeMethod($cmd, 'clearQueue'); + } + + /** + * Test that the command truncates the main table when no store IDs are provided + * + * @throws \ReflectionException + */ + public function testClearQueueForAllStoresTruncatesMainTable(): void + { + $adapter = $this->createMock(AdapterInterface::class); + + $this->jobResourceModel->method('getConnection')->willReturn($adapter); + $this->jobResourceModel->method('getMainTable')->willReturn('algoliasearch_queue'); + + $adapter->expects($this->once())->method('truncateTable')->with('algoliasearch_queue'); + + $cmd = $this->makePartial(); + $this->invokeMethod($cmd, 'clearQueue'); + } + + /** + * @throws \ReflectionException + */ + public function testClearQueueForStoreSuccess(): void + { + $cmd = $this->makePartial(['clearQueueTableForStore']); + $output = $this->bufOut(); + + // Inject output (protected on the parent); easiest is via reflection: + $this->setPrivateProperty($cmd, 'output', $output); + + $this->storeNameFetcher->method('getStoreName')->with(1)->willReturn('Default Store'); + $cmd->expects($this->once())->method('clearQueueTableForStore')->with(1); + + $this->invokeMethod($cmd, 'clearQueueForStore', [1]); + + $text = $output->fetch(); + $this->assertStringContainsString('Clearing indexing queue for Default Store', $text); + $this->assertStringContainsString('Indexing queue cleared for Default Store', $text); + } + + /** + * @throws \ReflectionException + * @throws NoSuchEntityException + */ + public function testClearQueueForStoreErrorPrinted(): void + { + $cmd = $this->makePartial(['clearQueueTableForStore']); + $output = $this->bufOut(); + $this->setPrivateProperty($cmd, 'output', $output); + + $this->storeNameFetcher->method('getStoreName')->with(1)->willReturn('Default Store'); + $errorMsg = 'DB operation failed'; + $cmd->method('clearQueueTableForStore')->willThrowException(new \Exception($errorMsg)); + + $this->invokeMethod($cmd, 'clearQueueForStore', [1]); + + $this->assertStringContainsString("Failed to clear indexing queue for Default Store: $errorMsg", $output->fetch()); + } + + /** + * @throws \ReflectionException + */ + public function testClearQueueForStoreJsonPathDeletesJobs(): void + { + $adapter = $this->getMockSearchAdapter(); + $output = $this->bufOut(); + + $cmd = $this->makePartial(); + + $this->setPrivateProperty($cmd, 'output', $output); + + $this->jobResourceModel->method('getConnection')->willReturn($adapter); + $this->jobResourceModel->method('getMainTable')->willReturn('algoliasearch_queue'); + + $adapter->expects($this->once()) + ->method('fetchCol') + ->willReturn([10, 11]); + + $adapter->expects($this->once()) + ->method('delete') + ->with('algoliasearch_queue', ['job_id IN (?)' => [10,11]]) + ->willReturn(2); + + $this->invokeMethod($cmd, 'clearQueueForStore', [5]); + + $this->assertStringContainsString('Deleted 2 jobs for store ID 5', $output->fetch()); + } + + /** + * @throws NoSuchEntityException + * @throws \ReflectionException + */ + public function testClearQueueForStoreJsonPathNoJobs(): void + { + $adapter = $this->getMockSearchAdapter(); + $output = $this->bufOut(); + + $cmd = $this->makePartial(); + + $this->setPrivateProperty($cmd, 'output', $output); + + $this->jobResourceModel->method('getConnection')->willReturn($adapter); + $this->jobResourceModel->method('getMainTable')->willReturn('algoliasearch_queue'); + + $adapter->method('fetchCol')->willReturn([]); + + $this->invokeMethod($cmd, 'clearQueueForStore', [5]); + + $this->assertStringContainsString('No jobs found for store ID 5', $output->fetch()); + } + + /** + * @throws NoSuchEntityException + * @throws \ReflectionException + */ + public function testClearQueueForStoreFallsBackWhenJsonThrows(): void + { + $cmd = $this->makePartial(['clearQueueTableForStoreFallback']); + $output = $this->bufOut(); + $this->setPrivateProperty($cmd, 'output', $output); + + $adapter = $this->getMockSearchAdapter(); + $this->jobResourceModel->method('getConnection')->willReturn($adapter); + + $adapter->method('select')->willThrowException(new \Exception('No JSON support')); + + $cmd->expects($this->once())->method('clearQueueTableForStoreFallback')->with(5); + + $this->invokeMethod($cmd, 'clearQueueForStore', [5]); + + $this->assertStringContainsString('JSON filtering not supported', $output->fetch()); + } + + /** + * @throws \ReflectionException + */ + public function testClearQueueForStoreFallbackDeletesMatching(): void + { + $cmd = $this->makePartial(); + $output = $this->bufOut(); + $this->setPrivateProperty($cmd, 'output', $output); + + $adapter = $this->getMockSearchAdapter(); + + $this->jobResourceModel->method('getConnection')->willReturn($adapter); + $this->jobResourceModel->method('getMainTable')->willReturn('algoliasearch_queue'); + + $adapter->method('fetchAll')->willReturn([ + ['job_id' => 1, 'data' => '{"storeId":5,"foo":1}'], + ['job_id' => 2, 'data' => '{"storeId":7}'], + ['job_id' => 3, 'data' => '{"storeId":5}'], + ]); + + $adapter->expects($this->once()) + ->method('delete') + ->with('algoliasearch_queue', ['job_id IN (?)' => [1,3]]) + ->willReturn(2); + + $this->invokeMethod($cmd, 'clearQueueTableForStoreFallback', [5]); + + $this->assertStringContainsString('Deleted 2 jobs for store ID 5 (fallback method)', $output->fetch()); + } + + /** + * @throws \ReflectionException + */ + public function testClearQueueForStoreFallbackNoMatch(): void + { + $cmd = $this->makePartial(); + $output = $this->bufOut(); + $this->setPrivateProperty($cmd, 'output', $output); + + $adapter = $this->getMockSearchAdapter(); + $this->jobResourceModel->method('getConnection')->willReturn($adapter); + + $adapter->method('fetchAll')->willReturn([ + ['job_id' => 1, 'data' => '{"storeId":8}'], + ]); + + $this->invokeMethod($cmd, 'clearQueueTableForStoreFallback', [5]); + + $this->assertStringContainsString('No jobs found for store ID 5 (fallback method)', $output->fetch()); + } + + public function testClearQueueForStoreFallbackThrowsWrapped(): void + { + $cmd = $this->makePartial(); + $adapter = $this->getMockSearchAdapter(); + $this->jobResourceModel->method('getConnection')->willReturn($adapter); + + $adapter->method('fetchAll')->willThrowException(new \Exception('db fail')); + + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Failed to clear queue for store 5: db fail'); + + $this->invokeMethod($cmd, 'clearQueueTableForStoreFallback', [5]); + } + + public function testMetadataStrings(): void + { + $cmd = $this->makePartial(); + $this->assertSame('clear', $this->invokeMethod($cmd, 'getCommandName')); + $this->assertStringContainsString('queue:', $this->invokeMethod($cmd, 'getCommandPrefix')); + $this->assertStringContainsString('Clear the indexing queue', $this->invokeMethod($cmd, 'getCommandDescription')); + $this->assertStringContainsString('algolia:queue:clear', $this->invokeMethod($cmd, 'getStoreArgumentDescription')); + } + + /** + * Create a partial mock of the ClearQueueCommand + * + * @param array $methodsToMock List of methods to mock + * @return ClearQueueCommand&MockObject + */ + private function makePartial(array $methodsToMock = []): ClearQueueCommand&MockObject + { + /** @var ClearQueueCommand&MockObject $cmd */ + $cmd = $this->getMockBuilder(ClearQueueCommand::class) + ->setConstructorArgs([ + $this->state, + $this->storeNameFetcher, + $this->storeManager, + $this->jobResourceModel, + null + ]) + ->onlyMethods($methodsToMock) + ->getMock(); + + return $cmd; + } + + private function getMockSearchAdapter(): AdapterInterface&MockObject + { + $adapter = $this->createMock(AdapterInterface::class); + $select = $this->createMock(Select::class); + + // Build chainable select + $adapter->method('select')->willReturn($select); + $select->method('from')->willReturnSelf(); + $select->method('where')->willReturnSelf(); + + return $adapter; + } + + private function bufOut(): BufferedOutput + { + // Verbosity NORMAL is fine; change if you want debug output + return new BufferedOutput(); + } + + /** + * @throws \ReflectionException + */ + private function invokeExecute(ClearQueueCommand $cmd, $input, $output): int + { + return $this->invokeMethod($cmd, 'execute', [$input, $output]); + } + +} diff --git a/etc/di.xml b/etc/di.xml index f6e987b97..31dbd6a2c 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -163,8 +163,8 @@ Algolia\AlgoliaSearch\Console\Command\Indexer\IndexAllCommand - Algolia\AlgoliaSearch\Console\Command\Queue\ProcessQueueCommand - Algolia\AlgoliaSearch\Console\Command\Queue\ClearQueueCommand + Algolia\AlgoliaSearch\Console\Command\Queue\ProcessQueueCommand + Algolia\AlgoliaSearch\Console\Command\Queue\ClearQueueCommand Algolia\AlgoliaSearch\Console\Command\SynonymDeduplicateCommand From 399cfeea837d9dc6556e3ceaff63107fd8da370d Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 20 Aug 2025 15:43:37 -0400 Subject: [PATCH 060/119] MAGE-1394 Address Codacy complaint --- Console/Command/Queue/ClearQueueCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Console/Command/Queue/ClearQueueCommand.php b/Console/Command/Queue/ClearQueueCommand.php index 529d3a466..413ac27ea 100644 --- a/Console/Command/Queue/ClearQueueCommand.php +++ b/Console/Command/Queue/ClearQueueCommand.php @@ -226,7 +226,7 @@ protected function clearQueueTableForStoreFallback(int $storeId): void $this->output->writeln('Deleted ' . $deletedCount . ' jobs for store ID ' . $storeId . ' (fallback method)'); } catch (\Exception $e) { - throw new \Exception('Failed to clear queue for store ' . $storeId . ': ' . $e->getMessage()); + throw new \Exception(sprintf('Failed to clear queue for store %d: %s', $storeId, $e->getMessage())); } } } From 002f8fd980e8cbd80bcbcd0306ccf1f0b8cbdebf Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 20 Aug 2025 15:59:05 -0400 Subject: [PATCH 061/119] MAGE-1394 Address Codacy complaint --- Console/Command/Queue/ClearQueueCommand.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Console/Command/Queue/ClearQueueCommand.php b/Console/Command/Queue/ClearQueueCommand.php index 413ac27ea..d1ad6846b 100644 --- a/Console/Command/Queue/ClearQueueCommand.php +++ b/Console/Command/Queue/ClearQueueCommand.php @@ -226,6 +226,10 @@ protected function clearQueueTableForStoreFallback(int $storeId): void $this->output->writeln('Deleted ' . $deletedCount . ' jobs for store ID ' . $storeId . ' (fallback method)'); } catch (\Exception $e) { + /** + * Escaping complaints represent a false positive as sprintf is used to convert the $storeId to an int and the error message is generated internally + * phpcs:ignore + */ throw new \Exception(sprintf('Failed to clear queue for store %d: %s', $storeId, $e->getMessage())); } } From 8ee22e966524d6d230a9b3515e663eb9dbb438ff Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 20 Aug 2025 16:06:42 -0400 Subject: [PATCH 062/119] MAGE-1394 Address Codacy complaint --- Console/Command/Queue/ClearQueueCommand.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Console/Command/Queue/ClearQueueCommand.php b/Console/Command/Queue/ClearQueueCommand.php index d1ad6846b..d50a6bf11 100644 --- a/Console/Command/Queue/ClearQueueCommand.php +++ b/Console/Command/Queue/ClearQueueCommand.php @@ -226,10 +226,8 @@ protected function clearQueueTableForStoreFallback(int $storeId): void $this->output->writeln('Deleted ' . $deletedCount . ' jobs for store ID ' . $storeId . ' (fallback method)'); } catch (\Exception $e) { - /** - * Escaping complaints represent a false positive as sprintf is used to convert the $storeId to an int and the error message is generated internally - * phpcs:ignore - */ + // Safe: $storeId is cast to int, exception messages are internal. + // phpcs:ignore throw new \Exception(sprintf('Failed to clear queue for store %d: %s', $storeId, $e->getMessage())); } } From 62737b050bc908358b08a3ea640069d6e1b1ed8f Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Mon, 25 Aug 2025 10:52:24 +0200 Subject: [PATCH 063/119] MAGE-1100: cleanup --- Model/Job.php | 45 ++++++++++++++------------------------------- Model/Queue.php | 47 ++++++----------------------------------------- 2 files changed, 20 insertions(+), 72 deletions(-) diff --git a/Model/Job.php b/Model/Job.php index c1ac8e680..f5a007d10 100644 --- a/Model/Job.php +++ b/Model/Job.php @@ -32,28 +32,15 @@ class Job extends \Magento\Framework\Model\AbstractModel implements JobInterface { protected $_eventPrefix = 'algoliasearch_queue_job'; - /** @var ObjectManagerInterface */ - protected ObjectManagerInterface $objectManager; - - /** - * @param Context $context - * @param Registry $registry - * @param ObjectManagerInterface $objectManager - * @param AbstractResource|null $resource - * @param AbstractDb|null $resourceCollection - * @param array $data - */ public function __construct( - Context $context, - Registry $registry, - ObjectManagerInterface $objectManager, - ?AbstractResource $resource = null, - ?AbstractDb $resourceCollection = null, - array $data = [] + protected Context $context, + protected Registry $registry, + protected ObjectManagerInterface $objectManager, + protected ?AbstractResource $resource = null, + protected ?AbstractDb $resourceCollection = null, + array $data = [] ) { parent::__construct($context, $registry, $resource, $resourceCollection, $data); - - $this->objectManager = $objectManager; } /** @@ -61,7 +48,7 @@ public function __construct( * * @return void */ - protected function _construct() + protected function _construct(): void { $this->_init(ResourceModel\Job::class); } @@ -71,7 +58,7 @@ protected function _construct() * @throws AlreadyExistsException|\Exception * */ - public function execute() + public function execute(): Job { $model = $this->objectManager->get($this->getClass()); $method = $this->getMethod(); @@ -89,7 +76,7 @@ public function execute() /** * @return $this */ - public function prepare() + public function prepare(): Job { if ($this->getMergedIds() === null) { $this->setMergedIds([$this->getId()]); @@ -100,10 +87,6 @@ public function prepare() $this->setDecodedData($decodedData); - if (isset($decodedData['store_id'])) { - $this->setStoreId($decodedData['store_id']); - } - if (isset($decodedData['storeId'])) { $this->setStoreId($decodedData['storeId']); } @@ -118,7 +101,7 @@ public function prepare() * * @return bool */ - public function canMerge(Job $job, $maxJobDataSize) + public function canMerge(Job $job, $maxJobDataSize): bool { if ($this->getClass() !== $job->getClass()) { return false; @@ -156,7 +139,7 @@ public function canMerge(Job $job, $maxJobDataSize) * * @return Job */ - public function merge(Job $mergedJob) + public function merge(Job $mergedJob): Job { $mergedIds = $this->getMergedIds(); array_push($mergedIds, $mergedJob->getId()); @@ -186,7 +169,7 @@ public function merge(Job $mergedJob) /** * @return array */ - public function getDefaultValues() + public function getDefaultValues(): array { $values = []; @@ -196,7 +179,7 @@ public function getDefaultValues() /** * @return string */ - public function getStatus() + public function getStatus(): string { $status = JobInterface::STATUS_PROCESSING; @@ -218,7 +201,7 @@ public function getStatus() * * @return Job */ - public function saveError(\Exception $e) + public function saveError(\Exception $e): Job { $this->setErrorLog($e->getMessage()); $this->getResource()->save($this); diff --git a/Model/Queue.php b/Model/Queue.php index 7209bdd6e..3c5dea6f9 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -21,9 +21,6 @@ class Queue public const UNLOCK_STACKED_JOBS_AFTER_MINUTES = 15; public const CLEAR_ARCHIVE_LOGS_AFTER_DAYS = 30; - public const SUCCESS_LOG = 'algoliasearch_queue_log.txt'; - public const ERROR_LOG = 'algoliasearch_queue_errors.log'; - public const FAILED_JOB_ARCHIVE_CRITERIA = 'retries >= max_retries'; public const MOVE_INDEX_METHOD_NAME = 'moveIndexWithSetSettings'; @@ -39,23 +36,9 @@ class Queue /** @var string */ protected $archiveTable; - /** @var ObjectManagerInterface */ - protected $objectManager; - - /** @var ConsoleOutput */ - protected $output; - /** @var int */ protected $elementsPerPage; - /** @var ConfigHelper */ - protected $configHelper; - - /** @var DiagnosticsLogger */ - protected $logger; - - protected $jobCollectionFactory; - /** @var int */ protected $maxSingleJobDataSize; @@ -72,38 +55,20 @@ class Queue /** @var array */ protected $logRecord; - /** - * @param ConfigHelper $configHelper - * @param DiagnosticsLogger $logger - * @param JobCollectionFactory $jobCollectionFactory - * @param ResourceConnection $resourceConnection - * @param ObjectManagerInterface $objectManager - * @param ConsoleOutput $output - */ public function __construct( - ConfigHelper $configHelper, - DiagnosticsLogger $logger, - JobCollectionFactory $jobCollectionFactory, - ResourceConnection $resourceConnection, - ObjectManagerInterface $objectManager, - ConsoleOutput $output + protected ConfigHelper $configHelper, + protected DiagnosticsLogger $logger, + protected JobCollectionFactory $jobCollectionFactory, + protected ResourceConnection $resourceConnection, + protected ObjectManagerInterface $objectManager, + protected ConsoleOutput $output ) { - $this->configHelper = $configHelper; - $this->logger = $logger; - $this->jobCollectionFactory = $jobCollectionFactory; - $this->table = $resourceConnection->getTableName('algoliasearch_queue'); $this->logTable = $resourceConnection->getTableName('algoliasearch_queue_log'); $this->archiveTable = $resourceConnection->getTableName('algoliasearch_queue_archive'); - - //$this->db = $resourceConnection->getConnection(); - - $this->objectManager = $objectManager; $this->db = $objectManager->create(ResourceConnection::class)->getConnection('core_write'); - $this->output = $output; $this->elementsPerPage = $this->configHelper->getNumberOfElementByPage(); - $this->maxSingleJobDataSize = $this->configHelper->getNumberOfElementByPage(); } From 6a203c1bc78baeb7c5f4da29f003b39ffc067c89 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Mon, 25 Aug 2025 13:04:43 +0200 Subject: [PATCH 064/119] MAGE-1100: add new max batch size calculation based on stores --- Model/Queue.php | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/Model/Queue.php b/Model/Queue.php index 3c5dea6f9..7a379f433 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -55,6 +55,12 @@ class Queue /** @var array */ protected $logRecord; + /** @var int */ + protected $maxBatchSize = 0; + + /** @var array */ + protected array $storeMaxBatchSizes; + public function __construct( protected ConfigHelper $configHelper, protected DiagnosticsLogger $logger, @@ -404,15 +410,13 @@ protected function fetchJobs(int $jobsLimit, bool $fetchFullReindexJobs = false, { $jobs = []; - $actualBatchSize = 0; - $maxBatchSize = $this->configHelper->getNumberOfElementByPage(); - + $actualBatchSize = -1; $limit = $jobsLimit; $offset = 0; $fetchFullReindexJobs = $fetchFullReindexJobs ? 1 : 0; - while ($actualBatchSize < $maxBatchSize) { + while ($actualBatchSize < $this->maxBatchSize) { $jobsCollection = $this->jobCollectionFactory->create(); $jobsCollection ->addFieldToFilter('pid', ['null' => true]) @@ -455,18 +459,36 @@ protected function fetchJobs(int $jobsLimit, bool $fetchFullReindexJobs = false, // This will determine if we can continue to loop over the jobs $jobSizes = []; + $this->maxBatchSize = 0; + foreach ($rawJobs as $job) { $jobSize = (int) $job->getDataSize(); $jobSizes[$job->getId()] = $jobSize; $jobs[] = $job; + $this->maxBatchSize += $this->getStoreMaxBatchSize($job->getStoreId()); } + // Final calculation for the loop $actualBatchSize = array_sum($jobSizes); + $this->maxBatchSize = round($this->maxBatchSize / count($jobSizes)); } return $jobs; } + /** + * @param int $storeId + * @return int + */ + protected function getStoreMaxBatchSize(int $storeId): int + { + if (!isset($this->storeMaxBatchSizes[$storeId])) { + $this->storeMaxBatchSizes[$storeId] = $this->configHelper->getNumberOfElementByPage($storeId); + } + + return $this->storeMaxBatchSizes[$storeId]; + } + /** * @param Job[] $unmergedJobs * From 06f46fb115748e2d8083223028f3b980fa225305 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Mon, 25 Aug 2025 19:05:50 +0200 Subject: [PATCH 065/119] MAGE-1100: fix tests --- Model/Queue.php | 7 ++++++- Test/Integration/Indexing/Queue/QueueTest.php | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Model/Queue.php b/Model/Queue.php index 7a379f433..5f0c5f1d5 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -483,7 +483,12 @@ protected function fetchJobs(int $jobsLimit, bool $fetchFullReindexJobs = false, protected function getStoreMaxBatchSize(int $storeId): int { if (!isset($this->storeMaxBatchSizes[$storeId])) { - $this->storeMaxBatchSizes[$storeId] = $this->configHelper->getNumberOfElementByPage($storeId); + try { + $this->storeMaxBatchSizes[$storeId] = $this->configHelper->getNumberOfElementByPage($storeId); + } catch (\Exception $e) { + // In case a job was created before a store deletion + $this->storeMaxBatchSizes[$storeId] = $this->configHelper->getNumberOfElementByPage(); + } } return $this->storeMaxBatchSizes[$storeId]; diff --git a/Test/Integration/Indexing/Queue/QueueTest.php b/Test/Integration/Indexing/Queue/QueueTest.php index 342cba814..00559137e 100644 --- a/Test/Integration/Indexing/Queue/QueueTest.php +++ b/Test/Integration/Indexing/Queue/QueueTest.php @@ -848,7 +848,7 @@ public function testHugeJob() /** @var Job[] $jobs */ $jobs = $this->invokeMethod($this->queue, 'getJobs', ['maxJobs' => 10]); - $this->assertEquals(1, count($jobs)); + $this->assertEquals(2, count($jobs)); $job = reset($jobs); $this->assertEquals(5000, $job->getDataSize()); @@ -862,7 +862,7 @@ public function testHugeJob() $lastJob = end($dbJobs); $this->assertEquals($pid, $firstJob['pid']); - $this->assertNull($lastJob['pid']); + $this->assertEquals($pid, $lastJob['pid']); } /** From 6d6331eea1af5664ab7088e381f4cbef28afc6f8 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Tue, 26 Aug 2025 10:31:37 +0200 Subject: [PATCH 066/119] MAGE-1110: add missing store scope for max batch size --- Service/Category/BatchQueueProcessor.php | 2 +- Service/Category/IndexBuilder.php | 4 ++-- Service/Product/BatchQueueProcessor.php | 2 +- Service/Suggestion/IndexBuilder.php | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Service/Category/BatchQueueProcessor.php b/Service/Category/BatchQueueProcessor.php index bd193a276..f1715db95 100644 --- a/Service/Category/BatchQueueProcessor.php +++ b/Service/Category/BatchQueueProcessor.php @@ -41,7 +41,7 @@ public function processBatch(int $storeId, ?array $entityIds = null): void return; } - $categoriesPerPage = $this->configHelper->getNumberOfElementByPage(); + $categoriesPerPage = $this->configHelper->getNumberOfElementByPage($storeId); if (is_array($entityIds) && count($entityIds) > 0) { $this->processSpecificCategories($entityIds, $categoriesPerPage, $storeId); diff --git a/Service/Category/IndexBuilder.php b/Service/Category/IndexBuilder.php index 17eddee9c..66c8a5bd3 100644 --- a/Service/Category/IndexBuilder.php +++ b/Service/Category/IndexBuilder.php @@ -117,14 +117,14 @@ protected function rebuildEntityIds($storeId, $categoryIds = null): void } if ($size > 0) { - $pages = ceil($size / $this->configHelper->getNumberOfElementByPage()); + $pages = ceil($size / $this->configHelper->getNumberOfElementByPage($storeId)); $page = 1; while ($page <= $pages) { $this->buildIndexPage( $storeId, $collection, $page, - $this->configHelper->getNumberOfElementByPage(), + $this->configHelper->getNumberOfElementByPage($storeId), $categoryIds ); $page++; diff --git a/Service/Product/BatchQueueProcessor.php b/Service/Product/BatchQueueProcessor.php index a41456e62..c26ebeb8a 100644 --- a/Service/Product/BatchQueueProcessor.php +++ b/Service/Product/BatchQueueProcessor.php @@ -51,7 +51,7 @@ public function processBatch(int $storeId, ?array $entityIds = null): void return; } - $productsPerPage = $this->configHelper->getNumberOfElementByPage(); + $productsPerPage = $this->configHelper->getNumberOfElementByPage($storeId); if (!empty($entityIds)) { $this->handleDeltaIndex($entityIds, $storeId, $productsPerPage); diff --git a/Service/Suggestion/IndexBuilder.php b/Service/Suggestion/IndexBuilder.php index 779344683..5a41a6f39 100644 --- a/Service/Suggestion/IndexBuilder.php +++ b/Service/Suggestion/IndexBuilder.php @@ -75,7 +75,7 @@ public function buildIndex(int $storeId, ?array $entityIds, ?array $options): vo $size = $collection->getSize(); if ($size > 0) { - $pages = ceil($size / $this->configHelper->getNumberOfElementByPage()); + $pages = ceil($size / $this->configHelper->getNumberOfElementByPage($storeId)); $collection->clear(); $page = 1; @@ -84,7 +84,7 @@ public function buildIndex(int $storeId, ?array $entityIds, ?array $options): vo $storeId, $collection, $page, - $this->configHelper->getNumberOfElementByPage() + $this->configHelper->getNumberOfElementByPage($storeId) ); $page++; } From 683a7079dc09513a30f0bdf56aecdcf642e5569c Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Tue, 26 Aug 2025 11:14:34 +0200 Subject: [PATCH 067/119] MAGE-1110: some more cleaning --- Model/Queue.php | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/Model/Queue.php b/Model/Queue.php index 5f0c5f1d5..b615bda99 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -36,12 +36,6 @@ class Queue /** @var string */ protected $archiveTable; - /** @var int */ - protected $elementsPerPage; - - /** @var int */ - protected $maxSingleJobDataSize; - /** @var int */ protected $noOfFailedJobs = 0; @@ -73,9 +67,6 @@ public function __construct( $this->logTable = $resourceConnection->getTableName('algoliasearch_queue_log'); $this->archiveTable = $resourceConnection->getTableName('algoliasearch_queue_archive'); $this->db = $objectManager->create(ResourceConnection::class)->getConnection('core_write'); - - $this->elementsPerPage = $this->configHelper->getNumberOfElementByPage(); - $this->maxSingleJobDataSize = $this->configHelper->getNumberOfElementByPage(); } /** @@ -513,7 +504,7 @@ protected function mergeJobs(array $unmergedJobs): array if (count($unmergedJobs) > 0) { $nextJob = array_shift($unmergedJobs); - if ($currentJob->canMerge($nextJob, $this->maxSingleJobDataSize)) { + if ($currentJob->canMerge($nextJob, $this->getStoreMaxBatchSize($currentJob->getStoreId()))) { $currentJob->merge($nextJob); continue; From edbf3a624daf796c057ec74284d587cff5754aa0 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 27 Aug 2025 10:23:03 +0200 Subject: [PATCH 068/119] MAGE-1110: address feedback --- Model/Job.php | 8 ++++---- Model/Queue.php | 26 ++++++++++++++++++-------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/Model/Job.php b/Model/Job.php index f5a007d10..8661dc56f 100644 --- a/Model/Job.php +++ b/Model/Job.php @@ -33,11 +33,11 @@ class Job extends \Magento\Framework\Model\AbstractModel implements JobInterface protected $_eventPrefix = 'algoliasearch_queue_job'; public function __construct( - protected Context $context, - protected Registry $registry, + Context $context, + Registry $registry, protected ObjectManagerInterface $objectManager, - protected ?AbstractResource $resource = null, - protected ?AbstractDb $resourceCollection = null, + ?AbstractResource $resource = null, + ?AbstractDb $resourceCollection = null, array $data = [] ) { parent::__construct($context, $registry, $resource, $resourceCollection, $data); diff --git a/Model/Queue.php b/Model/Queue.php index b615bda99..e9047cbb0 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -49,9 +49,6 @@ class Queue /** @var array */ protected $logRecord; - /** @var int */ - protected $maxBatchSize = 0; - /** @var array */ protected array $storeMaxBatchSizes; @@ -402,12 +399,13 @@ protected function fetchJobs(int $jobsLimit, bool $fetchFullReindexJobs = false, $jobs = []; $actualBatchSize = -1; + $maxBatchSize = 0; $limit = $jobsLimit; $offset = 0; $fetchFullReindexJobs = $fetchFullReindexJobs ? 1 : 0; - while ($actualBatchSize < $this->maxBatchSize) { + while ($actualBatchSize < $maxBatchSize) { $jobsCollection = $this->jobCollectionFactory->create(); $jobsCollection ->addFieldToFilter('pid', ['null' => true]) @@ -450,23 +448,35 @@ protected function fetchJobs(int $jobsLimit, bool $fetchFullReindexJobs = false, // This will determine if we can continue to loop over the jobs $jobSizes = []; - $this->maxBatchSize = 0; - foreach ($rawJobs as $job) { $jobSize = (int) $job->getDataSize(); $jobSizes[$job->getId()] = $jobSize; $jobs[] = $job; - $this->maxBatchSize += $this->getStoreMaxBatchSize($job->getStoreId()); } // Final calculation for the loop $actualBatchSize = array_sum($jobSizes); - $this->maxBatchSize = round($this->maxBatchSize / count($jobSizes)); + $maxBatchSize = $this->calculateMaxBatchSize($jobs); } return $jobs; } + /** + * @param Job[] $jobs + * @return int + */ + protected function calculateMaxBatchSize(array $jobs): int + { + $maxBatchSize = 0; + + foreach ($jobs as $job) { + $maxBatchSize += $this->getStoreMaxBatchSize($job->getStoreId()); + } + + return round($maxBatchSize / count($jobs)); + } + /** * @param int $storeId * @return int From 4131a3e18fdb8701544559962c2879328cac5b48 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 28 Aug 2025 12:45:51 +0200 Subject: [PATCH 069/119] MAGE-1391: update max batch size config --- Console/Command/BatchingOptimizeCommand.php | 2 +- Helper/ConfigHelper.php | 22 +++---- Helper/Configuration/NoticeHelper.php | 2 +- .../Data/MigrateBatchSizeConfigPatch.php | 63 +++++++++++++++++++ Setup/Patch/Schema/ConfigPatch.php | 2 +- etc/adminhtml/system.xml | 18 +++--- etc/config.xml | 2 +- 7 files changed, 87 insertions(+), 24 deletions(-) create mode 100644 Setup/Patch/Data/MigrateBatchSizeConfigPatch.php diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index c257357ab..9e32ffd4d 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -292,7 +292,7 @@ protected function scanProductRecordsForStore(int $storeId): void $this->output->writeln('Important: Those numbers are estimates only. Indexing activity should be monitored after making changes to ensure batches are not exceeding the recommended size of 10 MB.'); $this->output->writeln(' ============ '); $this->output->writeln( - 'This will override your "Maximum number of records processed per indexing job" configuration to ' . $recommendedBatchCount . ' for store "' . $storeName . '".'); + 'This will override your "Maximum number of records sent per indexing request" configuration to ' . $recommendedBatchCount . ' for store "' . $storeName . '".'); $this->output->writeln(' '); if ($this->confirmOperation()) { diff --git a/Helper/ConfigHelper.php b/Helper/ConfigHelper.php index d7783d059..19d412b61 100755 --- a/Helper/ConfigHelper.php +++ b/Helper/ConfigHelper.php @@ -126,6 +126,7 @@ class ConfigHelper public const BACKEND_RENDERING_ALLOWED_USER_AGENTS = 'algoliasearch_advanced/advanced/backend_rendering_allowed_user_agents'; public const NON_CASTABLE_ATTRIBUTES = 'algoliasearch_advanced/advanced/non_castable_attributes'; + public const NUMBER_OF_ELEMENT_BY_PAGE = 'algoliasearch_advanced/advanced/number_of_element_by_page'; public const MAX_RECORD_SIZE_LIMIT = 'algoliasearch_advanced/advanced/max_record_size_limit'; public const ANALYTICS_REGION = 'algoliasearch_advanced/advanced/analytics_region'; public const CONNECTION_TIMEOUT = 'algoliasearch_advanced/advanced/connection_timeout'; @@ -137,7 +138,6 @@ class ConfigHelper // Indexing Queue advanced settings public const ENHANCED_QUEUE_ARCHIVE = 'algoliasearch_advanced/queue/enhanced_archive'; - public const NUMBER_OF_ELEMENT_BY_PAGE = 'algoliasearch_advanced/queue/number_of_element_by_page'; public const ARCHIVE_LOG_CLEAR_LIMIT = 'algoliasearch_advanced/queue/archive_clear_limit'; // --- Extra index settings --- // @@ -1290,6 +1290,15 @@ public function getNonCastableAttributes($storeId = null) return $nonCastableAttributes; } + /** + * @param $storeId + * @return int + */ + public function getNumberOfElementByPage($storeId = null): int + { + return (int) $this->configInterface->getValue(self::NUMBER_OF_ELEMENT_BY_PAGE, ScopeInterface::SCOPE_STORE, $storeId); + } + /** * @param $storeId * @return int @@ -1366,16 +1375,7 @@ public function isProfilerEnabled(?int $storeId = null): bool return $this->configInterface->isSetFlag(self::PROFILER_ENABLED, ScopeInterface::SCOPE_STORE, $storeId); } - // Indexing Queue advanced settings - /** - * @param $storeId - * @return int - */ - public function getNumberOfElementByPage($storeId = null): int - { - return (int) $this->configInterface->getValue(self::NUMBER_OF_ELEMENT_BY_PAGE, ScopeInterface::SCOPE_STORE, $storeId); - } - + // Indexing Queue advanced settingg /** * @param $storeId * @return int diff --git a/Helper/Configuration/NoticeHelper.php b/Helper/Configuration/NoticeHelper.php index bebba33c3..918795303 100644 --- a/Helper/Configuration/NoticeHelper.php +++ b/Helper/Configuration/NoticeHelper.php @@ -100,7 +100,7 @@ protected function getQueueNotice() in approx. ' . $eta . '. You may want to clear the queue or configure indexing queue.

- Depending on your configuration set on "Advanced > Maximum number of records processed per indexing job" and if the jobs can be merged into batches, you can expect higher performances. + Depending on your configuration set on "Advanced > Maximum number of records sent per indexing request" and if the jobs can be merged into batches, you can expect higher performances.

Find out more about Indexing Queue in documentation.'; } diff --git a/Setup/Patch/Data/MigrateBatchSizeConfigPatch.php b/Setup/Patch/Data/MigrateBatchSizeConfigPatch.php new file mode 100644 index 000000000..fe8097a70 --- /dev/null +++ b/Setup/Patch/Data/MigrateBatchSizeConfigPatch.php @@ -0,0 +1,63 @@ +moduleDataSetup->getConnection()->startSetup(); + + $this->moveIndexingSettings(); + + $this->moduleDataSetup->getConnection()->endSetup(); + + return $this; + } + + /** + * Migrate old Indexing configurations + * @return void + */ + protected function moveIndexingSettings(): void + { + $movedConfig = [ + 'algoliasearch_advanced/queue/number_of_element_by_page' => ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, + ]; + + $connection = $this->moduleDataSetup->getConnection(); + foreach ($movedConfig as $from => $to) { + $configDataTable = $this->moduleDataSetup->getTable('core_config_data'); + $whereConfigPath = $connection->quoteInto('path = ?', $from); + $connection->update($configDataTable, ['path' => $to], $whereConfigPath); + } + } + + /** + * @inheritDoc + */ + public static function getDependencies(): array + { + return []; + } + + /** + * @inheritDoc + */ + public function getAliases(): array + { + return []; + } +} diff --git a/Setup/Patch/Schema/ConfigPatch.php b/Setup/Patch/Schema/ConfigPatch.php index d3918504b..299713b17 100644 --- a/Setup/Patch/Schema/ConfigPatch.php +++ b/Setup/Patch/Schema/ConfigPatch.php @@ -81,7 +81,7 @@ class ConfigPatch implements SchemaPatchInterface 'algoliasearch_synonyms/synonyms_group/enable_synonyms' => '0', - 'algoliasearch_advanced/queue/number_of_element_by_page' => '300', + 'algoliasearch_advanced/advanced/number_of_element_by_page' => '1000', 'algoliasearch_advanced/advanced/remove_words_if_no_result' => 'allOptional', 'algoliasearch_advanced/advanced/partial_update' => '0', 'algoliasearch_advanced/advanced/customer_groups_enable' => '0', diff --git a/etc/adminhtml/system.xml b/etc/adminhtml/system.xml index 57c3c4c07..c231c489a 100755 --- a/etc/adminhtml/system.xml +++ b/etc/adminhtml/system.xml @@ -1518,6 +1518,15 @@ ]]>
+ + validate-digits + + + + + @@ -1578,15 +1587,6 @@ Indexing Queue is enabled under the "Indexing Queue / Cron" tab. ]]> - - validate-digits - - - - - diff --git a/etc/config.xml b/etc/config.xml index 6c633c3f8..a454e0232 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -121,6 +121,7 @@ + 1000 10000 us 2 @@ -131,7 +132,6 @@ 0 - 300 30 0 From faab56f0e989ff379bf612f865d776d4e615e70c Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Fri, 29 Aug 2025 11:19:41 +0200 Subject: [PATCH 070/119] MAGE-1407: fixed indexing queue notices display in phtml --- view/adminhtml/templates/queue/status.phtml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/view/adminhtml/templates/queue/status.phtml b/view/adminhtml/templates/queue/status.phtml index 4f260146c..84bf2a9ad 100644 --- a/view/adminhtml/templates/queue/status.phtml +++ b/view/adminhtml/templates/queue/status.phtml @@ -1,6 +1,9 @@
@@ -30,18 +33,18 @@ isQueueActive()): ?>
- escapeHtml(__('Status of the queue : %1.', $block->getQueueRunnerStatus())); ?> -

escapeHtml(__('Last Update : %1.', $block->getLastQueueUpdate())); ?>

+ escapeHtml(__('Status of the queue : %1.', $block->getQueueRunnerStatus()), $allowedTags); ?> +

escapeHtml(__('Last Update : %1.', $block->getLastQueueUpdate()), $allowedTags); ?>

getNotices() ?> -

escapeHtml($notice) ?>

+

escapeHtml($notice, $allowedTags) ?>

-

escapeHtml(__('See how well your indexing queue is performing by viewing your %1.', 'indexing queue run logs')) ?>

-

escapeHtml(__('Access the %1.', 'Queue Archive')) ?>

+

escapeHtml(__('See how well your indexing queue is performing by viewing your %1.', 'indexing queue run logs'), $allowedTags) ?>

+

escapeHtml(__('Access the %1.', 'Queue Archive'), $allowedTags) ?>

From 1c17d805f700f99b7288ae9bb2abbbbc429fade7 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Fri, 29 Aug 2025 16:56:12 +0200 Subject: [PATCH 071/119] MAGE-1803: restore value to 300 for tests --- Setup/Patch/Schema/ConfigPatch.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Setup/Patch/Schema/ConfigPatch.php b/Setup/Patch/Schema/ConfigPatch.php index 299713b17..8884a2f3f 100644 --- a/Setup/Patch/Schema/ConfigPatch.php +++ b/Setup/Patch/Schema/ConfigPatch.php @@ -81,7 +81,7 @@ class ConfigPatch implements SchemaPatchInterface 'algoliasearch_synonyms/synonyms_group/enable_synonyms' => '0', - 'algoliasearch_advanced/advanced/number_of_element_by_page' => '1000', + 'algoliasearch_advanced/advanced/number_of_element_by_page' => '300', 'algoliasearch_advanced/advanced/remove_words_if_no_result' => 'allOptional', 'algoliasearch_advanced/advanced/partial_update' => '0', 'algoliasearch_advanced/advanced/customer_groups_enable' => '0', From 15f4b378ffff20e10405122fa88bfa394d6d3f9b Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Tue, 26 Aug 2025 09:07:48 -0400 Subject: [PATCH 072/119] MAGE-1304 Add eager cleanup to no clone collection handling --- Service/Product/IndexBuilder.php | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Service/Product/IndexBuilder.php b/Service/Product/IndexBuilder.php index a4950b3ba..790eabf53 100644 --- a/Service/Product/IndexBuilder.php +++ b/Service/Product/IndexBuilder.php @@ -96,14 +96,19 @@ public function buildIndex(int $storeId, ?array $entityIds, ?array $options): vo $onlyVisible = !$this->configHelper->includeNonVisibleProductsInIndex($storeId); $collection = $this->productHelper->getProductCollectionQuery($storeId, $entityIds, $onlyVisible); - $this->buildIndexPage( - $storeId, - $collection, - $options['page'] ?? 1, - $options['pageSize'] ?? $this->configHelper->getNumberOfElementByPage($storeId), - $entityIds, - $options['useTmpIndex'] ?? false - ); + try { + $this->buildIndexPage( + $storeId, + $collection, + $options['page'] ?? 1, + $options['pageSize'] ?? $this->configHelper->getNumberOfElementByPage($storeId), + $entityIds, + $options['useTmpIndex'] ?? false + ); + } finally { + $collection->walk('clearInstance'); + $collection->clear(); + } $this->stopEmulation(); } From 6b1bc30c68427f51cb63fbcefe22cc3b878d218a Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Tue, 26 Aug 2025 09:37:47 -0400 Subject: [PATCH 073/119] MAGE-1304 Adjust batching optimizer for non interactive mode --- Console/Command/BatchingOptimizeCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index 9e32ffd4d..544bd23be 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -295,7 +295,7 @@ protected function scanProductRecordsForStore(int $storeId): void 'This will override your "Maximum number of records sent per indexing request" configuration to ' . $recommendedBatchCount . ' for store "' . $storeName . '".'); $this->output->writeln(' '); - if ($this->confirmOperation()) { + if ($this->confirmOperation('Applying optimized batching configuration', 'Batching optimization cancelled - settings were NOT changed', true)) { $this->configWriter->save( ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, $recommendedBatchCount, From 876e0b8bde61b887a24ceda93c0f50adeff114c9 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Tue, 2 Sep 2025 23:37:34 -0400 Subject: [PATCH 074/119] MAGE-1304 Revert eager cleanup due to regression on large catalogs --- Service/Product/IndexBuilder.php | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/Service/Product/IndexBuilder.php b/Service/Product/IndexBuilder.php index 790eabf53..a4950b3ba 100644 --- a/Service/Product/IndexBuilder.php +++ b/Service/Product/IndexBuilder.php @@ -96,19 +96,14 @@ public function buildIndex(int $storeId, ?array $entityIds, ?array $options): vo $onlyVisible = !$this->configHelper->includeNonVisibleProductsInIndex($storeId); $collection = $this->productHelper->getProductCollectionQuery($storeId, $entityIds, $onlyVisible); - try { - $this->buildIndexPage( - $storeId, - $collection, - $options['page'] ?? 1, - $options['pageSize'] ?? $this->configHelper->getNumberOfElementByPage($storeId), - $entityIds, - $options['useTmpIndex'] ?? false - ); - } finally { - $collection->walk('clearInstance'); - $collection->clear(); - } + $this->buildIndexPage( + $storeId, + $collection, + $options['page'] ?? 1, + $options['pageSize'] ?? $this->configHelper->getNumberOfElementByPage($storeId), + $entityIds, + $options['useTmpIndex'] ?? false + ); $this->stopEmulation(); } From 72f7419c38b67d194162f0240c3058c1ca03f08e Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 3 Sep 2025 13:39:57 +0200 Subject: [PATCH 075/119] MAGE-1354: Update version info and change log --- CHANGELOG.md | 19 ++++++++++++++----- README.md | 1 + composer.json | 2 +- etc/module.xml | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa5da7555..7b9c095c8 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,23 @@ # CHANGE LOG -## 3.17.0 +## 3.17.0-beta.1 ### Features - -(This is a WIP until release) - - Added an Algolia indexing cache for storing metadata to prevent extra queries. Large collections that run periodic full indexes can benefit from this cache. - Price indexing is now optional. If the price attribute is not present in the product attributes, sorts, facets or custom ranking then that information will not be indexed in the product record. -- Creating a temporary index while performing a full index is now optional and can be enabled or disabled in the Magento admin. +- Creating a temporary index while performing a full index is now optional and can be enabled or disabled in the Magento admin. +- Added Batching optimizer feature which performs a catalog analysis and provides recommendation regarding optimal batching size for indexing. +- Added Clear queue CLI command which handles indexing queue clearing from the console. + +### Updates +- Fixed Indexing Queue merging mechanism, it should now have way better performances with delta indexing (updates) jobs. +- Updated CLI command organization, queue related commands now have a different prefix compared to the indexing ones. +- Updated default "Maximum number of records sent per indexing request" to 1000 (previously 300). +- Updated `ConfigHelper` class, it now has more methods deprecated and ported to separate helper classes. +- Updated Unit and Integration tests. + +### Bug fixes +- Fixed indexing queue templates escaping. ## 3.16.0 diff --git a/README.md b/README.md index d839f286d..558854773 100755 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ Algolia Search & Discovery extension for Magento 2 ![Latest version](https://img.shields.io/badge/latest-3.16.0-green) ![Magento 2](https://img.shields.io/badge/Magento-2.4.7+-orange) +![Beta version](https://img.shields.io/badge/beta-3.17.0--beta.1-purple) ![PHP](https://img.shields.io/badge/PHP-8.1%2C8.2%2C8.3%2C8.4-blue) diff --git a/composer.json b/composer.json index fa7a32f6a..7d1d325df 100755 --- a/composer.json +++ b/composer.json @@ -3,7 +3,7 @@ "description": "Algolia Search & Discovery extension for Magento 2", "type": "magento2-module", "license": ["MIT"], - "version": "3.17.0-dev", + "version": "3.17.0-beta.1", "require": { "php": "~8.2|~8.3|~8.4", "magento/framework": "~103.0", diff --git a/etc/module.xml b/etc/module.xml index 93947a403..179ecc164 100755 --- a/etc/module.xml +++ b/etc/module.xml @@ -1,6 +1,6 @@ - + From 216aa7f6a2147a2151a9cd5ad69762e12d902998 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 3 Sep 2025 15:48:07 +0200 Subject: [PATCH 076/119] MAGE-1354: address Codacy complaint --- Model/Cache/Product/IndexCollectionSize.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Model/Cache/Product/IndexCollectionSize.php b/Model/Cache/Product/IndexCollectionSize.php index b4661af67..138cadd11 100644 --- a/Model/Cache/Product/IndexCollectionSize.php +++ b/Model/Cache/Product/IndexCollectionSize.php @@ -57,7 +57,7 @@ protected function getCacheKey(int $storeId): string public function clear(?int $storeId = null): void { - if (is_null($storeId)) { + if ($storeId === null) { $this->typeList->invalidate(Indexer::TYPE_IDENTIFIER); $this->typeList->cleanType(Indexer::TYPE_IDENTIFIER); } From e8b575f65156626341f5a74118a25f65df529154 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 3 Sep 2025 16:46:16 -0400 Subject: [PATCH 077/119] MAGE-1411 Eliminate extra hydration of products to conserve memory use --- Console/Command/BatchingOptimizeCommand.php | 31 +++++++++++++++------ 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index 544bd23be..b336b9bf5 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -248,6 +248,7 @@ protected function scanProductRecordsForStore(int $storeId): void } if (!isset($this->storeCounts[$storeId])) { + $this->output->writeln('Scanning products for store ' . $storeName . '...'); $this->setStoreCounts($storeId); } @@ -319,8 +320,8 @@ protected function setStoreCounts(int $storeId): void $complexProducts = $this->getProductsCollectionForStore($storeId, self::PRODUCTS_COMPLEX_TYPES); $this->storeCounts[$storeId] = [ - 'simple' => $simpleProducts->count(), - 'complex' => $complexProducts->count() + 'simple' => $this->getRawCount($simpleProducts), + 'complex' => $this->getRawCount($complexProducts) ]; $this->storeCounts[$storeId]['total'] = @@ -372,6 +373,24 @@ protected function getProductsCollectionForStore(int $storeId, array $productTyp return $collection; } + /** + * Relying on Collection count method will unnecesarily hydrate the collection and consume memory + * This method will return the count of the collection without hydrating it + * + * @param Collection $collection + * @return int + */ + protected function getRawCount(Collection $collection): int + { + $sql = $collection->getSelect()->__toString(); + + $connection = $collection->getConnection(); + + $rows = $connection->fetchAll($sql); + + return count($rows); + } + /** * @param Collection $products * @param int $sampleSize @@ -384,13 +403,10 @@ protected function getProductsCollectionForStore(int $storeId, array $productTyp protected function getProductsSizes(Collection $products, int $sampleSize): array { $stats = []; - $limit = 0; - foreach ($products as $product) { - if ($limit >= $sampleSize) { - break; - } + $products->setPageSize($sampleSize)->setCurPage(1); + foreach ($products as $product) { $serializedRecord = json_encode($this->recordBuilder->buildRecord($product)); if (function_exists('mb_strlen')) { @@ -400,7 +416,6 @@ protected function getProductsSizes(Collection $products, int $sampleSize): arra } $stats[$product->getSku()] = $size; - $limit++; } return $stats; From b1b4047bdfcc081d4943c6a028c381ff4fa55ab9 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 4 Sep 2025 10:20:16 +0200 Subject: [PATCH 078/119] MAGE-1411: cover edge case where sample size is zero --- Console/Command/BatchingOptimizeCommand.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Console/Command/BatchingOptimizeCommand.php b/Console/Command/BatchingOptimizeCommand.php index b336b9bf5..26e0c3266 100644 --- a/Console/Command/BatchingOptimizeCommand.php +++ b/Console/Command/BatchingOptimizeCommand.php @@ -374,7 +374,7 @@ protected function getProductsCollectionForStore(int $storeId, array $productTyp } /** - * Relying on Collection count method will unnecesarily hydrate the collection and consume memory + * Relying on Collection count method will unnecessarily hydrate the collection and consume memory * This method will return the count of the collection without hydrating it * * @param Collection $collection @@ -402,6 +402,10 @@ protected function getRawCount(Collection $collection): int */ protected function getProductsSizes(Collection $products, int $sampleSize): array { + if ($sampleSize === 0) { + return []; + } + $stats = []; $products->setPageSize($sampleSize)->setCurPage(1); From 993cb3fc7cdb28ef88fe48b85664b33bf0b9299d Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Thu, 11 Sep 2025 15:25:43 -0400 Subject: [PATCH 079/119] MAGE-1413 Fix implicit nullable types for PHP 8.4 --- Model/Queue.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Model/Queue.php b/Model/Queue.php index e9047cbb0..9a1a08d7e 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -124,7 +124,7 @@ public function getAverageProcessingTime(): ?float * * @throws Exception */ - public function runCron(int $nbJobs = null, bool $force = false): void + public function runCron(?int $nbJobs = null, bool $force = false): void { if (!$this->configHelper->isQueueActive() && $force === false) { return; @@ -394,7 +394,7 @@ protected function getJobs(int $maxJobs): array * * @return Job[] */ - protected function fetchJobs(int $jobsLimit, bool $fetchFullReindexJobs = false, int $lastJobId = null): array + protected function fetchJobs(int $jobsLimit, bool $fetchFullReindexJobs = false, ?int $lastJobId = null): array { $jobs = []; From d7fd24690b05da3a359eb5b39b0939529c4c447f Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Thu, 11 Sep 2025 15:38:42 -0400 Subject: [PATCH 080/119] MAGE-1413 Bump version --- composer.json | 2 +- etc/module.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 7d1d325df..506307b03 100755 --- a/composer.json +++ b/composer.json @@ -3,7 +3,7 @@ "description": "Algolia Search & Discovery extension for Magento 2", "type": "magento2-module", "license": ["MIT"], - "version": "3.17.0-beta.1", + "version": "3.17.0-beta.2", "require": { "php": "~8.2|~8.3|~8.4", "magento/framework": "~103.0", diff --git a/etc/module.xml b/etc/module.xml index 179ecc164..b22b86847 100755 --- a/etc/module.xml +++ b/etc/module.xml @@ -1,6 +1,6 @@ - + From e6e75b2440e9bfb9cf78f0b6a766b29202c4edc2 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Thu, 11 Sep 2025 15:42:37 -0400 Subject: [PATCH 081/119] MAGE-1413 Add changelog note --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b9c095c8..429cf4f6f 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # CHANGE LOG +## 3.17.0-beta.2 + +### Bug fixes + +- Fixed 3.17 setup:upgrade on PHP 8.4 + ## 3.17.0-beta.1 ### Features From 96fb50be2681130190ffafb45c31ee760b01dee8 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 18 Sep 2025 15:34:23 +0200 Subject: [PATCH 082/119] MAGE-1404: add checks on configuration migration processed on data patches --- .../Data/MigrateBatchSizeConfigPatch.php | 14 ++++----- .../Patch/Data/MigrateIndexingConfigPatch.php | 10 +++---- .../Data/MigrateInstantSearchConfigPatch.php | 11 ++++--- Setup/Patch/DataMigrationTrait.php | 29 +++++++++++++++++++ 4 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 Setup/Patch/DataMigrationTrait.php diff --git a/Setup/Patch/Data/MigrateBatchSizeConfigPatch.php b/Setup/Patch/Data/MigrateBatchSizeConfigPatch.php index fe8097a70..3e28b2eaf 100644 --- a/Setup/Patch/Data/MigrateBatchSizeConfigPatch.php +++ b/Setup/Patch/Data/MigrateBatchSizeConfigPatch.php @@ -3,12 +3,15 @@ namespace Algolia\AlgoliaSearch\Setup\Patch\Data; use Algolia\AlgoliaSearch\Helper\ConfigHelper; +use Algolia\AlgoliaSearch\Setup\Patch\DataMigrationTrait; use Magento\Framework\Setup\ModuleDataSetupInterface; use Magento\Framework\Setup\Patch\DataPatchInterface; use Magento\Framework\Setup\Patch\PatchInterface; class MigrateBatchSizeConfigPatch implements DataPatchInterface { + use DataMigrationTrait; + public function __construct( protected ModuleDataSetupInterface $moduleDataSetup, ) {} @@ -35,14 +38,9 @@ protected function moveIndexingSettings(): void { $movedConfig = [ 'algoliasearch_advanced/queue/number_of_element_by_page' => ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, - ]; - - $connection = $this->moduleDataSetup->getConnection(); - foreach ($movedConfig as $from => $to) { - $configDataTable = $this->moduleDataSetup->getTable('core_config_data'); - $whereConfigPath = $connection->quoteInto('path = ?', $from); - $connection->update($configDataTable, ['path' => $to], $whereConfigPath); - } + ]; + + $this->migrateConfig($movedConfig); } /** diff --git a/Setup/Patch/Data/MigrateIndexingConfigPatch.php b/Setup/Patch/Data/MigrateIndexingConfigPatch.php index 172e749d0..55bab9154 100644 --- a/Setup/Patch/Data/MigrateIndexingConfigPatch.php +++ b/Setup/Patch/Data/MigrateIndexingConfigPatch.php @@ -3,12 +3,15 @@ namespace Algolia\AlgoliaSearch\Setup\Patch\Data; use Algolia\AlgoliaSearch\Helper\ConfigHelper; +use Algolia\AlgoliaSearch\Setup\Patch\DataMigrationTrait; use Magento\Framework\Setup\ModuleDataSetupInterface; use Magento\Framework\Setup\Patch\DataPatchInterface; use Magento\Framework\Setup\Patch\PatchInterface; class MigrateIndexingConfigPatch implements DataPatchInterface { + use DataMigrationTrait; + public function __construct( protected ModuleDataSetupInterface $moduleDataSetup, ) {} @@ -39,12 +42,7 @@ protected function moveIndexingSettings(): void 'algoliasearch_credentials/credentials/enable_pages_index' => ConfigHelper::ENABLE_PAGES_INDEX, ]; - $connection = $this->moduleDataSetup->getConnection(); - foreach ($movedConfig as $from => $to) { - $configDataTable = $this->moduleDataSetup->getTable('core_config_data'); - $whereConfigPath = $connection->quoteInto('path = ?', $from); - $connection->update($configDataTable, ['path' => $to], $whereConfigPath); - } + $this->migrateConfig($movedConfig); } /** diff --git a/Setup/Patch/Data/MigrateInstantSearchConfigPatch.php b/Setup/Patch/Data/MigrateInstantSearchConfigPatch.php index c43c87832..737a9f95d 100644 --- a/Setup/Patch/Data/MigrateInstantSearchConfigPatch.php +++ b/Setup/Patch/Data/MigrateInstantSearchConfigPatch.php @@ -2,12 +2,15 @@ namespace Algolia\AlgoliaSearch\Setup\Patch\Data; +use Algolia\AlgoliaSearch\Setup\Patch\DataMigrationTrait; use Magento\Framework\Setup\ModuleDataSetupInterface; use Magento\Framework\Setup\Patch\DataPatchInterface; use Magento\Framework\Setup\Patch\PatchInterface; class MigrateInstantSearchConfigPatch implements DataPatchInterface { + use DataMigrationTrait; + public function __construct( protected ModuleDataSetupInterface $moduleDataSetup, ) {} @@ -44,12 +47,8 @@ protected function moveInstantSearchSettings(): void 'algoliasearch_instant/instant/infinite_scroll_enable' => 'algoliasearch_instant/instant_options/infinite_scroll_enable', 'algoliasearch_instant/instant/hide_pagination' => 'algoliasearch_instant/instant_options/hide_pagination' ]; - $connection = $this->moduleDataSetup->getConnection(); - foreach ($movedConfig as $from => $to) { - $configDataTable = $this->moduleDataSetup->getTable('core_config_data'); - $whereConfigPath = $connection->quoteInto('path = ?', $from); - $connection->update($configDataTable, ['path' => $to], $whereConfigPath); - } + + $this->migrateConfig($movedConfig); } /** diff --git a/Setup/Patch/DataMigrationTrait.php b/Setup/Patch/DataMigrationTrait.php new file mode 100644 index 000000000..1c85e204f --- /dev/null +++ b/Setup/Patch/DataMigrationTrait.php @@ -0,0 +1,29 @@ +moduleDataSetup->getConnection(); + foreach ($configurations as $from => $to) { + $configDataTable = $this->moduleDataSetup->getTable('core_config_data'); + $whereConfigPathFrom = $connection->quoteInto('path = ?', $from); + $whereConfigPathTo = $connection->quoteInto('path = ?', $to); + + $select = $connection->select() + ->from($configDataTable) + ->where($whereConfigPathTo); + $existingValues = $connection->fetchAll($select); + + if (count($existingValues) === 0) { + $connection->update($configDataTable, ['path' => $to], $whereConfigPathFrom); + } + } + } +} From 4e5a651efcb3924d162ba05c5fbc79681fd2b777 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 18 Sep 2025 15:48:48 +0200 Subject: [PATCH 083/119] MAGE-1404: codacy warning --- Setup/Patch/DataMigrationTrait.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Setup/Patch/DataMigrationTrait.php b/Setup/Patch/DataMigrationTrait.php index 1c85e204f..3832f1612 100644 --- a/Setup/Patch/DataMigrationTrait.php +++ b/Setup/Patch/DataMigrationTrait.php @@ -14,11 +14,10 @@ public function migrateConfig(array $configurations): void foreach ($configurations as $from => $to) { $configDataTable = $this->moduleDataSetup->getTable('core_config_data'); $whereConfigPathFrom = $connection->quoteInto('path = ?', $from); - $whereConfigPathTo = $connection->quoteInto('path = ?', $to); $select = $connection->select() ->from($configDataTable) - ->where($whereConfigPathTo); + ->where('path = ?', $to); $existingValues = $connection->fetchAll($select); if (count($existingValues) === 0) { From 98ff621dc266aca06a690a66528320c56d17b661 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 18 Sep 2025 17:46:10 +0200 Subject: [PATCH 084/119] MAGE-1404: refactor config.xml and create DefaultConfigProvider for integration tests --- .../Config/DefaultConfigProvider.php | 269 ++++++++++++++++++ Test/Integration/TestCase.php | 8 +- etc/config.xml | 111 ++++++-- 3 files changed, 364 insertions(+), 24 deletions(-) create mode 100644 Test/Integration/Config/DefaultConfigProvider.php diff --git a/Test/Integration/Config/DefaultConfigProvider.php b/Test/Integration/Config/DefaultConfigProvider.php new file mode 100644 index 000000000..6b6c1316b --- /dev/null +++ b/Test/Integration/Config/DefaultConfigProvider.php @@ -0,0 +1,269 @@ + '1', + 'algoliasearch_credentials/credentials/enable_frontend' => '1', + 'algoliasearch_credentials/credentials/application_id' => '', + 'algoliasearch_credentials/credentials/search_only_api_key' => '', + 'algoliasearch_credentials/credentials/api_key' => '', + 'algoliasearch_credentials/credentials/debug' => '0', + 'algoliasearch_credentials/credentials/index_prefix' => 'magento2_', + + 'algoliasearch_autocomplete/autocomplete/is_popup_enabled' => '1', + 'algoliasearch_autocomplete/autocomplete/nb_of_products_suggestions' => '6', + 'algoliasearch_autocomplete/autocomplete/nb_of_categories_suggestions' => '2', + 'algoliasearch_autocomplete/autocomplete/nb_of_queries_suggestions' => '0', + 'algoliasearch_autocomplete/autocomplete/min_popularity' => '1000', + 'algoliasearch_autocomplete/autocomplete/min_number_of_results' => '2', + 'algoliasearch_autocomplete/autocomplete/render_template_directives' => '1', + 'algoliasearch_autocomplete/autocomplete/debug' => '0', + + 'algoliasearch_instant/instant/is_instant_enabled' => '0', + 'algoliasearch_instant/instant/instant_selector' => '.columns', + 'algoliasearch_instant/instant/number_product_results' => '9', + 'algoliasearch_instant/instant/max_values_per_facet' => '10', + 'algoliasearch_instant/instant/show_suggestions_on_no_result_page' => '1', + 'algoliasearch_instant/instant/add_to_cart_enable' => '1', + 'algoliasearch_instant/instant/infinite_scroll_enable' => '0', + + 'algoliasearch_products/products/use_adaptive_image' => '0', + + 'algoliasearch_categories/categories/show_cats_not_included_in_navigation' => '0', + 'algoliasearch_categories/categories/index_empty_categories' => '0', + + 'algoliasearch_images/image/width' => '265', + 'algoliasearch_images/image/height' => '265', + 'algoliasearch_images/image/type' => 'image', + + 'algoliasearch_queue/queue/active' => '0', + 'algoliasearch_queue/queue/number_of_job_to_run' => '5', + 'algoliasearch_queue/queue/number_of_retries' => '3', + 'algoliasearch_queue/queue/use_tmp_index' => '1', + + 'algoliasearch_cc_analytics/cc_analytics_group/enable' => '0', + 'algoliasearch_cc_analytics/cc_analytics_group/is_selector' => '.ais-Hits-item a.result, .ais-InfiniteHits-item a.result', + 'algoliasearch_cc_analytics/cc_analytics_group/enable_conversion_analytics' => 'disabled', + 'algoliasearch_cc_analytics/cc_analytics_group/add_to_cart_selector' => '.action.primary.tocart', + + 'algoliasearch_analytics/analytics_group/enable' => '0', + 'algoliasearch_analytics/analytics_group/delay' => '3000', + 'algoliasearch_analytics/analytics_group/trigger_on_ui_interaction' => '1', + 'algoliasearch_analytics/analytics_group/push_initial_search' => '0', + + 'algoliasearch_synonyms/synonyms_group/enable_synonyms' => '0', + + 'algoliasearch_advanced/advanced/number_of_element_by_page' => '300', + 'algoliasearch_advanced/advanced/remove_words_if_no_result' => 'allOptional', + 'algoliasearch_advanced/advanced/partial_update' => '0', + 'algoliasearch_advanced/advanced/customer_groups_enable' => '0', + 'algoliasearch_advanced/advanced/remove_branding' => '0', + 'algoliasearch_autocomplete/autocomplete/autocomplete_selector' => '.algolia-search-input', + 'algoliasearch_advanced/advanced/index_product_on_category_products_update' => '1', + 'algoliasearch_advanced/advanced/prevent_backend_rendering' => '0', + 'algoliasearch_advanced/advanced/prevent_backend_rendering_display_mode' => 'all', + 'algoliasearch_advanced/advanced/backend_rendering_allowed_user_agents' => "Googlebot\nBingbot", + 'algoliasearch_advanced/queue/archive_clear_limit' => '30', + ]; + + /** + * @var string[][][] + */ + protected $defaultArrayConfigData = [ + 'algoliasearch_autocomplete/autocomplete/sections' => [ + [ + 'name' => 'pages', + 'label' => 'Pages', + 'hitsPerPage' => '2', + ], + ], + 'algoliasearch_autocomplete/autocomplete/excluded_pages' => [ + [ + 'attribute' => 'no-route', + ], + ], + + 'algoliasearch_instant/instant_facets/facets' => [ + [ + 'attribute' => 'price', + 'type' => 'slider', + 'label' => 'Price', + 'searchable' => '2', + 'create_rule' => '2', + ], + [ + 'attribute' => 'categories', + 'type' => 'conjunctive', + 'label' => 'Categories', + 'searchable' => '2', + 'create_rule' => '2', + ], + [ + 'attribute' => 'color', + 'type' => 'disjunctive', + 'label' => 'Colors', + 'searchable' => '1', + 'create_rule' => '2', + ], + ], + 'algoliasearch_instant/instant_sorts/sorts' => [ + [ + 'attribute' => 'price', + 'sort' => 'asc', + 'sortLabel' => 'Lowest price', + ], + [ + 'attribute' => 'price', + 'sort' => 'desc', + 'sortLabel' => 'Highest price', + ], + [ + 'attribute' => 'created_at', + 'sort' => 'desc', + 'sortLabel' => 'Newest first', + ], + ], + + 'algoliasearch_products/products/product_additional_attributes' => [ + [ + 'attribute' => 'name', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'sku', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'manufacturer', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'categories', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'color', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'price', + 'searchable' => '2', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'rating_summary', + 'searchable' => '2', + 'order' => 'unordered', + 'retrievable' => '1', + ], + ], + 'algoliasearch_products/products/custom_ranking_product_attributes' => [ + [ + 'attribute' => 'in_stock', + 'order' => 'desc', + ], + [ + 'attribute' => 'ordered_qty', + 'order' => 'desc', + ], + [ + 'attribute' => 'created_at', + 'order' => 'desc', + ], + ], + + 'algoliasearch_categories/categories/category_additional_attributes' => [ + [ + 'attribute' => 'name', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'path', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'meta_title', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'meta_keywords', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'meta_description', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'product_count', + 'searchable' => '2', + 'order' => 'unordered', + 'retrievable' => '1', + ], + ], + 'algoliasearch_categories/categories/custom_ranking_category_attributes' => [ + [ + 'attribute' => 'product_count', + 'order' => 'desc', + ], + ], + ]; + + public function __construct() + { + $this->serializeDefaultArrayConfigData(); + $this->mergeDefaultDataWithArrayData(); + } + + /** + * @return string[] + */ + public function getDefaultConfigData(): array + { + return $this->defaultConfigData; + } + + /** + * @return void + */ + protected function serializeDefaultArrayConfigData(): void + { + foreach ($this->defaultArrayConfigData as $path => $array) { + $this->defaultArrayConfigData[$path] = json_encode($array); + } + } + + /** + * @return void + */ + protected function mergeDefaultDataWithArrayData(): void + { + $this->defaultConfigData = array_merge($this->defaultConfigData, $this->defaultArrayConfigData); + } +} diff --git a/Test/Integration/TestCase.php b/Test/Integration/TestCase.php index 04153ed8a..8d1378067 100644 --- a/Test/Integration/TestCase.php +++ b/Test/Integration/TestCase.php @@ -8,11 +8,11 @@ use Algolia\AlgoliaSearch\Service\AlgoliaConnector; use Algolia\AlgoliaSearch\Service\IndexNameFetcher; use Algolia\AlgoliaSearch\Service\IndexOptionsBuilder; -use Algolia\AlgoliaSearch\Setup\Patch\Schema\ConfigPatch; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento246CE; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento246EE; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento247CE; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento247EE; +use Algolia\AlgoliaSearch\Test\Integration\Config\DefaultConfigProvider; use Magento\Framework\App\Config\ScopeConfigInterface; use Magento\Framework\App\ProductMetadataInterface; use Magento\Framework\ObjectManagerInterface; @@ -70,9 +70,9 @@ protected function getIndexName(string $storeIndexPart): string protected function resetConfigs($configs = []) { - /** @var ConfigPatch $installClass */ - $installClass = $this->getObjectManager()->get(ConfigPatch::class); - $defaultConfigData = $installClass->getDefaultConfigData(); + /** @var DefaultConfigProvider $defaultConfigProvider */ + $defaultConfigProvider = $this->getObjectManager()->get(DefaultConfigProvider::class); + $defaultConfigData = $defaultConfigProvider->getDefaultConfigData(); foreach ($configs as $config) { $value = (string) $defaultConfigData[$config]; diff --git a/etc/config.xml b/etc/config.xml index a454e0232..402cdc96b 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -7,17 +7,33 @@ --> + + + 1 + 0 + magento2_ + user_allowed_save_cookie #btn-cookie-allow 15552000000 + + 1 + .algolia-search-input + 6 + 2 + 0 + 1000 + 2 + 1 + 0 1 300 0 @@ -28,8 +44,12 @@ 0 + + 0 + .columns + 9 0 @@ -42,6 +62,9 @@ 1 + 1 + 1 + 0 0 @@ -49,29 +72,68 @@ 1,3 + + 0 0 categoryPageId 0 + + 0 + 0 /// + + + + + Frequently bought together + 6 + + + Related products + 6 + + + Trending items + 6 + + + Looking Similar + 6 + + + + + + + 265 + 265 + image + + + + 0 0 */5 * * * * + 5 + 3 1 + 1 @@ -88,6 +150,16 @@ 1 + + + + 0 + .ais-Hits-item a.result, .ais-InfiniteHits-item a.result + disable + .action.primary.tocart + + + @@ -99,28 +171,27 @@ - - - - Frequently bought together - 6 - - - Related products - 6 - - - Trending items - 6 - - - Looking Similar - 6 - - - + + + + 0 + 3000 + 1 + 0 + + + + allOptional + 0 + 0 + 0 + 0 + 1 + 0 + all + Googlebot\nBingbot 1000 10000 us From 0391979b2586380233b2e63dbab6ff3fd2c50b03 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 18 Sep 2025 17:50:48 +0200 Subject: [PATCH 085/119] Revert "MAGE-1404: refactor config.xml and create DefaultConfigProvider for integration tests" This reverts commit 98ff621dc266aca06a690a66528320c56d17b661. --- .../Config/DefaultConfigProvider.php | 269 ------------------ Test/Integration/TestCase.php | 8 +- etc/config.xml | 111 ++------ 3 files changed, 24 insertions(+), 364 deletions(-) delete mode 100644 Test/Integration/Config/DefaultConfigProvider.php diff --git a/Test/Integration/Config/DefaultConfigProvider.php b/Test/Integration/Config/DefaultConfigProvider.php deleted file mode 100644 index 6b6c1316b..000000000 --- a/Test/Integration/Config/DefaultConfigProvider.php +++ /dev/null @@ -1,269 +0,0 @@ - '1', - 'algoliasearch_credentials/credentials/enable_frontend' => '1', - 'algoliasearch_credentials/credentials/application_id' => '', - 'algoliasearch_credentials/credentials/search_only_api_key' => '', - 'algoliasearch_credentials/credentials/api_key' => '', - 'algoliasearch_credentials/credentials/debug' => '0', - 'algoliasearch_credentials/credentials/index_prefix' => 'magento2_', - - 'algoliasearch_autocomplete/autocomplete/is_popup_enabled' => '1', - 'algoliasearch_autocomplete/autocomplete/nb_of_products_suggestions' => '6', - 'algoliasearch_autocomplete/autocomplete/nb_of_categories_suggestions' => '2', - 'algoliasearch_autocomplete/autocomplete/nb_of_queries_suggestions' => '0', - 'algoliasearch_autocomplete/autocomplete/min_popularity' => '1000', - 'algoliasearch_autocomplete/autocomplete/min_number_of_results' => '2', - 'algoliasearch_autocomplete/autocomplete/render_template_directives' => '1', - 'algoliasearch_autocomplete/autocomplete/debug' => '0', - - 'algoliasearch_instant/instant/is_instant_enabled' => '0', - 'algoliasearch_instant/instant/instant_selector' => '.columns', - 'algoliasearch_instant/instant/number_product_results' => '9', - 'algoliasearch_instant/instant/max_values_per_facet' => '10', - 'algoliasearch_instant/instant/show_suggestions_on_no_result_page' => '1', - 'algoliasearch_instant/instant/add_to_cart_enable' => '1', - 'algoliasearch_instant/instant/infinite_scroll_enable' => '0', - - 'algoliasearch_products/products/use_adaptive_image' => '0', - - 'algoliasearch_categories/categories/show_cats_not_included_in_navigation' => '0', - 'algoliasearch_categories/categories/index_empty_categories' => '0', - - 'algoliasearch_images/image/width' => '265', - 'algoliasearch_images/image/height' => '265', - 'algoliasearch_images/image/type' => 'image', - - 'algoliasearch_queue/queue/active' => '0', - 'algoliasearch_queue/queue/number_of_job_to_run' => '5', - 'algoliasearch_queue/queue/number_of_retries' => '3', - 'algoliasearch_queue/queue/use_tmp_index' => '1', - - 'algoliasearch_cc_analytics/cc_analytics_group/enable' => '0', - 'algoliasearch_cc_analytics/cc_analytics_group/is_selector' => '.ais-Hits-item a.result, .ais-InfiniteHits-item a.result', - 'algoliasearch_cc_analytics/cc_analytics_group/enable_conversion_analytics' => 'disabled', - 'algoliasearch_cc_analytics/cc_analytics_group/add_to_cart_selector' => '.action.primary.tocart', - - 'algoliasearch_analytics/analytics_group/enable' => '0', - 'algoliasearch_analytics/analytics_group/delay' => '3000', - 'algoliasearch_analytics/analytics_group/trigger_on_ui_interaction' => '1', - 'algoliasearch_analytics/analytics_group/push_initial_search' => '0', - - 'algoliasearch_synonyms/synonyms_group/enable_synonyms' => '0', - - 'algoliasearch_advanced/advanced/number_of_element_by_page' => '300', - 'algoliasearch_advanced/advanced/remove_words_if_no_result' => 'allOptional', - 'algoliasearch_advanced/advanced/partial_update' => '0', - 'algoliasearch_advanced/advanced/customer_groups_enable' => '0', - 'algoliasearch_advanced/advanced/remove_branding' => '0', - 'algoliasearch_autocomplete/autocomplete/autocomplete_selector' => '.algolia-search-input', - 'algoliasearch_advanced/advanced/index_product_on_category_products_update' => '1', - 'algoliasearch_advanced/advanced/prevent_backend_rendering' => '0', - 'algoliasearch_advanced/advanced/prevent_backend_rendering_display_mode' => 'all', - 'algoliasearch_advanced/advanced/backend_rendering_allowed_user_agents' => "Googlebot\nBingbot", - 'algoliasearch_advanced/queue/archive_clear_limit' => '30', - ]; - - /** - * @var string[][][] - */ - protected $defaultArrayConfigData = [ - 'algoliasearch_autocomplete/autocomplete/sections' => [ - [ - 'name' => 'pages', - 'label' => 'Pages', - 'hitsPerPage' => '2', - ], - ], - 'algoliasearch_autocomplete/autocomplete/excluded_pages' => [ - [ - 'attribute' => 'no-route', - ], - ], - - 'algoliasearch_instant/instant_facets/facets' => [ - [ - 'attribute' => 'price', - 'type' => 'slider', - 'label' => 'Price', - 'searchable' => '2', - 'create_rule' => '2', - ], - [ - 'attribute' => 'categories', - 'type' => 'conjunctive', - 'label' => 'Categories', - 'searchable' => '2', - 'create_rule' => '2', - ], - [ - 'attribute' => 'color', - 'type' => 'disjunctive', - 'label' => 'Colors', - 'searchable' => '1', - 'create_rule' => '2', - ], - ], - 'algoliasearch_instant/instant_sorts/sorts' => [ - [ - 'attribute' => 'price', - 'sort' => 'asc', - 'sortLabel' => 'Lowest price', - ], - [ - 'attribute' => 'price', - 'sort' => 'desc', - 'sortLabel' => 'Highest price', - ], - [ - 'attribute' => 'created_at', - 'sort' => 'desc', - 'sortLabel' => 'Newest first', - ], - ], - - 'algoliasearch_products/products/product_additional_attributes' => [ - [ - 'attribute' => 'name', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'sku', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'manufacturer', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'categories', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'color', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'price', - 'searchable' => '2', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'rating_summary', - 'searchable' => '2', - 'order' => 'unordered', - 'retrievable' => '1', - ], - ], - 'algoliasearch_products/products/custom_ranking_product_attributes' => [ - [ - 'attribute' => 'in_stock', - 'order' => 'desc', - ], - [ - 'attribute' => 'ordered_qty', - 'order' => 'desc', - ], - [ - 'attribute' => 'created_at', - 'order' => 'desc', - ], - ], - - 'algoliasearch_categories/categories/category_additional_attributes' => [ - [ - 'attribute' => 'name', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'path', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'meta_title', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'meta_keywords', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'meta_description', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'product_count', - 'searchable' => '2', - 'order' => 'unordered', - 'retrievable' => '1', - ], - ], - 'algoliasearch_categories/categories/custom_ranking_category_attributes' => [ - [ - 'attribute' => 'product_count', - 'order' => 'desc', - ], - ], - ]; - - public function __construct() - { - $this->serializeDefaultArrayConfigData(); - $this->mergeDefaultDataWithArrayData(); - } - - /** - * @return string[] - */ - public function getDefaultConfigData(): array - { - return $this->defaultConfigData; - } - - /** - * @return void - */ - protected function serializeDefaultArrayConfigData(): void - { - foreach ($this->defaultArrayConfigData as $path => $array) { - $this->defaultArrayConfigData[$path] = json_encode($array); - } - } - - /** - * @return void - */ - protected function mergeDefaultDataWithArrayData(): void - { - $this->defaultConfigData = array_merge($this->defaultConfigData, $this->defaultArrayConfigData); - } -} diff --git a/Test/Integration/TestCase.php b/Test/Integration/TestCase.php index 8d1378067..04153ed8a 100644 --- a/Test/Integration/TestCase.php +++ b/Test/Integration/TestCase.php @@ -8,11 +8,11 @@ use Algolia\AlgoliaSearch\Service\AlgoliaConnector; use Algolia\AlgoliaSearch\Service\IndexNameFetcher; use Algolia\AlgoliaSearch\Service\IndexOptionsBuilder; +use Algolia\AlgoliaSearch\Setup\Patch\Schema\ConfigPatch; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento246CE; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento246EE; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento247CE; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento247EE; -use Algolia\AlgoliaSearch\Test\Integration\Config\DefaultConfigProvider; use Magento\Framework\App\Config\ScopeConfigInterface; use Magento\Framework\App\ProductMetadataInterface; use Magento\Framework\ObjectManagerInterface; @@ -70,9 +70,9 @@ protected function getIndexName(string $storeIndexPart): string protected function resetConfigs($configs = []) { - /** @var DefaultConfigProvider $defaultConfigProvider */ - $defaultConfigProvider = $this->getObjectManager()->get(DefaultConfigProvider::class); - $defaultConfigData = $defaultConfigProvider->getDefaultConfigData(); + /** @var ConfigPatch $installClass */ + $installClass = $this->getObjectManager()->get(ConfigPatch::class); + $defaultConfigData = $installClass->getDefaultConfigData(); foreach ($configs as $config) { $value = (string) $defaultConfigData[$config]; diff --git a/etc/config.xml b/etc/config.xml index 402cdc96b..a454e0232 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -7,33 +7,17 @@ --> - - - 1 - 0 - magento2_ - user_allowed_save_cookie #btn-cookie-allow 15552000000 - - 1 - .algolia-search-input - 6 - 2 - 0 - 1000 - 2 - 1 - 0 1 300 0 @@ -44,12 +28,8 @@ 0 - - 0 - .columns - 9 0 @@ -62,9 +42,6 @@ 1 - 1 - 1 - 0 0 @@ -72,68 +49,29 @@ 1,3 - - 0 0 categoryPageId 0 - - 0 - 0 /// - - - - - Frequently bought together - 6 - - - Related products - 6 - - - Trending items - 6 - - - Looking Similar - 6 - - - - - - - 265 - 265 - image - - - - 0 0 */5 * * * * - 5 - 3 1 - 1 @@ -150,16 +88,6 @@ 1 - - - - 0 - .ais-Hits-item a.result, .ais-InfiniteHits-item a.result - disable - .action.primary.tocart - - - @@ -171,27 +99,28 @@ - - - - 0 - 3000 - 1 - 0 - - - + + + + Frequently bought together + 6 + + + Related products + 6 + + + Trending items + 6 + + + Looking Similar + 6 + + + - allOptional - 0 - 0 - 0 - 0 - 1 - 0 - all - Googlebot\nBingbot 1000 10000 us From 96b71f0e8b55bbc4ec76441041c23ce8a5a25974 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 18 Sep 2025 17:52:56 +0200 Subject: [PATCH 086/119] MAGE-1404: refactor config.xml and create DefaultConfigProvider for integration tests --- .../Config/DefaultConfigProvider.php | 269 ++++++++++++++++++ Test/Integration/TestCase.php | 8 +- etc/config.xml | 111 ++++++-- 3 files changed, 364 insertions(+), 24 deletions(-) create mode 100644 Test/Integration/Config/DefaultConfigProvider.php diff --git a/Test/Integration/Config/DefaultConfigProvider.php b/Test/Integration/Config/DefaultConfigProvider.php new file mode 100644 index 000000000..6b6c1316b --- /dev/null +++ b/Test/Integration/Config/DefaultConfigProvider.php @@ -0,0 +1,269 @@ + '1', + 'algoliasearch_credentials/credentials/enable_frontend' => '1', + 'algoliasearch_credentials/credentials/application_id' => '', + 'algoliasearch_credentials/credentials/search_only_api_key' => '', + 'algoliasearch_credentials/credentials/api_key' => '', + 'algoliasearch_credentials/credentials/debug' => '0', + 'algoliasearch_credentials/credentials/index_prefix' => 'magento2_', + + 'algoliasearch_autocomplete/autocomplete/is_popup_enabled' => '1', + 'algoliasearch_autocomplete/autocomplete/nb_of_products_suggestions' => '6', + 'algoliasearch_autocomplete/autocomplete/nb_of_categories_suggestions' => '2', + 'algoliasearch_autocomplete/autocomplete/nb_of_queries_suggestions' => '0', + 'algoliasearch_autocomplete/autocomplete/min_popularity' => '1000', + 'algoliasearch_autocomplete/autocomplete/min_number_of_results' => '2', + 'algoliasearch_autocomplete/autocomplete/render_template_directives' => '1', + 'algoliasearch_autocomplete/autocomplete/debug' => '0', + + 'algoliasearch_instant/instant/is_instant_enabled' => '0', + 'algoliasearch_instant/instant/instant_selector' => '.columns', + 'algoliasearch_instant/instant/number_product_results' => '9', + 'algoliasearch_instant/instant/max_values_per_facet' => '10', + 'algoliasearch_instant/instant/show_suggestions_on_no_result_page' => '1', + 'algoliasearch_instant/instant/add_to_cart_enable' => '1', + 'algoliasearch_instant/instant/infinite_scroll_enable' => '0', + + 'algoliasearch_products/products/use_adaptive_image' => '0', + + 'algoliasearch_categories/categories/show_cats_not_included_in_navigation' => '0', + 'algoliasearch_categories/categories/index_empty_categories' => '0', + + 'algoliasearch_images/image/width' => '265', + 'algoliasearch_images/image/height' => '265', + 'algoliasearch_images/image/type' => 'image', + + 'algoliasearch_queue/queue/active' => '0', + 'algoliasearch_queue/queue/number_of_job_to_run' => '5', + 'algoliasearch_queue/queue/number_of_retries' => '3', + 'algoliasearch_queue/queue/use_tmp_index' => '1', + + 'algoliasearch_cc_analytics/cc_analytics_group/enable' => '0', + 'algoliasearch_cc_analytics/cc_analytics_group/is_selector' => '.ais-Hits-item a.result, .ais-InfiniteHits-item a.result', + 'algoliasearch_cc_analytics/cc_analytics_group/enable_conversion_analytics' => 'disabled', + 'algoliasearch_cc_analytics/cc_analytics_group/add_to_cart_selector' => '.action.primary.tocart', + + 'algoliasearch_analytics/analytics_group/enable' => '0', + 'algoliasearch_analytics/analytics_group/delay' => '3000', + 'algoliasearch_analytics/analytics_group/trigger_on_ui_interaction' => '1', + 'algoliasearch_analytics/analytics_group/push_initial_search' => '0', + + 'algoliasearch_synonyms/synonyms_group/enable_synonyms' => '0', + + 'algoliasearch_advanced/advanced/number_of_element_by_page' => '300', + 'algoliasearch_advanced/advanced/remove_words_if_no_result' => 'allOptional', + 'algoliasearch_advanced/advanced/partial_update' => '0', + 'algoliasearch_advanced/advanced/customer_groups_enable' => '0', + 'algoliasearch_advanced/advanced/remove_branding' => '0', + 'algoliasearch_autocomplete/autocomplete/autocomplete_selector' => '.algolia-search-input', + 'algoliasearch_advanced/advanced/index_product_on_category_products_update' => '1', + 'algoliasearch_advanced/advanced/prevent_backend_rendering' => '0', + 'algoliasearch_advanced/advanced/prevent_backend_rendering_display_mode' => 'all', + 'algoliasearch_advanced/advanced/backend_rendering_allowed_user_agents' => "Googlebot\nBingbot", + 'algoliasearch_advanced/queue/archive_clear_limit' => '30', + ]; + + /** + * @var string[][][] + */ + protected $defaultArrayConfigData = [ + 'algoliasearch_autocomplete/autocomplete/sections' => [ + [ + 'name' => 'pages', + 'label' => 'Pages', + 'hitsPerPage' => '2', + ], + ], + 'algoliasearch_autocomplete/autocomplete/excluded_pages' => [ + [ + 'attribute' => 'no-route', + ], + ], + + 'algoliasearch_instant/instant_facets/facets' => [ + [ + 'attribute' => 'price', + 'type' => 'slider', + 'label' => 'Price', + 'searchable' => '2', + 'create_rule' => '2', + ], + [ + 'attribute' => 'categories', + 'type' => 'conjunctive', + 'label' => 'Categories', + 'searchable' => '2', + 'create_rule' => '2', + ], + [ + 'attribute' => 'color', + 'type' => 'disjunctive', + 'label' => 'Colors', + 'searchable' => '1', + 'create_rule' => '2', + ], + ], + 'algoliasearch_instant/instant_sorts/sorts' => [ + [ + 'attribute' => 'price', + 'sort' => 'asc', + 'sortLabel' => 'Lowest price', + ], + [ + 'attribute' => 'price', + 'sort' => 'desc', + 'sortLabel' => 'Highest price', + ], + [ + 'attribute' => 'created_at', + 'sort' => 'desc', + 'sortLabel' => 'Newest first', + ], + ], + + 'algoliasearch_products/products/product_additional_attributes' => [ + [ + 'attribute' => 'name', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'sku', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'manufacturer', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'categories', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'color', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'price', + 'searchable' => '2', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'rating_summary', + 'searchable' => '2', + 'order' => 'unordered', + 'retrievable' => '1', + ], + ], + 'algoliasearch_products/products/custom_ranking_product_attributes' => [ + [ + 'attribute' => 'in_stock', + 'order' => 'desc', + ], + [ + 'attribute' => 'ordered_qty', + 'order' => 'desc', + ], + [ + 'attribute' => 'created_at', + 'order' => 'desc', + ], + ], + + 'algoliasearch_categories/categories/category_additional_attributes' => [ + [ + 'attribute' => 'name', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'path', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'meta_title', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'meta_keywords', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'meta_description', + 'searchable' => '1', + 'order' => 'unordered', + 'retrievable' => '1', + ], + [ + 'attribute' => 'product_count', + 'searchable' => '2', + 'order' => 'unordered', + 'retrievable' => '1', + ], + ], + 'algoliasearch_categories/categories/custom_ranking_category_attributes' => [ + [ + 'attribute' => 'product_count', + 'order' => 'desc', + ], + ], + ]; + + public function __construct() + { + $this->serializeDefaultArrayConfigData(); + $this->mergeDefaultDataWithArrayData(); + } + + /** + * @return string[] + */ + public function getDefaultConfigData(): array + { + return $this->defaultConfigData; + } + + /** + * @return void + */ + protected function serializeDefaultArrayConfigData(): void + { + foreach ($this->defaultArrayConfigData as $path => $array) { + $this->defaultArrayConfigData[$path] = json_encode($array); + } + } + + /** + * @return void + */ + protected function mergeDefaultDataWithArrayData(): void + { + $this->defaultConfigData = array_merge($this->defaultConfigData, $this->defaultArrayConfigData); + } +} diff --git a/Test/Integration/TestCase.php b/Test/Integration/TestCase.php index 04153ed8a..8d1378067 100644 --- a/Test/Integration/TestCase.php +++ b/Test/Integration/TestCase.php @@ -8,11 +8,11 @@ use Algolia\AlgoliaSearch\Service\AlgoliaConnector; use Algolia\AlgoliaSearch\Service\IndexNameFetcher; use Algolia\AlgoliaSearch\Service\IndexOptionsBuilder; -use Algolia\AlgoliaSearch\Setup\Patch\Schema\ConfigPatch; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento246CE; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento246EE; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento247CE; use Algolia\AlgoliaSearch\Test\Integration\AssertValues\Magento247EE; +use Algolia\AlgoliaSearch\Test\Integration\Config\DefaultConfigProvider; use Magento\Framework\App\Config\ScopeConfigInterface; use Magento\Framework\App\ProductMetadataInterface; use Magento\Framework\ObjectManagerInterface; @@ -70,9 +70,9 @@ protected function getIndexName(string $storeIndexPart): string protected function resetConfigs($configs = []) { - /** @var ConfigPatch $installClass */ - $installClass = $this->getObjectManager()->get(ConfigPatch::class); - $defaultConfigData = $installClass->getDefaultConfigData(); + /** @var DefaultConfigProvider $defaultConfigProvider */ + $defaultConfigProvider = $this->getObjectManager()->get(DefaultConfigProvider::class); + $defaultConfigData = $defaultConfigProvider->getDefaultConfigData(); foreach ($configs as $config) { $value = (string) $defaultConfigData[$config]; diff --git a/etc/config.xml b/etc/config.xml index a454e0232..402cdc96b 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -7,17 +7,33 @@ --> + + + 1 + 0 + magento2_ + user_allowed_save_cookie #btn-cookie-allow 15552000000 + + 1 + .algolia-search-input + 6 + 2 + 0 + 1000 + 2 + 1 + 0 1 300 0 @@ -28,8 +44,12 @@ 0 + + 0 + .columns + 9 0 @@ -42,6 +62,9 @@ 1 + 1 + 1 + 0 0 @@ -49,29 +72,68 @@ 1,3 + + 0 0 categoryPageId 0 + + 0 + 0 /// + + + + + Frequently bought together + 6 + + + Related products + 6 + + + Trending items + 6 + + + Looking Similar + 6 + + + + + + + 265 + 265 + image + + + + 0 0 */5 * * * * + 5 + 3 1 + 1 @@ -88,6 +150,16 @@ 1 + + + + 0 + .ais-Hits-item a.result, .ais-InfiniteHits-item a.result + disable + .action.primary.tocart + + + @@ -99,28 +171,27 @@ - - - - Frequently bought together - 6 - - - Related products - 6 - - - Trending items - 6 - - - Looking Similar - 6 - - - + + + + 0 + 3000 + 1 + 0 + + + + allOptional + 0 + 0 + 0 + 0 + 1 + 0 + all + Googlebot\nBingbot 1000 10000 us From a33cd23db0deb4d5403bc0be2135fc7bc7d8967e Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Thu, 18 Sep 2025 17:56:51 +0200 Subject: [PATCH 087/119] MAGE-1404: removed ConfigPatch.php --- Setup/Patch/Schema/ConfigPatch.php | 365 ----------------------------- 1 file changed, 365 deletions(-) delete mode 100644 Setup/Patch/Schema/ConfigPatch.php diff --git a/Setup/Patch/Schema/ConfigPatch.php b/Setup/Patch/Schema/ConfigPatch.php deleted file mode 100644 index 8884a2f3f..000000000 --- a/Setup/Patch/Schema/ConfigPatch.php +++ /dev/null @@ -1,365 +0,0 @@ - '1', - 'algoliasearch_credentials/credentials/enable_frontend' => '1', - 'algoliasearch_credentials/credentials/application_id' => '', - 'algoliasearch_credentials/credentials/search_only_api_key' => '', - 'algoliasearch_credentials/credentials/api_key' => '', - 'algoliasearch_credentials/credentials/debug' => '0', - 'algoliasearch_credentials/credentials/index_prefix' => 'magento2_', - - 'algoliasearch_autocomplete/autocomplete/is_popup_enabled' => '1', - 'algoliasearch_autocomplete/autocomplete/nb_of_products_suggestions' => '6', - 'algoliasearch_autocomplete/autocomplete/nb_of_categories_suggestions' => '2', - 'algoliasearch_autocomplete/autocomplete/nb_of_queries_suggestions' => '0', - 'algoliasearch_autocomplete/autocomplete/min_popularity' => '1000', - 'algoliasearch_autocomplete/autocomplete/min_number_of_results' => '2', - 'algoliasearch_autocomplete/autocomplete/render_template_directives' => '1', - 'algoliasearch_autocomplete/autocomplete/debug' => '0', - - 'algoliasearch_instant/instant/is_instant_enabled' => '0', - 'algoliasearch_instant/instant/instant_selector' => '.columns', - 'algoliasearch_instant/instant/number_product_results' => '9', - 'algoliasearch_instant/instant/max_values_per_facet' => '10', - 'algoliasearch_instant/instant/show_suggestions_on_no_result_page' => '1', - 'algoliasearch_instant/instant/add_to_cart_enable' => '1', - 'algoliasearch_instant/instant/infinite_scroll_enable' => '0', - - 'algoliasearch_products/products/use_adaptive_image' => '0', - - 'algoliasearch_categories/categories/show_cats_not_included_in_navigation' => '0', - 'algoliasearch_categories/categories/index_empty_categories' => '0', - - 'algoliasearch_images/image/width' => '265', - 'algoliasearch_images/image/height' => '265', - 'algoliasearch_images/image/type' => 'image', - - 'algoliasearch_queue/queue/active' => '0', - 'algoliasearch_queue/queue/number_of_job_to_run' => '5', - 'algoliasearch_queue/queue/number_of_retries' => '3', - 'algoliasearch_queue/queue/use_tmp_index' => '1', - - 'algoliasearch_cc_analytics/cc_analytics_group/enable' => '0', - 'algoliasearch_cc_analytics/cc_analytics_group/is_selector' => '.ais-Hits-item a.result, .ais-InfiniteHits-item a.result', - 'algoliasearch_cc_analytics/cc_analytics_group/enable_conversion_analytics' => 'disabled', - 'algoliasearch_cc_analytics/cc_analytics_group/add_to_cart_selector' => '.action.primary.tocart', - - 'algoliasearch_analytics/analytics_group/enable' => '0', - 'algoliasearch_analytics/analytics_group/delay' => '3000', - 'algoliasearch_analytics/analytics_group/trigger_on_ui_interaction' => '1', - 'algoliasearch_analytics/analytics_group/push_initial_search' => '0', - - 'algoliasearch_synonyms/synonyms_group/enable_synonyms' => '0', - - 'algoliasearch_advanced/advanced/number_of_element_by_page' => '300', - 'algoliasearch_advanced/advanced/remove_words_if_no_result' => 'allOptional', - 'algoliasearch_advanced/advanced/partial_update' => '0', - 'algoliasearch_advanced/advanced/customer_groups_enable' => '0', - 'algoliasearch_advanced/advanced/remove_branding' => '0', - 'algoliasearch_autocomplete/autocomplete/autocomplete_selector' => '.algolia-search-input', - 'algoliasearch_advanced/advanced/index_product_on_category_products_update' => '1', - 'algoliasearch_advanced/advanced/prevent_backend_rendering' => '0', - 'algoliasearch_advanced/advanced/prevent_backend_rendering_display_mode' => 'all', - 'algoliasearch_advanced/advanced/backend_rendering_allowed_user_agents' => "Googlebot\nBingbot", - 'algoliasearch_advanced/queue/archive_clear_limit' => '30', - ]; - - /** - * @var string[][][] - */ - protected $defaultArrayConfigData = [ - 'algoliasearch_autocomplete/autocomplete/sections' => [ - [ - 'name' => 'pages', - 'label' => 'Pages', - 'hitsPerPage' => '2', - ], - ], - 'algoliasearch_autocomplete/autocomplete/excluded_pages' => [ - [ - 'attribute' => 'no-route', - ], - ], - - 'algoliasearch_instant/instant_facets/facets' => [ - [ - 'attribute' => 'price', - 'type' => 'slider', - 'label' => 'Price', - 'searchable' => '2', - 'create_rule' => '2', - ], - [ - 'attribute' => 'categories', - 'type' => 'conjunctive', - 'label' => 'Categories', - 'searchable' => '2', - 'create_rule' => '2', - ], - [ - 'attribute' => 'color', - 'type' => 'disjunctive', - 'label' => 'Colors', - 'searchable' => '1', - 'create_rule' => '2', - ], - ], - 'algoliasearch_instant/instant_sorts/sorts' => [ - [ - 'attribute' => 'price', - 'sort' => 'asc', - 'sortLabel' => 'Lowest price', - ], - [ - 'attribute' => 'price', - 'sort' => 'desc', - 'sortLabel' => 'Highest price', - ], - [ - 'attribute' => 'created_at', - 'sort' => 'desc', - 'sortLabel' => 'Newest first', - ], - ], - - 'algoliasearch_products/products/product_additional_attributes' => [ - [ - 'attribute' => 'name', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'sku', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'manufacturer', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'categories', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'color', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'price', - 'searchable' => '2', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'rating_summary', - 'searchable' => '2', - 'order' => 'unordered', - 'retrievable' => '1', - ], - ], - 'algoliasearch_products/products/custom_ranking_product_attributes' => [ - [ - 'attribute' => 'in_stock', - 'order' => 'desc', - ], - [ - 'attribute' => 'ordered_qty', - 'order' => 'desc', - ], - [ - 'attribute' => 'created_at', - 'order' => 'desc', - ], - ], - - 'algoliasearch_categories/categories/category_additional_attributes' => [ - [ - 'attribute' => 'name', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'path', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'meta_title', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'meta_keywords', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'meta_description', - 'searchable' => '1', - 'order' => 'unordered', - 'retrievable' => '1', - ], - [ - 'attribute' => 'product_count', - 'searchable' => '2', - 'order' => 'unordered', - 'retrievable' => '1', - ], - ], - 'algoliasearch_categories/categories/custom_ranking_category_attributes' => [ - [ - 'attribute' => 'product_count', - 'order' => 'desc', - ], - ], - ]; - - /** - * @param ConfigInterface $config - * @param ProductMetadataInterface $productMetadata - * @param ModuleDataSetupInterface $moduleDataSetup Magento\Framework\App\ResourceConnection - */ - public function __construct( - ConfigInterface $config, - ProductMetadataInterface $productMetadata, - ModuleDataSetupInterface $moduleDataSetup - ) { - $this->config = $config; - $this->productMetadata = $productMetadata; - $this->moduleDataSetup = $moduleDataSetup; - - $this->serializeDefaultArrayConfigData(); - $this->mergeDefaultDataWithArrayData(); - } - - /** - * @return array|string[] - */ - public static function getDependencies() - { - return []; - } - - /** - * @return array|string[] - */ - public function getAliases() - { - return []; - } - - /** - * @return ConfigPatch|void - */ - public function apply() - { - $movedConfigDirectives = [ - 'algoliasearch_credentials/credentials/use_adaptive_image' => 'algoliasearch_products/products/use_adaptive_image', - 'algoliasearch_products/products/number_product_results' => 'algoliasearch_instant/instant/number_product_results', - 'algoliasearch_products/products/show_suggestions_on_no_result_page' => 'algoliasearch_instant/instant/show_suggestions_on_no_result_page', - 'algoliasearch_credentials/credentials/is_popup_enabled' => 'algoliasearch_autocomplete/autocomplete/is_popup_enabled', - 'algoliasearch_credentials/credentials/is_instant_enabled' => 'algoliasearch_instant/instant/is_instant_enabled', - ]; - - $this->moduleDataSetup->getConnection()->startSetup(); - $connection = $this->moduleDataSetup->getConnection(); - $table = $this->moduleDataSetup->getTable('core_config_data'); - foreach ($movedConfigDirectives as $from => $to) { - try { - $connection->query('UPDATE ' . $table . ' SET path = "' . $to . '" WHERE path = "' . $from . '"'); - } catch (\Magento\Framework\DB\Adapter\DuplicateException) { - // Skip - } - } - - /* SET DEFAULT CONFIG DATA */ - $alreadyInserted = $connection->getConnection() - ->query('SELECT path, value FROM ' . $table . ' WHERE path LIKE "algoliasearch_%"') - ->fetchAll(\PDO::FETCH_KEY_PAIR); - - foreach ($this->defaultConfigData as $path => $value) { - if (isset($alreadyInserted[$path])) { - continue; - } - $this->config->saveConfig($path, $value, 'default', 0); - } - - $this->moduleDataSetup->getConnection()->endSetup(); - } - - /** - * @return string[] - */ - public function getDefaultConfigData() - { - return $this->defaultConfigData; - } - - /** - * @return void - */ - protected function serializeDefaultArrayConfigData() - { - $serializeMethod = 'serialize'; - - $magentoVersion = $this->productMetadata->getVersion(); - if (version_compare($magentoVersion, '2.2.0-dev', '>=') === true) { - $serializeMethod = 'json_encode'; - } - - foreach ($this->defaultArrayConfigData as $path => $array) { - $this->defaultArrayConfigData[$path] = $serializeMethod($array); - } - } - - /** - * @return void - */ - protected function mergeDefaultDataWithArrayData() - { - $this->defaultConfigData = array_merge($this->defaultConfigData, $this->defaultArrayConfigData); - } -} From 3fd6e0443f1969e8d61a064ad50bc8d5a5696290 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Tue, 23 Sep 2025 14:44:40 +0200 Subject: [PATCH 088/119] MAGE-1070: add FPT feature --- Helper/ConfigHelper.php | 15 ++++++++++++++- .../PriceManager/ProductWithChildren.php | 5 +++++ .../PriceManager/ProductWithoutChildren.php | 17 ++++++++++++++++- etc/adminhtml/system.xml | 10 ++++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/Helper/ConfigHelper.php b/Helper/ConfigHelper.php index 19d412b61..a1ae1e13e 100755 --- a/Helper/ConfigHelper.php +++ b/Helper/ConfigHelper.php @@ -18,6 +18,7 @@ use Magento\Framework\Locale\Currency; use Magento\Store\Model\ScopeInterface; use Magento\Store\Model\StoreManagerInterface; +use Magento\Weee\Helper\Data as WeeeHelper; class ConfigHelper { @@ -117,6 +118,7 @@ class ConfigHelper public const REMOVE_IF_NO_RESULT = 'algoliasearch_advanced/advanced/remove_words_if_no_result'; public const PARTIAL_UPDATES = 'algoliasearch_advanced/advanced/partial_update'; public const CUSTOMER_GROUPS_ENABLE = 'algoliasearch_advanced/advanced/customer_groups_enable'; + public const FPT_ENABLE = 'algoliasearch_advanced/advanced/fpt_enable'; public const REMOVE_PUB_DIR_IN_URL = 'algoliasearch_advanced/advanced/remove_pub_dir_in_url'; public const REMOVE_BRANDING = 'algoliasearch_advanced/advanced/remove_branding'; public const IDX_PRODUCT_ON_CAT_PRODUCTS_UPD = 'algoliasearch_advanced/advanced/index_product_on_category_products_update'; @@ -172,7 +174,8 @@ public function __construct( protected CookieHelper $cookieHelper, protected AutocompleteHelper $autocompleteConfig, protected InstantSearchHelper $instantSearchConfig, - protected QueueHelper $queueHelper + protected QueueHelper $queueHelper, + protected WeeeHelper $weeeHelper ) {} @@ -1173,6 +1176,16 @@ public function isCustomerGroupsEnabled($storeId = null): bool return $this->configInterface->isSetFlag(self::CUSTOMER_GROUPS_ENABLE, ScopeInterface::SCOPE_STORE, $storeId); } + /** + * @param $storeId + * @return bool + */ + public function isFptEnabled($storeId = null): bool + { + return $this->weeeHelper->isEnabled($storeId) && + $this->configInterface->isSetFlag(self::FPT_ENABLE, ScopeInterface::SCOPE_STORE, $storeId); + } + public function setCustomerGroupsEnabled(bool $val, ?string $scope = null, ?int $scopeId = null): void { $this->configWriter->save( diff --git a/Helper/Entity/Product/PriceManager/ProductWithChildren.php b/Helper/Entity/Product/PriceManager/ProductWithChildren.php index b929edd97..300d908de 100755 --- a/Helper/Entity/Product/PriceManager/ProductWithChildren.php +++ b/Helper/Entity/Product/PriceManager/ProductWithChildren.php @@ -69,6 +69,11 @@ protected function getMinMaxPrices(Product $product, $withTax, $subProducts, $cu $price = $minPrice ?? $this->getTaxPrice($product, $finalPrice, $withTax); $basePrice = $this->getTaxPrice($product, $basePrice, $withTax); + + if ($this->configHelper->isFptEnabled($subProduct->getStoreId())) { + $basePrice += $this->weeeTax->getWeeeAmount($subProduct); + } + $min = min($min, $price); $original = min($original, $basePrice); $max = max($max, $price); diff --git a/Helper/Entity/Product/PriceManager/ProductWithoutChildren.php b/Helper/Entity/Product/PriceManager/ProductWithoutChildren.php index 4c5883b1c..3432a57ad 100755 --- a/Helper/Entity/Product/PriceManager/ProductWithoutChildren.php +++ b/Helper/Entity/Product/PriceManager/ProductWithoutChildren.php @@ -16,6 +16,7 @@ use Magento\Customer\Model\ResourceModel\Group\CollectionFactory; use Magento\Framework\Pricing\PriceCurrencyInterface; use Magento\Tax\Helper\Data as TaxHelper; +use Magento\Weee\Model\Tax as WeeeTax; use Magento\Tax\Model\Config as TaxConfig; abstract class ProductWithoutChildren @@ -40,6 +41,10 @@ abstract class ProductWithoutChildren * @var TaxHelper */ protected $taxHelper; + /** + * @var WeeeTax + */ + protected $weeeTax; /** * @var Rule */ @@ -77,6 +82,7 @@ abstract class ProductWithoutChildren * @param PriceCurrencyInterface $priceCurrency * @param CatalogHelper $catalogHelper * @param TaxHelper $taxHelper + * @param WeeeTax $weeeTax * @param Rule $rule * @param ProductFactory $productloader * @param ScopedProductTierPriceManagementInterface $productTierPrice @@ -89,6 +95,7 @@ public function __construct( PriceCurrencyInterface $priceCurrency, CatalogHelper $catalogHelper, TaxHelper $taxHelper, + WeeeTax $weeeTax, Rule $rule, ProductFactory $productloader, ScopedProductTierPriceManagementInterface $productTierPrice, @@ -100,6 +107,7 @@ public function __construct( $this->priceCurrency = $priceCurrency; $this->catalogHelper = $catalogHelper; $this->taxHelper = $taxHelper; + $this->weeeTax = $weeeTax; $this->rule = $rule; $this->productloader = $productloader; $this->productTierPrice = $productTierPrice; @@ -146,6 +154,9 @@ public function addPriceData($customData, Product $product, $subProducts): array foreach ($currencies as $currencyCode) { $this->customData[$field][$currencyCode] = []; $price = $product->getPrice(); + if ($this->configHelper->isFptEnabled($product->getStoreId())) { + $price += $this->weeeTax->getWeeeAmount($product); + } if ($currencyCode !== $this->baseCurrencyCode) { $price = $this->convertPrice($price, $currencyCode); } @@ -257,7 +268,11 @@ protected function getSpecialPrice(Product $product, $currencyCode, $withTax, $s $specialPrices[$groupId] = []; $specialPrices[$groupId][] = $this->getRulePrice($groupId, $product, $subProducts); // The price with applied catalog rules - $specialPrices[$groupId][] = $product->getFinalPrice(); // The product's special price + $finalPrice = $product->getFinalPrice(); // The product's special price + if ($this->configHelper->isFptEnabled($product->getStoreId())) { + $finalPrice += $this->weeeTax->getWeeeAmount($product); + } + $specialPrices[$groupId][] = $finalPrice; $specialPrices[$groupId] = array_filter($specialPrices[$groupId], fn($price) => $price > 0); $specialPrice[$groupId] = false; if ($specialPrices[$groupId] && $specialPrices[$groupId] !== []) { diff --git a/etc/adminhtml/system.xml b/etc/adminhtml/system.xml index c231c489a..d841cc003 100755 --- a/etc/adminhtml/system.xml +++ b/etc/adminhtml/system.xml @@ -1454,6 +1454,16 @@ ]]> + + + Magento\Config\Model\Config\Source\Yesno + + The extension will only take into account global FPT per country (state scoped FPT are not supported). + ]]> + + Magento\Config\Model\Config\Source\Yesno From a4aaf6225de83291c9263d48d48a50dbbfacf1d4 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Tue, 23 Sep 2025 15:00:02 +0200 Subject: [PATCH 089/119] MAGE-1070: fix test --- Test/Unit/Helper/ConfigHelperTest.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Test/Unit/Helper/ConfigHelperTest.php b/Test/Unit/Helper/ConfigHelperTest.php index 78071ca34..c4032556d 100644 --- a/Test/Unit/Helper/ConfigHelperTest.php +++ b/Test/Unit/Helper/ConfigHelperTest.php @@ -17,8 +17,8 @@ use Magento\Framework\Event\ManagerInterface; use Magento\Framework\Locale\Currency; use Magento\Framework\Module\ResourceInterface; - use Magento\Store\Model\StoreManagerInterface; +use Magento\Weee\Helper\Data as WeeeHelper; use PHPUnit\Framework\TestCase; class ConfigHelperTest extends TestCase @@ -41,6 +41,8 @@ class ConfigHelperTest extends TestCase protected ?InstantSearchHelper $instantSearchHelper; protected ?QueueHelper $queueHelper; + protected ?WeeeHelper $weeeHelper; + protected function setUp(): void { $this->configInterface = $this->createMock(ScopeConfigInterface::class); @@ -59,6 +61,7 @@ protected function setUp(): void $this->autocompleteHelper = $this->createMock(AutocompleteHelper::class); $this->instantSearchHelper = $this->createMock(InstantSearchHelper::class); $this->queueHelper = $this->createMock(QueueHelper::class); + $this->weeeHelper = $this->createMock(WeeeHelper::class); $this->configHelper = new ConfigHelperTestable( $this->configInterface, @@ -76,7 +79,8 @@ protected function setUp(): void $this->cookieHelper, $this->autocompleteHelper, $this->instantSearchHelper, - $this->queueHelper + $this->queueHelper, + $this->weeeHelper ); } From c47604caab786ca9f91dad7417e8b0133421ebee Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Fri, 19 Sep 2025 11:02:59 -0400 Subject: [PATCH 090/119] Port replica forwarding fix to 3.17 MAGE-1426 Add unit test experiment with state machine MAGE-1426 Add wait operation MAGE-1426 Simplify state machine and clarify intent of unit tests MAGE-1427 Allow override of facet sort by renderingContent MAGE-1426 Add integration test for renderingContent MAGE-1426 Suppress Codacy false positive --- .gitignore | 1 + Service/IndexSettingsHandler.php | 1 + Service/Product/FacetBuilder.php | 2 +- .../Indexing/Config/ConfigTest.php | 88 ++++++--- .../Unit/Service/IndexSettingsHandlerTest.php | 181 +++++++++++++++++- view/frontend/web/js/instantsearch.js | 11 +- 6 files changed, 258 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 6319cde4d..8d7d00605 100755 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ vendor/ .php_cs.cache .vscode .idea +node_modules \ No newline at end of file diff --git a/Service/IndexSettingsHandler.php b/Service/IndexSettingsHandler.php index f17859674..577135273 100644 --- a/Service/IndexSettingsHandler.php +++ b/Service/IndexSettingsHandler.php @@ -55,6 +55,7 @@ public function setSettings( true, false ); + $this->connector->waitLastTask($indexOptions->getStoreId()); } if ($noForward) { $this->connector->setSettings( diff --git a/Service/Product/FacetBuilder.php b/Service/Product/FacetBuilder.php index 244e464aa..474faf826 100644 --- a/Service/Product/FacetBuilder.php +++ b/Service/Product/FacetBuilder.php @@ -95,7 +95,7 @@ protected function getRenderingContentValues(array $attributes): array { return array_combine( $attributes, - array_fill(0, count($attributes), [ 'sortRemainingBy' => 'alpha' ]) + array_fill(0, count($attributes), [ 'sortRemainingBy' => 'count' ]) ); } diff --git a/Test/Integration/Indexing/Config/ConfigTest.php b/Test/Integration/Indexing/Config/ConfigTest.php index d20367662..2f6af8375 100644 --- a/Test/Integration/Indexing/Config/ConfigTest.php +++ b/Test/Integration/Indexing/Config/ConfigTest.php @@ -3,18 +3,24 @@ namespace Algolia\AlgoliaSearch\Test\Integration\Indexing\Config; use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; +use Algolia\AlgoliaSearch\Exceptions\ExceededRetriesException; use Algolia\AlgoliaSearch\Model\IndicesConfigurator; use Algolia\AlgoliaSearch\Test\Integration\TestCase; +use Magento\Framework\Exception\LocalizedException; +use Magento\Framework\Exception\NoSuchEntityException; class ConfigTest extends TestCase { + + /** + * @throws NoSuchEntityException + * @throws ExceededRetriesException + * @throws AlgoliaException + * @throws LocalizedException + */ public function testFacets() { - /** @var IndicesConfigurator $indicesConfigurator */ - $indicesConfigurator = $this->getObjectManager()->create(IndicesConfigurator::class); - $indicesConfigurator->saveConfigurationToAlgolia(1); - - $this->algoliaConnector->waitLastTask(); + $this->syncSettingsToAlgolia(); $indexOptions = $this->indexOptionsBuilder->buildWithEnforcedIndex($this->indexPrefix . 'default_products'); $indexSettings = $this->algoliaConnector->getSettings($indexOptions); @@ -22,13 +28,30 @@ public function testFacets() $this->assertEquals($this->assertValues->attributesForFaceting, count($indexSettings['attributesForFaceting'])); } - public function testQueryRules() + public function testRenderingContent() { - /** @var IndicesConfigurator $indicesConfigurator */ - $indicesConfigurator = $this->getObjectManager()->create(IndicesConfigurator::class); - $indicesConfigurator->saveConfigurationToAlgolia(1); + $this->setConfig('algoliasearch_instant/instant_facets/enable_dynamic_facets', '1'); - $this->algoliaConnector->waitLastTask(); + $this->syncSettingsToAlgolia(); + + $indexOptions = $this->indexOptionsBuilder->buildWithEnforcedIndex($this->indexPrefix . 'default_products'); + $indexSettings = $this->algoliaConnector->getSettings($indexOptions); + + $renderingContent = $indexSettings['renderingContent']['facetOrdering']['values'] ?? null; + $this->assertNotNull($renderingContent, "Rendering content not found in product index"); + $this->assertEqualsCanonicalizing(['categories.level0', 'color', 'price.EUR.default', 'price.USD.default'], array_keys($renderingContent), "Expected facets not found in renderingContent"); + $this->assertEquals('count', $renderingContent['color']['sortRemainingBy'], "Default sort not set to count"); + } + + /** + * @throws NoSuchEntityException + * @throws ExceededRetriesException + * @throws AlgoliaException + * @throws LocalizedException + */ + public function testQueryRules() + { + $this->syncSettingsToAlgolia(); $client = $this->algoliaConnector->getClient(); @@ -94,6 +117,12 @@ public function testReplicaCreationWithCustomerGroups() $this->replicaCreationTest(true); } + /** + * @throws ExceededRetriesException + * @throws AlgoliaException + * @throws LocalizedException + * @throws NoSuchEntityException + */ private function replicaCreationTest($withCustomerGroups = false) { $enableCustomGroups = '0'; @@ -133,11 +162,7 @@ private function replicaCreationTest($withCustomerGroups = false) $this->indexPrefix . 'default_products_created_at_desc' => 'desc(created_at)', ]; - /** @var IndicesConfigurator $indicesConfigurator */ - $indicesConfigurator = $this->getObjectManager()->create(IndicesConfigurator::class); - $indicesConfigurator->saveConfigurationToAlgolia(1); - - $this->algoliaConnector->waitLastTask(); + $this->syncSettingsToAlgolia(); $indices = $this->algoliaConnector->listIndexes(); $indicesNames = array_map(fn($indexData) => $indexData['name'], $indices['items']); @@ -151,13 +176,15 @@ private function replicaCreationTest($withCustomerGroups = false) } } + /** + * @throws ExceededRetriesException + * @throws LocalizedException + * @throws NoSuchEntityException + * @throws AlgoliaException + */ public function testExtraSettings() { - /** @var IndicesConfigurator $indicesConfigurator */ - $indicesConfigurator = $this->getObjectManager()->create(IndicesConfigurator::class); - - $indicesConfigurator->saveConfigurationToAlgolia(1); - $this->algoliaConnector->waitLastTask(); + $this->syncSettingsToAlgolia(); $sections = ['products', 'categories', 'pages', 'suggestions']; @@ -182,8 +209,7 @@ public function testExtraSettings() $this->setConfig('algoliasearch_extra_settings/extra_settings/' . $section . '_extra_settings', '{"exactOnSingleWordQuery":"word"}'); } - $indicesConfigurator->saveConfigurationToAlgolia(1); - $this->algoliaConnector->waitLastTask(); + $this->syncSettingsToAlgolia(); foreach ($sections as $section) { $indexName = $this->indexPrefix . 'default_' . $section; @@ -223,4 +249,22 @@ public function testInvalidExtraSettings() $this->fail('AlgoliaException was not raised'); } + + /** + * @throws NoSuchEntityException + * @throws ExceededRetriesException + * @throws AlgoliaException + * @throws LocalizedException + */ + protected function syncSettingsToAlgolia(int $storeId = 1): IndicesConfigurator + { + /** @var IndicesConfigurator $indicesConfigurator */ + $indicesConfigurator = $this->getObjectManager()->get(IndicesConfigurator::class); + $indicesConfigurator->saveConfigurationToAlgolia($storeId); + + $this->algoliaConnector->waitLastTask(); + + return $indicesConfigurator; // return for reuse (as needed) + } + } diff --git a/Test/Unit/Service/IndexSettingsHandlerTest.php b/Test/Unit/Service/IndexSettingsHandlerTest.php index 62cd3c393..628db33af 100644 --- a/Test/Unit/Service/IndexSettingsHandlerTest.php +++ b/Test/Unit/Service/IndexSettingsHandlerTest.php @@ -18,17 +18,69 @@ class IndexSettingsHandlerTest extends TestCase private ?IndexSettingsHandler $handler = null; + /** + * State machine to track pending operations per store ID + * Format: [storeId => ['totalCalls' => int, 'waitCalled' => bool, 'batchesCompleted' => int]] + */ + private array $operationState = []; + protected function setUp(): void { $this->connector = $this->createMock(AlgoliaConnector::class); $this->config = $this->createMock(ConfigHelper::class); $this->indexOptions = $this->createMock(IndexOptionsInterface::class); + // Configure the mock to use our state machine + $this->setupStateMachineMock(); + $this->handler = new IndexSettingsHandlerTestable($this->connector, $this->config); } + private function setupStateMachineMock(): void + { + $this->connector->method('setSettings') + ->willReturnCallback(function($indexOptions, $settings, $forwardToReplicas, $mergeSettings, $mergeFrom = '') { + $storeId = $indexOptions->getStoreId(); + + // Initialize state if not exists + if (!isset($this->operationState[$storeId])) { + $this->operationState[$storeId] = [ + 'setSettingsCalled' => false, + 'waitCalled' => false + ]; + } + + // Check that setSettings is not stacked for this $storeId + if ($this->operationState[$storeId]['setSettingsCalled'] && + !$this->operationState[$storeId]['waitCalled']) { + throw new \RuntimeException( + // phpcs:ignore + "Cannot call setSettings on store $storeId: previous operation still pending. Call waitLastTask first." + ); + } + + // Update state + $this->operationState[$storeId]['setSettingsCalled'] = true; + $this->operationState[$storeId]['waitCalled'] = false; + }); + + $this->connector->method('waitLastTask') + ->willReturnCallback(function($storeId = null) { + if ($storeId !== null && isset($this->operationState[$storeId])) { + $this->operationState[$storeId]['waitCalled'] = true; + } + }); + } + + private function resetOperationState(): void + { + $this->operationState = []; + } + public function testSetSettingsWithForwardingEnabledAndMixedSettings(): void { + $this->resetOperationState(); + $storeId = 1; $settings = [ 'customRanking' => ['desc(price)'], @@ -44,7 +96,7 @@ public function testSetSettingsWithForwardingEnabledAndMixedSettings(): void $this->connector->expects($this->exactly(2)) ->method('setSettings') ->willReturnCallback( - function($indexOptions, $indexSettings, $forwardToReplicas, $mergeSettings, $mergeFrom) use (&$invocationCount) { + function($indexOptions, $indexSettings, $forwardToReplicas, $mergeSettings, $mergeFrom = '') use (&$invocationCount) { $invocationCount++; switch ($invocationCount) { @@ -66,6 +118,8 @@ function($indexOptions, $indexSettings, $forwardToReplicas, $mergeSettings, $mer public function testSetSettingsWithForwardingEnabledOnlyExcludedSettings(): void { + $this->resetOperationState(); + $storeId = 1; $settings = [ 'ranking' => ['asc(name)'], @@ -92,6 +146,8 @@ public function testSetSettingsWithForwardingEnabledOnlyExcludedSettings(): void public function testSetSettingsWithForwardingEnabledOnlyForwardableSettings(): void { + $this->resetOperationState(); + $storeId = 1; $settings = [ 'attributesToHighlight' => ['title'], @@ -117,6 +173,8 @@ public function testSetSettingsWithForwardingEnabledOnlyForwardableSettings(): v public function testSetSettingsWithForwardingDisabled(): void { + $this->resetOperationState(); + $storeId = 1; $settings = [ 'customRanking' => ['desc(price)'], @@ -142,6 +200,8 @@ public function testSetSettingsWithForwardingDisabled(): void public function testForwardSettingsWithEmptyInput(): void { + $this->resetOperationState(); + $storeId = 1; $settings = []; @@ -172,4 +232,123 @@ public function testSplitSettings(): void 'ranking' => ['asc(name)'] ], $noForward); } + + /** + * Ensure the state machine is working as expected by disabling replica forwarding + * and explicitly invoking subsequent setSettings operations + */ + public function testSubsequentSetSettingsWithoutWaitThrowsException(): void + { + $this->resetOperationState(); + + $storeId = 1; + $settings = ['attributesToRetrieve' => ['name']]; + + $this->indexOptions->method('getStoreId')->willReturn($storeId); + + // Disable forwarding for explicit test + $this->config->method('shouldForwardPrimaryIndexSettingsToReplicas') + ->willReturn(false); + + // First call should succeed + $this->handler->setSettings($this->indexOptions, $settings); + + // Second call without wait should throw exception + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage("Cannot call setSettings on store $storeId: previous operation still pending. Call waitLastTask first."); + + $this->handler->setSettings($this->indexOptions, $settings); + } + + /** + * Explicitly test the state machine succeeds by disabling replica forwarding + * and explicitly invoking the wait operation + * */ + public function testSubsequentSetSettingsAfterWaitSucceeds(): void + { + $this->resetOperationState(); + + $storeId = 1; + $settings = ['attributesToRetrieve' => ['name']]; + + $this->indexOptions->method('getStoreId')->willReturn($storeId); + // Disable forwarding for explicit test + $this->config->method('shouldForwardPrimaryIndexSettingsToReplicas') + ->willReturn(false); + + $this->connector->expects($this->exactly(2)) + ->method('setSettings'); + + $this->connector->expects($this->once()) + ->method('waitLastTask') + ->with($storeId); + + // First call should succeed + $this->handler->setSettings($this->indexOptions, $settings); + + // Wait for the task + $this->connector->waitLastTask($storeId); + + // Second call after wait should succeed + $this->handler->setSettings($this->indexOptions, $settings); + } + + public function testDifferentStoreIdsDontInterfere(): void + { + $this->resetOperationState(); + + $storeId1 = 1; + $storeId2 = 2; + $settings = ['attributesToRetrieve' => ['name']]; + + $indexOptions1 = $this->createMock(IndexOptionsInterface::class); + $indexOptions1->method('getStoreId')->willReturn($storeId1); + + $indexOptions2 = $this->createMock(IndexOptionsInterface::class); + $indexOptions2->method('getStoreId')->willReturn($storeId2); + + $this->config->method('shouldForwardPrimaryIndexSettingsToReplicas') + ->willReturn(false); + + // Both calls should succeed as they use different store IDs + $this->connector->expects($this->exactly(2)) + ->method('setSettings'); + + $this->handler->setSettings($indexOptions1, $settings); + $this->handler->setSettings($indexOptions2, $settings); + } + + /** + * Replica forwarding should abstract the wait operation internally + * However require caller to invoke wait for subsequent ops + * This is *by design* to minimize unnecessary IO blocking + * This test ensures this logic stays in place + */ + public function testForwardingEnabledMultipleCallsRequireWait(): void + { + $this->resetOperationState(); + + $storeId = 1; + $settings = [ + 'customRanking' => ['desc(price)'], + 'attributesToRetrieve' => ['name'] + ]; + + $this->indexOptions->method('getStoreId')->willReturn($storeId); + $this->config->method('shouldForwardPrimaryIndexSettingsToReplicas') + ->willReturn(true); + + // 2 internal calls + 1 explicit call + $this->connector->expects($this->exactly(3)) + ->method('setSettings'); + + // First call makes two internal setSettings calls + $this->handler->setSettings($this->indexOptions, $settings); + + // Second call to handler should fail because no wait was called + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage("Cannot call setSettings on store $storeId: previous operation still pending. Call waitLastTask first."); + + $this->handler->setSettings($this->indexOptions, $settings); + } } diff --git a/view/frontend/web/js/instantsearch.js b/view/frontend/web/js/instantsearch.js index 4cf433935..045bfed26 100644 --- a/view/frontend/web/js/instantsearch.js +++ b/view/frontend/web/js/instantsearch.js @@ -663,14 +663,21 @@ define([ }, getRefinementListOptions(facet) { - return { + const options = { container : this.getFacetContainer(facet), attribute : facet.attribute, limit : algoliaConfig.maxValuesPerFacet, templates : this.getRefinementsListTemplates(), - sortBy : ['count:desc', 'name:asc'], panelOptions: this.getRefinementFacetPanelOptions(facet) }; + if (!algoliaConfig.instant.isDynamicFacetsEnabled) { + options['sortBy'] = this.getFacetSortBy() + } + return options; + }, + + getFacetSortBy() { + return ['count:desc', 'name:asc']; }, getRefinementFacetPanelOptions(facet) { From 68864504b35ed75b4091a9b41903d4b2c6741669 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 24 Sep 2025 11:22:28 +0200 Subject: [PATCH 091/119] MAGE-1404: add data patch for legacy config migration --- Setup/Patch/Data/MigrateLegacyConfigPatch.php | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Setup/Patch/Data/MigrateLegacyConfigPatch.php diff --git a/Setup/Patch/Data/MigrateLegacyConfigPatch.php b/Setup/Patch/Data/MigrateLegacyConfigPatch.php new file mode 100644 index 000000000..41e30d563 --- /dev/null +++ b/Setup/Patch/Data/MigrateLegacyConfigPatch.php @@ -0,0 +1,64 @@ +moduleDataSetup->getConnection()->startSetup(); + + $this->moveIndexingSettings(); + + $this->moduleDataSetup->getConnection()->endSetup(); + + return $this; + } + + /** + * Migrate legacy configurations + * @return void + */ + protected function moveIndexingSettings(): void + { + $movedConfig = [ + 'algoliasearch_credentials/credentials/use_adaptive_image' => 'algoliasearch_products/products/use_adaptive_image', + 'algoliasearch_products/products/number_product_results' => 'algoliasearch_instant/instant/number_product_results', + 'algoliasearch_products/products/show_suggestions_on_no_result_page' => 'algoliasearch_instant/instant/show_suggestions_on_no_result_page', + 'algoliasearch_credentials/credentials/is_popup_enabled' => 'algoliasearch_autocomplete/autocomplete/is_popup_enabled', + 'algoliasearch_credentials/credentials/is_instant_enabled' => 'algoliasearch_instant/instant/is_instant_enabled', + ]; + + $this->migrateConfig($movedConfig); + } + + /** + * @inheritDoc + */ + public static function getDependencies(): array + { + return []; + } + + /** + * @inheritDoc + */ + public function getAliases(): array + { + return []; + } +} From a3316d8778f69e2ea528ce4b92e59372430d02b0 Mon Sep 17 00:00:00 2001 From: AISSAOUI Date: Fri, 20 Sep 2024 17:05:34 +0200 Subject: [PATCH 092/119] Including FPT on indexed product final price! From e8ed3901ee651abf2eed1380a10b539cf9dd8611 Mon Sep 17 00:00:00 2001 From: fasimana Date: Wed, 24 Sep 2025 19:16:31 -0400 Subject: [PATCH 093/119] MAGE-1422 Fix product getUrl method by aligning with Magento core behavior (#1796) This commit adds the UrlRewrite::REDIRECT_TYPE => 0 parameter to the Algolia product URL filter data, ensuring that only non-redirect URL rewrites are considered when generating product URLs. This aligns the Algolia URL generation with Magento's core URL handling behavior and prevents potential issues with redirect loops or incorrect URL generation. --- Model/Product/Url.php | 1 + 1 file changed, 1 insertion(+) diff --git a/Model/Product/Url.php b/Model/Product/Url.php index 53f92e564..a43da214b 100755 --- a/Model/Product/Url.php +++ b/Model/Product/Url.php @@ -67,6 +67,7 @@ public function getUrl(Product $product, $params = []) UrlRewrite::ENTITY_ID => $product->getId(), UrlRewrite::ENTITY_TYPE => ProductUrlRewriteGenerator::ENTITY_TYPE, UrlRewrite::STORE_ID => $storeId, + UrlRewrite::REDIRECT_TYPE => 0, ]; if ($categoryId) { $filterData[UrlRewrite::METADATA]['category_id'] = $categoryId; From afb933771fb684ce73e3c1ec8d52c6fcb378debd Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 24 Sep 2025 20:24:15 -0400 Subject: [PATCH 094/119] MAGE-1422 Update change log --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 429cf4f6f..8edee5aba 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,11 @@ ### Bug fixes - Fixed indexing queue templates escaping. +## 3.16.1 + +### Bug fixes +- Ensure that only non-redirect URL rewrites are considered when generating product URLs - thank you @fasimana + ## 3.16.0 ### Features From a1fdf28a35c58ff54a8803cf88ed5e333b14c7f2 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 24 Sep 2025 22:06:18 -0400 Subject: [PATCH 095/119] MAGE-1422 Fix branch matching --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3c389a1c..af2f5aabc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -173,6 +173,10 @@ jobs: workflows: magento-build-and-test-workflow: + when: + matches: + pattern: "^(feat|fix|chore)/MAGE.*" + value: << pipeline.git.branch >> jobs: - magento-build: matrix: From dd850c77bf0d5f86525e99ab77711962f2b47312 Mon Sep 17 00:00:00 2001 From: pikulsky Date: Mon, 29 Sep 2025 18:50:19 +0400 Subject: [PATCH 096/119] Fix store id for queue job (#1769) * Follow camel case for store ID * Fix using queue job's store ID for jobs sorting --- Model/Queue.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Model/Queue.php b/Model/Queue.php index 9a1a08d7e..b3891fe0e 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -578,7 +578,7 @@ protected function stackSortedJobs(array $sortedJobs, array $tempSortableJobs, ? SORT_ASC, 'method', SORT_ASC, - 'store_id', + 'storeId', SORT_ASC, 'job_id', SORT_ASC From 4a6c2e891d6993626bfde5a14cca6b67e91400c2 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Mon, 29 Sep 2025 17:06:54 +0200 Subject: [PATCH 097/119] MAGE-1420: update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8edee5aba..8ce1f0d9f 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,9 @@ ## 3.16.1 ### Bug fixes -- Ensure that only non-redirect URL rewrites are considered when generating product URLs - thank you @fasimana +- Ensure that only non-redirect URL rewrites are considered when generating product URLs - thank you @fasimana +- Fix store id for queue jobs sorting - thank you @pikulsky + ## 3.16.0 From c2c217255b707f247a2cc978fdf7526ca3ce6120 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Tue, 30 Sep 2025 09:17:52 -0400 Subject: [PATCH 098/119] MAGE-1434 Fix floating point precision bug in EventProcessor for v3.17 (#1830) * MAGE-1419 Prevent 422 Insights Event - Add to Cart with Discount (#1780) Adding a product to the cart may trigger a PHP and Algolia error due to the discount amount being too large. ## PHP Error ``` main.CRITICAL: Unable to send add to cart event due to Algolia events model misconfiguration: Discount must be a valid decimal number and total length must be no longer than 16 characters [] [] ``` * MAGE-1419 Add unit test class for EventProcessor * MAGE-1419 Test add to cart conversion * MAGE-1419 Test purchase conversion * MAGE-1419 Refactor and merge unit tests for 3.16 * MAGE-1419 Switch to type casting * MAGE-1434 Remove StoreManager as runtime dependency * MAGE-1434 Utilize Magento derived locale for determining decimial precision * MAGE-1434 Add tests for variable precision per locale * MAGE-1434 Fix convert purchase object ID type casting bug * MAGE-1434 Update change log * MAGE-1434 Add unit test for string IDs on request payload --------- Co-authored-by: PromInc Co-authored-by: Damien Couchez --- Api/Insights/EventProcessorInterface.php | 1 + CHANGELOG.md | 7 +- Helper/InsightsHelper.php | 3 +- Service/Insights/EventProcessor.php | 74 +- Test/Unit/Service/EventProcessorTest.php | 188 ---- .../Service/Insights/EventProcessorTest.php | 823 ++++++++++++++++++ .../{ => Insights}/EventProcessorTestable.php | 7 +- 7 files changed, 890 insertions(+), 213 deletions(-) delete mode 100644 Test/Unit/Service/EventProcessorTest.php create mode 100644 Test/Unit/Service/Insights/EventProcessorTest.php rename Test/Unit/Service/{ => Insights}/EventProcessorTestable.php (71%) diff --git a/Api/Insights/EventProcessorInterface.php b/Api/Insights/EventProcessorInterface.php index 54e5107e0..91ddc10c7 100644 --- a/Api/Insights/EventProcessorInterface.php +++ b/Api/Insights/EventProcessorInterface.php @@ -43,6 +43,7 @@ public function setAuthenticatedUserToken(string $token): EventProcessorInterfac public function setAnonymousUserToken(string $token): EventProcessorInterface; + /** @deprecated Store Manager now handled as injected dependency */ public function setStoreManager(StoreManagerInterface $storeManager): EventProcessorInterface; /** diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ce1f0d9f..496156077 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,11 +27,14 @@ ## 3.16.1 -### Bug fixes +### Updates +- `EventProcessor` now calculates decimal precision on currency based on `Magento\Framework\Locale\FormatInterface` + +### Bug fixes - Ensure that only non-redirect URL rewrites are considered when generating product URLs - thank you @fasimana +- Apply rounding to insight events revenue values to avoid floating point precision errors - thank you @PromInc - Fix store id for queue jobs sorting - thank you @pikulsky - ## 3.16.0 ### Features diff --git a/Helper/InsightsHelper.php b/Helper/InsightsHelper.php index c79088f66..35cd37d34 100644 --- a/Helper/InsightsHelper.php +++ b/Helper/InsightsHelper.php @@ -91,8 +91,7 @@ public function getEventProcessor(): EventProcessorInterface $this->eventProcessor = $this->eventProcessorFactory->create([ 'client' => $this->getInsightsClient(), 'userToken' => $this->getAnonymousUserToken(), - 'authenticatedUserToken' => $this->getAuthenticatedUserToken(), - 'storeManager' => $this->storeManager + 'authenticatedUserToken' => $this->getAuthenticatedUserToken() ]); } return $this->eventProcessor; diff --git a/Service/Insights/EventProcessor.php b/Service/Insights/EventProcessor.php index 0c2bb2ba1..2ab37b66a 100644 --- a/Service/Insights/EventProcessor.php +++ b/Service/Insights/EventProcessor.php @@ -7,6 +7,8 @@ use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; use Algolia\AlgoliaSearch\Helper\InsightsHelper; use Magento\Framework\Exception\LocalizedException; +use Magento\Framework\Locale\FormatInterface as LocalFormatInterface; +use Magento\Framework\Pricing\PriceCurrencyInterface; use Magento\Quote\Model\Quote\Item; use Magento\Sales\Model\Order; use Magento\Sales\Model\Order\Item as OrderItem; @@ -18,13 +20,24 @@ class EventProcessor implements EventProcessorInterface /** @var string */ protected const NO_QUERY_ID_KEY = '__NO_QUERY_ID__'; + protected int $decimalPrecision; + public function __construct( - protected TaxConfig $taxConfig, - protected ?InsightsClient $client = null, - protected ?string $userToken = null, - protected ?string $authenticatedUserToken = null, - protected ?StoreManagerInterface $storeManager = null - ) {} + protected TaxConfig $taxConfig, + protected StoreManagerInterface $storeManager, + protected LocalFormatInterface $localeFormat, + protected ?InsightsClient $client = null, + protected ?string $userToken = null, + protected ?string $authenticatedUserToken = null, + ) { + $this->initDecimalPrecision(); + } + + protected function initDecimalPrecision(): void + { + $this->decimalPrecision = $this->localeFormat->getPriceFormat()['requiredPrecision'] + ?? PriceCurrencyInterface::DEFAULT_PRECISION; + } public function setInsightsClient(InsightsClient $client): EventProcessorInterface { @@ -135,7 +148,7 @@ public function convertAddToCart(string $eventName, string $indexName, Item $ite $this->checkDependencies(); $price = $this->getQuoteItemSalePrice($item); - $qty = intval($item->getData('qty_to_add')); + $qty = (int) $item->getData('qty_to_add'); $event = [ self::EVENT_KEY_SUBTYPE => self::EVENT_SUBTYPE_CART, @@ -238,10 +251,11 @@ protected function restrictMaxObjectsPerEvent(array $items): array */ protected function getTotalRevenueForEvent(array $objectData): float { - return array_reduce( + $total = array_reduce( $objectData, - fn($carry, $item) => floatval($carry) + floatval($item['quantity']) * floatval($item['price']) + fn($carry, $item) => (float) $carry + (float) $item['quantity'] * (float) $item['price'] ); + return $this->applyPrecision($total); } /** @@ -253,7 +267,7 @@ protected function getTotalRevenueForEvent(array $objectData): float */ protected function getQuoteItemSalePrice(Item $item): float { - return floatval($item->getData('base_price') ?? $item->getPrice()); + return $this->applyPrecision((float) ($item->getData('base_price') ?? $item->getPrice())); } /** @@ -262,7 +276,7 @@ protected function getQuoteItemSalePrice(Item $item): float */ protected function getQuoteItemDiscount(Item $item): float { - return floatval($item->getProduct()->getPrice()) - $this->getQuoteItemSalePrice($item); + return $this->applyPrecision($item->getProduct()->getPrice() - $this->getQuoteItemSalePrice($item)); } /** @@ -271,18 +285,21 @@ protected function getQuoteItemDiscount(Item $item): float */ protected function getOrderItemSalePrice(OrderItem $item): float { - return $this->taxConfig->priceIncludesTax($this->storeManager->getStore()->getId()) ? - floatval($item->getPriceInclTax()) - $this->getOrderItemCartDiscount($item): - floatval($item->getPrice()) - $this->getOrderItemCartDiscount($item); + $value = $this->taxConfig->priceIncludesTax($this->storeManager->getStore()->getId()) ? + (float) $item->getPriceInclTax() - $this->getOrderItemCartDiscount($item): + (float) $item->getPrice() - $this->getOrderItemCartDiscount($item); + return $this->applyPrecision($value); } /** + * Get discount for line item for a single product (qty = 1) which is what Algolia uses + * Line item discount retrieved from Magento for a cart rule is for all products (discount * qty) in the line item * @param OrderItem $item * @return float */ protected function getOrderItemCartDiscount(OrderItem $item): float { - return floatval($item->getDiscountAmount()) / intval($item->getQtyOrdered()); + return $this->applyPrecision((float) $item->getDiscountAmount() / (int) $item->getQtyOrdered()); } /** @@ -292,9 +309,9 @@ protected function getOrderItemCartDiscount(OrderItem $item): float protected function getOrderItemDiscount(OrderItem $item): float { $itemDiscount = $this->taxConfig->priceIncludesTax($this->storeManager->getStore()->getId()) ? - floatval($item->getOriginalPrice()) - floatval($item->getPriceInclTax()) : - floatval($item->getOriginalPrice()) - floatval($item->getPrice()); - return $itemDiscount + $this->getOrderItemCartDiscount($item); + (float) $item->getOriginalPrice() - (float) $item->getPriceInclTax() : + (float) $item->getOriginalPrice() - (float) $item->getPrice(); + return $this->applyPrecision($itemDiscount + $this->getOrderItemCartDiscount($item)); } /** @@ -310,7 +327,7 @@ protected function getObjectDataForPurchase(array $items): array return array_map(fn($item) => [ 'price' => $this->getOrderItemSalePrice($item), 'discount' => max(0, $this->getOrderItemDiscount($item)), - 'quantity' => intval($item->getQtyOrdered()) + 'quantity' => (int) $item->getQtyOrdered() ], $items); } @@ -320,7 +337,7 @@ protected function getObjectDataForPurchase(array $items): array */ protected function getObjectIdsForPurchase(array $items): array { - return array_map(fn($item) => $item->getProduct()->getId(), $items); + return array_map(fn($item) => (string) $item->getProduct()->getId(), $items); } @@ -348,4 +365,21 @@ protected function getItemsByQueryId(Order $order): array return $itemsByQueryId; } + + /** + * A public method is provided to easily override this behavior as needed via plugins + * as different currencies may have different precision requirements + * Default behavior is to rely on the store locale and currency configuration via + * \Magento\Framework\Locale\FormatInterface + * + * e.g. + * Some currencies have rounding rules (e.g., CHF (Swiss Franc) often rounds to 0.05 for cash) + * KWD (Kuwaiti Dinar), BHD (Bahraini Dinar), JOD (Jordanian Dinar) → have 1,000 fils per unit + * JPY (Japanese Yen), KRW (Korean Won) do not use cents at all + * + */ + public function applyPrecision(float $value): float + { + return round($value, $this->decimalPrecision); + } } diff --git a/Test/Unit/Service/EventProcessorTest.php b/Test/Unit/Service/EventProcessorTest.php deleted file mode 100644 index b05ddec05..000000000 --- a/Test/Unit/Service/EventProcessorTest.php +++ /dev/null @@ -1,188 +0,0 @@ -client = $this->createMock(InsightsClient::class); - $this->userToken = 'foo'; - $this->authenticatedUserToken = 'authenticated-foo'; - $this->storeManager = $this->createMock(StoreManagerInterface::class); - - $store = $this->createMock(Store::class); - $store->method('getId')->willReturn(1); - $this->storeManager->method('getStore')->willReturn($store); - $this->taxConfig = $this->createMock(TaxConfig::class); - - $this->eventProcessor = new EventProcessorTestable( - $this->taxConfig, - $this->client, - $this->userToken, - $this->authenticatedUserToken, - $this->storeManager - ); - } - - /** - * @dataProvider orderItemsProvider - */ - public function testObjectDataForPurchase($priceIncludesTax, $orderItemsData, $expectedResult, $expectedTotalRevenue): void - { - $this->taxConfig->method('priceIncludesTax')->willReturn($priceIncludesTax); - - $orderItems = []; - - foreach ($orderItemsData as $orderItemData) { - $orderItem = $this->getMockBuilder(OrderItem::class) - ->disableOriginalConstructor() - ->getMock(); - - foreach ($orderItemData as $method => $value){ - $orderItem->method($method)->willReturn($value); - } - - $orderItems[] = $orderItem; - } - - $object = $this->eventProcessor->getObjectDataForPurchase($orderItems); - $this->assertEquals($expectedResult, $object); - - $totalRevenue = $this->eventProcessor->getTotalRevenueForEvent($object); - $this->assertEquals($expectedTotalRevenue, $totalRevenue); - } - - public static function orderItemsProvider(): array - { - return [ - [ // One item - 'priceIncludesTax' => true, - 'orderItemsData' => [ - [ - 'getPrice' => 32.00, - 'getPriceInclTax' => 32.00, - 'getOriginalPrice' => 32.00, - 'getDiscountAmount' => 0.00, - 'getQtyOrdered' => 1, - ] - ], - 'expectedResult' => [ - [ - 'price' => 32.00, - 'discount' => 0.00, - 'quantity' => 1, - ] - ], - 'expectedTotalRevenue' => 32.00 - ], - [ // One item (tax excluded) - 'priceIncludesTax' => false, - 'orderItemsData' => [ - [ - 'getPrice' => 25.00, - 'getPriceInclTax' => 32.00, - 'getOriginalPrice' => 25.00, - 'getDiscountAmount' => 0.00, - 'getQtyOrdered' => 1, - ] - ], - 'expectedResult' => [ - [ - 'price' => 25.00, - 'discount' => 0.00, - 'quantity' => 1, - ] - ], - 'expectedTotalRevenue' => 25.00 - ], - [ // One item with discount - 'priceIncludesTax' => true, - 'orderItemsData' => [ - [ - 'getPrice' => 32.00, - 'getPriceInclTax' => 32.00, - 'getOriginalPrice' => 32.00, - 'getDiscountAmount' => 7.00, - 'getQtyOrdered' => 1, - ] - ], - 'expectedResult' => [ - [ - 'price' => 25.00, - 'discount' => 7.00, - 'quantity' => 1, - ] - ], - 'expectedTotalRevenue' => 25.00 - ], - [ // One item with discount (tax excluded) - 'priceIncludesTax' => false, - 'orderItemsData' => [ - [ - 'getPrice' => 25.00, - 'getPriceInclTax' => 32.00, - 'getOriginalPrice' => 25.00, - 'getDiscountAmount' => 7.00, - 'getQtyOrdered' => 1, - ] - ], - 'expectedResult' => [ - [ - 'price' => 18.00, - 'discount' => 7.00, - 'quantity' => 1, - ] - ], - 'expectedTotalRevenue' => 18.00 - ], - [ // Two items - 'priceIncludesTax' => true, - 'orderItemsData' => [ - [ - 'getPrice' => 32.00, - 'getPriceInclTax' => 32.00, - 'getOriginalPrice' => 32.00, - 'getDiscountAmount' => 7.00, - 'getQtyOrdered' => 1, - ], - [ - 'getPrice' => 32.00, - 'getPriceInclTax' => 32.00, - 'getOriginalPrice' => 32.00, - 'getDiscountAmount' => 0.00, - 'getQtyOrdered' => 2, - ], - ], - 'expectedResult' => [ - [ - 'price' => 25.00, - 'discount' => 7.00, - 'quantity' => 1, - ], - [ - 'price' => 32.00, - 'discount' => 0.00, - 'quantity' => 2, - ] - ], - 'expectedTotalRevenue' => 89.00 // 25 + 32*2 - ], - ]; - } -} diff --git a/Test/Unit/Service/Insights/EventProcessorTest.php b/Test/Unit/Service/Insights/EventProcessorTest.php new file mode 100644 index 000000000..b47ca3829 --- /dev/null +++ b/Test/Unit/Service/Insights/EventProcessorTest.php @@ -0,0 +1,823 @@ +taxConfig = $this->createMock(TaxConfig::class); + $this->storeManager = $this->createMock(StoreManagerInterface::class); + $this->store = $this->createMock(Store::class); + $this->localeFormat = $this->createMock(LocaleFormatInterface::class); + $this->currency = $this->createMock(Currency::class); + $this->insightsClient = $this->createMock(InsightsClient::class); + $this->eventProcessor = new EventProcessorTestable($this->taxConfig, $this->storeManager, $this->localeFormat); + } + + // Test dependency validation and setup methods + + public function testConvertedObjectIDsAfterSearchThrowsExceptionWhenMissingDependencies(): void + { + $this->expectException(AlgoliaException::class); + $this->expectExceptionMessage("Events model is missing necessary dependencies to function."); + + $this->eventProcessor->convertedObjectIDsAfterSearch( + 'test-event', + 'test-index', + ['1', '2', '3'], + 'query-123' + ); + } + + public function testConvertedObjectIDsAfterSearchThrowsExceptionWhenUserTokenMissing(): void + { + $this->eventProcessor->setInsightsClient($this->insightsClient); + + $this->expectException(AlgoliaException::class); + $this->expectExceptionMessage("Events model is missing necessary dependencies to function."); + + $this->eventProcessor->convertedObjectIDsAfterSearch( + 'test-event', + 'test-index', + ['1', '2', '3'], + 'query-123' + ); + } + + public function testConvertedObjectIDsAfterSearchThrowsExceptionWhenInsightsClientMissing(): void + { + $this->eventProcessor + ->setAnonymousUserToken('user-token'); + + $this->expectException(AlgoliaException::class); + $this->expectExceptionMessage("Events model is missing necessary dependencies to function."); + + $this->eventProcessor->convertedObjectIDsAfterSearch( + 'test-event', + 'test-index', + ['1', '2', '3'], + 'query-123' + ); + } + + // Test convertedObjectIDsAfterSearch + + public function testConvertedObjectIDsAfterSearchWithAllDependencies(): void + { + $this->setupFullyConfiguredEventProcessor(); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $this->assertArrayHasKey('events', $payload); + $this->assertCount(1, $payload['events']); + + $event = $payload['events'][0]; + $this->assertEquals('conversion', $event['eventType']); + $this->assertEquals('test-event', $event['eventName']); + $this->assertEquals('test-index', $event['index']); + $this->assertEquals('user-token', $event['userToken']); + $this->assertEquals(['1', '2', '3'], $event['objectIDs']); + $this->assertEquals('query-123', $event['queryID']); + + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $result = $this->eventProcessor->convertedObjectIDsAfterSearch( + 'test-event', + 'test-index', + ['1', '2', '3'], + 'query-123' + ); + } + + public function testConvertedObjectIDsAfterSearchWithAuthenticatedToken(): void + { + $this->setupFullyConfiguredEventProcessor(); + $this->eventProcessor->setAuthenticatedUserToken('auth-token-123'); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $this->assertEquals('auth-token-123', $event['authenticatedUserToken']); + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertedObjectIDsAfterSearch( + 'test-event', + 'test-index', + ['1'], + 'query-123' + ); + } + + // Test convertedObjectIDs + + public function testConvertedObjectIDs(): void + { + $this->setupFullyConfiguredEventProcessor(); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $this->assertEquals('conversion', $event['eventType']); + $this->assertEquals('test-event', $event['eventName']); + $this->assertEquals('test-index', $event['index']); + $this->assertEquals(['1', '2'], $event['objectIDs']); + $this->assertArrayNotHasKey('queryID', $event); + + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertedObjectIDs( + 'test-event', + 'test-index', + ['1', '2'] + ); + } + + // Test convertAddToCart + + public function testConvertAddToCart(): void + { + $this->setupFullyConfiguredEventProcessor(); + + $product = $this->createMockProduct('123', 100.0); + $item = $this->createMockItem($product, 85.0, 2); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $this->assertEquals('addToCart', $event['eventSubtype']); + $this->assertEquals(['123'], $event['objectIDs']); + $this->assertEquals('USD', $event['currency']); + $this->assertEquals(170.0, $event['value']); // 85 * 2 + $this->assertEquals([['price' => 85.0, 'discount' => 15.0, 'quantity' => 2]], $event['objectData']); + $this->assertEquals('query-456', $event['queryID']); + + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertAddToCart( + 'add-to-cart-event', + 'products-index', + $item, + 'query-456' + ); + } + + public function testConvertAddToCartWithoutQueryID(): void + { + $this->setupFullyConfiguredEventProcessor(); + + $product = $this->createMockProduct('123', 100.0); + $item = $this->createMockItem($product, 80.0, 1); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $this->assertArrayNotHasKey('queryID', $event); + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertAddToCart( + 'add-to-cart-event', + 'products-index', + $item + ); + } + + public function testConvertAddToCartFloatingPointPrecision(): void + { + $this->setupFullyConfiguredEventProcessor(); + + $product = $this->createMockProduct('123', 23.99); + $item = $this->createMockItem($product, 23.93, 1); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $this->assertEquals([['price' => 23.93, 'discount' => .06, 'quantity' => 1]], $event['objectData']); + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertAddToCart( + 'add-to-cart-event', + 'products-index', + $item + ); + } + + public function testConvertAddToCartWithStandardDecimalPrecision(): void + { + $this->setupFullyConfiguredEventProcessor(); + $this->setupCurrencyPrecision(2); + + $product = $this->createMockProduct('123', 23.992); + $item = $this->createMockItem($product, 23.931, 1); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $this->assertEquals([['price' => 23.93, 'discount' => .06, 'quantity' => 1]], $event['objectData']); + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertAddToCart( + 'add-to-cart-event', + 'products-index', + $item + ); + } + + public function testConvertAddToCartWith3PointDecimalPrecision(): void + { + $this->setupFullyConfiguredEventProcessor(); + $this->setupCurrencyPrecision(3); + + $product = $this->createMockProduct('123', 23.992); + $item = $this->createMockItem($product, 23.931, 1); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $this->assertEquals([['price' => 23.931, 'discount' => .061, 'quantity' => 1]], $event['objectData']); + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertAddToCart( + 'add-to-cart-event', + 'products-index', + $item + ); + } + + // Test convertPurchaseForItems + public function testConvertPurchaseForItems(): void + { + $this->setupFullyConfiguredEventProcessor(); + + $items = $this->createOrderItems([ + ['id' => '1', 'price' => 50.0, 'originalPrice' => 60.0, 'cartDiscountAmount' => 10.0, 'qtyOrdered' => 2], + ['id' => '2', 'price' => 30.0, 'originalPrice' => 35.0, 'cartDiscountAmount' => 5.0, 'qtyOrdered' => 1], + ]); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $this->assertEquals('purchase', $event['eventSubtype']); + $this->assertEquals(['1', '2'], $event['objectIDs']); + $this->assertEquals('USD', $event['currency']); + $this->assertEquals(115.0, $event['value']); // (50 * 2 - 10) + (30 - 5) + $this->assertEquals('query-789', $event['queryID']); + + $objectData = $event['objectData']; + $this->assertCount(2, $objectData); + $this->assertEquals(45, $objectData[0]['price']); // 50 - (10/2) + $this->assertEquals(15, $objectData[0]['discount']); // 30 - (5/1) + $this->assertEquals(2, $objectData[0]['quantity']); + + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertPurchaseForItems( + 'purchase-event', + 'products-index', + $items, + 'query-789' + ); + } + + public function testConvertPurchaseForItemsEnforcesObjectLimit(): void + { + $this->setupFullyConfiguredEventProcessor(); + + // Create more items than the limit allows + $itemsData = []; + for ($i = 1; $i <= 25; $i++) { + $itemsData[] = ['id' => (string) $i, 'price' => 10.0, 'originalPrice' => 10.0, 'cartDiscountAmount' => 0.0, 'qtyOrdered' => 1]; + } + $items = $this->createOrderItems($itemsData); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + // Should be limited to MAX_OBJECT_IDS_PER_EVENT (20) + $this->assertCount(EventProcessorInterface::MAX_OBJECT_IDS_PER_EVENT, $event['objectIDs']); + $this->assertCount(EventProcessorInterface::MAX_OBJECT_IDS_PER_EVENT, $event['objectData']); + // But value should include all 25 items + $this->assertEquals(250.0, $event['value']); // 25 * 10 + + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertPurchaseForItems( + 'purchase-event', + 'products-index', + $items + ); + } + + public function testConvertPurchaseForItemsFloatingPointPrecision(): void + { + $this->setupFullyConfiguredEventProcessor(); + + // These additions should trigger floating point precision errors if rounding is not applied + $items = $this->createOrderItems([ + ['id' => '1', 'price' => 10.10, 'originalPrice' => 15.00, 'cartDiscountAmount' => 0, 'qtyOrdered' => 1], + ['id' => '2', 'price' => 33.20, 'originalPrice' => 35.00, 'cartDiscountAmount' => 0, 'qtyOrdered' => 1], + ]); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $this->assertEquals(43.30, $event['value']); // 10.10 + 33.20 + + $objectData = $event['objectData']; + $this->assertEquals(['price' => 10.10, 'discount' => 4.90, 'quantity' => 1], $objectData[0]); // 15.00 - 10.10 + $this->assertEquals(['price' => 33.20, 'discount' => 1.80, 'quantity' => 1], $objectData[1]); // 35.00 - 33.20 + + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertPurchaseForItems( + 'purchase-event', + 'products-index', + $items, + 'query-123' + ); + } + + /** + * The way discounts are recorded in Algolia are per product but Magento is per line item + * This test ensures the expected values are returned and that binary math does not impact the final value + */ + public function testConvertPurchaseForItemsFloatingPointPrecisionWithCartDiscount(): void + { + $this->setupFullyConfiguredEventProcessor(); + + // These additions should trigger floating point precision errors if rounding is not applied + $items = $this->createOrderItems([ + ['id' => '1', 'price' => 10.00, 'originalPrice' => 10.00, 'cartDiscountAmount' => .30, 'qtyOrdered' => 3], + ]); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $this->assertEquals(29.70, $event['value']); // 10.00 * 3 + + $objectData = $event['objectData']; + $this->assertEquals(['price' => 9.90, 'discount' => .10, 'quantity' => 3], $objectData[0]); + + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertPurchaseForItems( + 'purchase-event', + 'products-index', + $items, + 'query-123' + ); + } + + // Test convertPurchase + + public function testConvertPurchaseGroupsByQueryID(): void + { + $this->setupFullyConfiguredEventProcessor(); + + $order = $this->createMock(Order::class); + + $items = [ + $this->createOrderItemWithQueryId('1', 'query-1', 50.0, 1), + $this->createOrderItemWithQueryId('2', 'query-1', 30.0, 1), + $this->createOrderItemWithQueryId('3', 'query-2', 25.0, 2), + $this->createOrderItemWithQueryId('4', null, 15.0, 1), // No query ID + ]; + + $order->method('getAllVisibleItems')->willReturn($items); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + // There should be 3 different events for each query ID scenario: query-1, query-2, and no-query + $events = $payload['events']; + $this->assertCount(3, $events); + $this->assertEquals(80.0, $events[0]['value']); // 50 + 30 + $this->assertEquals(50.0, $events[1]['value']); // 25 * 2 + $this->assertEquals(15.0, $events[2]['value']); // missing query ID should be final event + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $result = $this->eventProcessor->convertPurchase( + 'purchase-event', + 'products-index', + $order + ); + // 3 events in one batch + $this->assertCount(1, $result); + } + + public function testConvertPurchaseHandlesLargeOrders(): void + { + $this->setupFullyConfiguredEventProcessor(); + + $order = $this->createMock(Order::class); + + // Create more events than MAX_EVENTS_PER_REQUEST allows + $items = []; + for ($i = 1; $i <= 1500; $i++) { + $items[] = $this->createOrderItemWithQueryId((string)$i, "query-$i", 10.0, 1); + } + + $order->method('getAllVisibleItems')->willReturn($items); + + // Should be called twice due to chunking (1000 + 500) + $this->insightsClient + ->expects($this->exactly(2)) + ->method('pushEvents') + ->willReturn(['status' => 'ok']); + + $result = $this->eventProcessor->convertPurchase( + 'purchase-event', + 'products-index', + $order + ); + + $this->assertCount(2, $result); // 2 chunks + } + + /** Insights API requires that `objectIDs` be submitted as strings */ + public function testConvertPurchaseUsesStringIds(): void + { + $this->setupFullyConfiguredEventProcessor(); + + // These additions should trigger floating point precision errors if rounding is not applied + $items = $this->createOrderItems([ + ['id' => 10, 'price' => 10.00, 'originalPrice' => 10.00, 'cartDiscountAmount' => 0, 'qtyOrdered' => 1], + ['id' => '20', 'price' => 20.00, 'originalPrice' => 20.00, 'cartDiscountAmount' => 0, 'qtyOrdered' => 1], + ['id' => 30.0, 'price' => 30.00, 'originalPrice' => 30.00, 'cartDiscountAmount' => 0, 'qtyOrdered' => 1], + ]); + + $this->insightsClient + ->expects($this->once()) + ->method('pushEvents') + ->with( + $this->callback(function ($payload) { + $event = $payload['events'][0]; + $objectIds = $event['objectIDs']; + foreach ($objectIds as $objectId) { + $this->assertIsString($objectId); + } + return true; + }), + [] + ) + ->willReturn(['status' => 'ok']); + + $this->eventProcessor->convertPurchaseForItems( + 'purchase-event', + 'products-index', + $items, + 'query-123' + ); + } + + + // Test protected methods + + /** + * @dataProvider orderItemsProvider + */ + public function testObjectDataForPurchase($priceIncludesTax, $orderItemsData, $expectedResult, $expectedTotalRevenue): void + { + $this->setupFullyConfiguredEventProcessor(); + + $this->taxConfig->method('priceIncludesTax')->willReturn($priceIncludesTax); + + $orderItems = []; + + foreach ($orderItemsData as $orderItemData) { + $orderItem = $this->getMockBuilder(OrderItem::class) + ->disableOriginalConstructor() + ->getMock(); + + foreach ($orderItemData as $method => $value){ + $orderItem->method($method)->willReturn($value); + } + + $orderItems[] = $orderItem; + } + + $object = $this->eventProcessor->getObjectDataForPurchase($orderItems); + $this->assertEquals($expectedResult, $object); + + $totalRevenue = $this->eventProcessor->getTotalRevenueForEvent($object); + $this->assertEquals($expectedTotalRevenue, $totalRevenue); + } + + // Data providers + + public static function orderItemsProvider(): array + { + return [ + [ // One item + 'priceIncludesTax' => true, + 'orderItemsData' => [ + [ + 'getPrice' => 32.00, + 'getPriceInclTax' => 32.00, + 'getOriginalPrice' => 32.00, + 'getDiscountAmount' => 0.00, + 'getQtyOrdered' => 1, + ] + ], + 'expectedResult' => [ + [ + 'price' => 32.00, + 'discount' => 0.00, + 'quantity' => 1, + ] + ], + 'expectedTotalRevenue' => 32.00 + ], + [ // One item (tax excluded) + 'priceIncludesTax' => false, + 'orderItemsData' => [ + [ + 'getPrice' => 25.00, + 'getPriceInclTax' => 32.00, + 'getOriginalPrice' => 25.00, + 'getDiscountAmount' => 0.00, + 'getQtyOrdered' => 1, + ] + ], + 'expectedResult' => [ + [ + 'price' => 25.00, + 'discount' => 0.00, + 'quantity' => 1, + ] + ], + 'expectedTotalRevenue' => 25.00 + ], + [ // One item with discount + 'priceIncludesTax' => true, + 'orderItemsData' => [ + [ + 'getPrice' => 32.00, + 'getPriceInclTax' => 32.00, + 'getOriginalPrice' => 32.00, + 'getDiscountAmount' => 7.00, + 'getQtyOrdered' => 1, + ] + ], + 'expectedResult' => [ + [ + 'price' => 25.00, + 'discount' => 7.00, + 'quantity' => 1, + ] + ], + 'expectedTotalRevenue' => 25.00 + ], + [ // One item with discount (tax excluded) + 'priceIncludesTax' => false, + 'orderItemsData' => [ + [ + 'getPrice' => 25.00, + 'getPriceInclTax' => 32.00, + 'getOriginalPrice' => 25.00, + 'getDiscountAmount' => 7.00, + 'getQtyOrdered' => 1, + ] + ], + 'expectedResult' => [ + [ + 'price' => 18.00, + 'discount' => 7.00, + 'quantity' => 1, + ] + ], + 'expectedTotalRevenue' => 18.00 + ], + [ // Two items + 'priceIncludesTax' => true, + 'orderItemsData' => [ + [ + 'getPrice' => 32.00, + 'getPriceInclTax' => 32.00, + 'getOriginalPrice' => 32.00, + 'getDiscountAmount' => 7.00, + 'getQtyOrdered' => 1, + ], + [ + 'getPrice' => 32.00, + 'getPriceInclTax' => 32.00, + 'getOriginalPrice' => 32.00, + 'getDiscountAmount' => 0.00, + 'getQtyOrdered' => 2, + ], + ], + 'expectedResult' => [ + [ + 'price' => 25.00, + 'discount' => 7.00, + 'quantity' => 1, + ], + [ + 'price' => 32.00, + 'discount' => 0.00, + 'quantity' => 2, + ] + ], + 'expectedTotalRevenue' => 89.00 // 25 + 32*2 + ], + ]; + } + + // Helper methods + + protected function setupFullyConfiguredEventProcessor(): void + { + $this->currency->method('getCode')->willReturn('USD'); + $this->store->method('getCurrentCurrency')->willReturn($this->currency); + $this->store->method('getId')->willReturn(1); + $this->storeManager->method('getStore')->willReturn($this->store); + + $this->eventProcessor + ->setInsightsClient($this->insightsClient) + ->setAnonymousUserToken('user-token'); + } + + protected function setupCurrencyPrecision(int $decimalPrecision = \Magento\Framework\Pricing\PriceCurrencyInterface::DEFAULT_PRECISION): void + { + $this->localeFormat->method('getPriceFormat')->willReturn([ + 'requiredPrecision' => $decimalPrecision + ]); + $this->eventProcessor->initDecimalPrecision(); + } + + protected function createMockProduct(string $id, float $price): Product + { + $product = $this->createMock(Product::class); + $product->method('getId')->willReturn($id); + $product->method('getPrice')->willReturn($price); + return $product; + } + + protected function createMockItem(Product $product, float $salePrice, int $qtyToAdd): Item + { + $item = $this->createMock(Item::class); + $item->method('getProduct')->willReturn($product); + $item->method('getData') + ->willReturnMap([ + ['base_price', null, $salePrice], + ['qty_to_add', null, $qtyToAdd] + ]); + $item->method('getPrice')->willReturn($salePrice); + return $item; + } + + protected function createOrderItems(array $itemsData): array + { + $items = []; + foreach ($itemsData as $data) { + $product = $this->createMock(Product::class); + $product->method('getId')->willReturn($data['id']); + + $item = $this->createMock(OrderItem::class); + $item->method('getProduct')->willReturn($product); + $item->method('getPrice')->willReturn($data['price']); + $item->method('getOriginalPrice')->willReturn($data['originalPrice']); + $item->method('getDiscountAmount')->willReturn($data['cartDiscountAmount']); + $item->method('getQtyOrdered')->willReturn($data['qtyOrdered']); + + $items[] = $item; + } + return $items; + } + + protected function createOrderItemWithQueryId(string $id, ?string $queryId, float $price, int $qty): OrderItem + { + $product = $this->createMock(Product::class); + $product->method('getId')->willReturn($id); + + $item = $this->createMock(OrderItem::class); + $item->method('getProduct')->willReturn($product); + $item->method('getPrice')->willReturn($price); + $item->method('getOriginalPrice')->willReturn($price); + $item->method('getDiscountAmount')->willReturn(0.0); + $item->method('getQtyOrdered')->willReturn($qty); + + if ($queryId !== null) { + $item->method('hasData')->with(InsightsHelper::QUOTE_ITEM_QUERY_PARAM)->willReturn(true); + $item->method('getData')->with(InsightsHelper::QUOTE_ITEM_QUERY_PARAM)->willReturn($queryId); + } else { + $item->method('hasData')->with(InsightsHelper::QUOTE_ITEM_QUERY_PARAM)->willReturn(false); + } + + return $item; + } +} diff --git a/Test/Unit/Service/EventProcessorTestable.php b/Test/Unit/Service/Insights/EventProcessorTestable.php similarity index 71% rename from Test/Unit/Service/EventProcessorTestable.php rename to Test/Unit/Service/Insights/EventProcessorTestable.php index 5a1e68493..334cc708c 100644 --- a/Test/Unit/Service/EventProcessorTestable.php +++ b/Test/Unit/Service/Insights/EventProcessorTestable.php @@ -1,6 +1,6 @@ Date: Wed, 1 Oct 2025 14:11:22 +0200 Subject: [PATCH 099/119] MAGE-1420: rollback change on store_id job property --- CHANGELOG.md | 1 - Model/Queue.php | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 496156077..fd9191015 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,6 @@ ### Bug fixes - Ensure that only non-redirect URL rewrites are considered when generating product URLs - thank you @fasimana - Apply rounding to insight events revenue values to avoid floating point precision errors - thank you @PromInc -- Fix store id for queue jobs sorting - thank you @pikulsky ## 3.16.0 diff --git a/Model/Queue.php b/Model/Queue.php index b3891fe0e..9a1a08d7e 100644 --- a/Model/Queue.php +++ b/Model/Queue.php @@ -578,7 +578,7 @@ protected function stackSortedJobs(array $sortedJobs, array $tempSortableJobs, ? SORT_ASC, 'method', SORT_ASC, - 'storeId', + 'store_id', SORT_ASC, 'job_id', SORT_ASC From 1bd8a30681755e206efcb4eb374f717766ca794e Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 1 Oct 2025 22:40:36 -0400 Subject: [PATCH 100/119] MAGE-1383 Add Codacy ignore patterns --- .codacy.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .codacy.yml diff --git a/.codacy.yml b/.codacy.yml new file mode 100644 index 000000000..ffa14d552 --- /dev/null +++ b/.codacy.yml @@ -0,0 +1,5 @@ +--- +exclude_paths: + - ".php-cs-fixer.php" + - "dev/**" + - "Test/**" From 1d78992adb96ab5186ed988601318b71002344ae Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 1 Oct 2025 22:52:47 -0400 Subject: [PATCH 101/119] MAGE-1383 Replace is_null in small sample --- Controller/Adminhtml/Queue/View.php | 2 +- Controller/Adminhtml/QueueArchive/View.php | 2 +- Model/ResourceModel/Query.php | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Controller/Adminhtml/Queue/View.php b/Controller/Adminhtml/Queue/View.php index 3f9a38a65..c85586402 100644 --- a/Controller/Adminhtml/Queue/View.php +++ b/Controller/Adminhtml/Queue/View.php @@ -10,7 +10,7 @@ class View extends AbstractAction public function execute() { $job = $this->initJob(); - if (is_null($job)) { + if ($job === null) { $this->messageManager->addErrorMessage(__('This job does not exist.')); /** @var \Magento\Backend\Model\View\Result\Redirect $resultRedirect */ $resultRedirect = $this->resultFactory->create(ResultFactory::TYPE_REDIRECT); diff --git a/Controller/Adminhtml/QueueArchive/View.php b/Controller/Adminhtml/QueueArchive/View.php index 7672d9bf9..5704315eb 100644 --- a/Controller/Adminhtml/QueueArchive/View.php +++ b/Controller/Adminhtml/QueueArchive/View.php @@ -12,7 +12,7 @@ class View extends AbstractAction public function execute() { $job = $this->initJob(); - if (is_null($job)) { + if ($job === null) { $this->messageManager->addErrorMessage(__('This job does not exist.')); /** @var Redirect $resultRedirect */ $resultRedirect = $this->resultFactory->create(ResultFactory::TYPE_REDIRECT); diff --git a/Model/ResourceModel/Query.php b/Model/ResourceModel/Query.php index dee152967..b63a3f6e9 100644 --- a/Model/ResourceModel/Query.php +++ b/Model/ResourceModel/Query.php @@ -29,12 +29,12 @@ public function checkQueryUnicity($queryText, $storeId = null, $queryId = null) ->where('query_text = ?', $queryText); // Only check a particular store if specified - if (!is_null($storeId) && $storeId != 0) { + if ($storeId !== null && $storeId != 0) { $select->where('store_id = ?', $storeId); } // Handle the already existing query text for the query - if (!is_null($queryId) && $queryId != 0) { + if ($queryId !== null && $queryId != 0) { $select->where('query_id != ?', $queryId); } From 1587efd11e175330af40cdd5e8050e7db012c86f Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 1 Oct 2025 23:11:02 -0400 Subject: [PATCH 102/119] MAGE-1383 Test nullsafe operator with Codacy --- .../Indexing/Product/Traits/ReplicaAssertionsTrait.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Test/Integration/Indexing/Product/Traits/ReplicaAssertionsTrait.php b/Test/Integration/Indexing/Product/Traits/ReplicaAssertionsTrait.php index 3d4dcd1fe..a6f731f1a 100644 --- a/Test/Integration/Indexing/Product/Traits/ReplicaAssertionsTrait.php +++ b/Test/Integration/Indexing/Product/Traits/ReplicaAssertionsTrait.php @@ -130,7 +130,7 @@ protected function assertNoSortingAttribute($sortAttr, $sortDir): void */ protected function mockSortUpdate(string $sortAttr, string $sortDir, array $attr, ?StoreInterface $store = null): void { - $sorting = $this->configHelper->getSorting(!is_null($store) ? $store->getId() : null); + $sorting = $this->configHelper->getSorting($store?->getId()); $existing = array_filter($sorting, fn($item) => $item['attribute'] === $sortAttr && $item['sort'] === $sortDir); if ($existing) { From fe10b049ef743e43fdb12552b4cffbc2359f26c9 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 1 Oct 2025 23:28:01 -0400 Subject: [PATCH 103/119] MAGE-1383 Refactor derived currency --- Helper/Data.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Helper/Data.php b/Helper/Data.php index bf1d9fa16..4193b16aa 100755 --- a/Helper/Data.php +++ b/Helper/Data.php @@ -239,9 +239,9 @@ public function getIndexDataByStoreIds(): array */ protected function buildIndexData(?StoreInterface $store = null): array { - $storeId = !is_null($store) ? $store->getStoreId() : null; - $currencyCode = !is_null($store) ? - $store->getCurrentCurrencyCode($storeId) : + $storeId = $store?->getStoreId(); + $currencyCode = + $store?->getCurrentCurrencyCode($storeId) ?? $this->configHelper->getCurrencyCode(); return [ @@ -250,7 +250,7 @@ protected function buildIndexData(?StoreInterface $store = null): array 'indexName' => $this->getBaseIndexName($storeId), 'priceKey' => '.' . $currencyCode . '.default', 'facets' => $this->configHelper->getFacets($storeId), - 'currencyCode' => $this->configHelper->getCurrencyCode($storeId), + 'currencyCode' => $currencyCode, 'maxValuesPerFacet' => (int) $this->configHelper->getMaxValuesPerFacet($storeId), 'categorySeparator' => $this->configHelper->getCategorySeparator($storeId), ]; From 20d3306a8bb9135d4fc0efd92af0b165eeaca827 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Thu, 2 Oct 2025 00:06:40 -0400 Subject: [PATCH 104/119] MAGE-1383 Remove all is_null calls --- Block/Adminhtml/Queue/Status.php | 2 +- .../Replica/ReplicaDisableVirtualCommand.php | 2 +- Controller/Adminhtml/IndexingManager/Reindex.php | 2 +- Controller/Adminhtml/Landingpage/Duplicate.php | 2 +- Controller/Adminhtml/Landingpage/Edit.php | 2 +- Controller/Adminhtml/Query/Edit.php | 2 +- Controller/Adminhtml/Query/Save.php | 4 ++-- Helper/Adapter/FiltersHelper.php | 8 ++++---- Helper/MerchandisingHelper.php | 4 ++-- Model/Job.php | 2 +- .../CatalogPermissions/CategoryPermissions.php | 4 ++-- .../CatalogPermissions/ProductPermissions.php | 4 ++-- Model/ResourceModel/LandingPage.php | 4 ++-- Service/AlgoliaConnector.php | 16 ++++++++-------- Service/Category/IndexBuilder.php | 2 +- Service/IndexOptionsBuilder.php | 4 ++-- Service/Product/ReplicaManager.php | 4 ++-- Service/Product/SortingTransformer.php | 4 ++-- .../Product/Traits/ReplicaAssertionsTrait.php | 2 +- 19 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Block/Adminhtml/Queue/Status.php b/Block/Adminhtml/Queue/Status.php index 10bd9c400..9c877ff13 100644 --- a/Block/Adminhtml/Queue/Status.php +++ b/Block/Adminhtml/Queue/Status.php @@ -158,7 +158,7 @@ private function isQueueFast() { $averageProcessingTime = $this->queue->getAverageProcessingTime(); - return !is_null($averageProcessingTime) && $averageProcessingTime < self::QUEUE_FAST_LIMIT; + return $averageProcessingTime !== null && $averageProcessingTime < self::QUEUE_FAST_LIMIT; } /** @return int */ diff --git a/Console/Command/Replica/ReplicaDisableVirtualCommand.php b/Console/Command/Replica/ReplicaDisableVirtualCommand.php index 2b5a7bac8..d0fe50a55 100644 --- a/Console/Command/Replica/ReplicaDisableVirtualCommand.php +++ b/Console/Command/Replica/ReplicaDisableVirtualCommand.php @@ -151,7 +151,7 @@ protected function disableVirtualReplicasForAllStores(): void public function removeLegacyVirtualReplicaConfig(string $scope = ScopeConfigInterface::SCOPE_TYPE_DEFAULT, int $scopeId = 0): void { $value = $this->scopeConfig->getValue(ConfigHelper::LEGACY_USE_VIRTUAL_REPLICA_ENABLED, $scope, $scopeId); - if (is_null($value)) { + if ($value == null) { return; } $this->output->writeln("Removing legacy configuration " . ConfigHelper::LEGACY_USE_VIRTUAL_REPLICA_ENABLED . " for $scope scope" . ($scope != ScopeConfigInterface::SCOPE_TYPE_DEFAULT ? " (ID=$scopeId)" : "") . ""); diff --git a/Controller/Adminhtml/IndexingManager/Reindex.php b/Controller/Adminhtml/IndexingManager/Reindex.php index f3214593d..060132c67 100644 --- a/Controller/Adminhtml/IndexingManager/Reindex.php +++ b/Controller/Adminhtml/IndexingManager/Reindex.php @@ -149,7 +149,7 @@ protected function reindexEntities(array $entities, ?array $storeIds = null, ?ar $message = $this->storeNameFetcher->getStoreName($storeId) . " "; $message .= "(" . $this->indexNameFetcher->getIndexName('_' . $entity, $storeId); - if (!is_null($entityIds)) { + if ($entityIds !== null) { $recordLabel = count($entityIds) > 1 ? "records" : "record"; $message .= " - " . count($entityIds) . " " . $recordLabel; } else { diff --git a/Controller/Adminhtml/Landingpage/Duplicate.php b/Controller/Adminhtml/Landingpage/Duplicate.php index 08c88a58a..fc39ece07 100644 --- a/Controller/Adminhtml/Landingpage/Duplicate.php +++ b/Controller/Adminhtml/Landingpage/Duplicate.php @@ -26,7 +26,7 @@ public function execute() $landingPage = $this->landingPageFactory->create(); $landingPage->getResource()->load($landingPage, $landingPageId); - if (is_null($landingPage)) { + if ($landingPage === null) { $this->messageManager->addErrorMessage(__('This landing page does not exists.')); return $resultRedirect->setPath('*/*/'); diff --git a/Controller/Adminhtml/Landingpage/Edit.php b/Controller/Adminhtml/Landingpage/Edit.php index a7f646381..e69616517 100644 --- a/Controller/Adminhtml/Landingpage/Edit.php +++ b/Controller/Adminhtml/Landingpage/Edit.php @@ -10,7 +10,7 @@ class Edit extends AbstractAction public function execute() { $landingPage = $this->initLandingPage(); - if (is_null($landingPage)) { + if ($landingPage === null) { $this->messageManager->addErrorMessage(__('This landing page does not exists.')); /** @var \Magento\Backend\Model\View\Result\Redirect $resultRedirect */ $resultRedirect = $this->resultFactory->create(ResultFactory::TYPE_REDIRECT); diff --git a/Controller/Adminhtml/Query/Edit.php b/Controller/Adminhtml/Query/Edit.php index 4b09a356c..d8088433e 100644 --- a/Controller/Adminhtml/Query/Edit.php +++ b/Controller/Adminhtml/Query/Edit.php @@ -10,7 +10,7 @@ class Edit extends AbstractAction public function execute() { $query = $this->initQuery(); - if (is_null($query)) { + if ($query === null) { $this->messageManager->addErrorMessage(__('This query does not exists.')); /** @var \Magento\Backend\Model\View\Result\Redirect $resultRedirect */ $resultRedirect = $this->resultFactory->create(ResultFactory::TYPE_REDIRECT); diff --git a/Controller/Adminhtml/Query/Save.php b/Controller/Adminhtml/Query/Save.php index d62dc6ccc..f67bdb395 100755 --- a/Controller/Adminhtml/Query/Save.php +++ b/Controller/Adminhtml/Query/Save.php @@ -120,7 +120,7 @@ public function execute() $query->getResource()->save($query); if (isset($data['algolia_merchandising_positions']) && $data['algolia_merchandising_positions'] != '' - || !is_null($data['banner_image'])) { + || $data['banner_image'] !== null) { $this->manageQueryRules($query->getId(), $data); } @@ -168,7 +168,7 @@ private function manageQueryRules(int $queryId, array $data): void $bannerContent = $this->prepareBannerContent($data); foreach ($stores as $storeId) { - if (!$positions && is_null($bannerContent)) { + if (!$positions && $bannerContent === null) { $this->merchandisingHelper->deleteQueryRule( $storeId, $queryId, diff --git a/Helper/Adapter/FiltersHelper.php b/Helper/Adapter/FiltersHelper.php index 66d034dd6..f69c538a7 100644 --- a/Helper/Adapter/FiltersHelper.php +++ b/Helper/Adapter/FiltersHelper.php @@ -59,7 +59,7 @@ public function getRequest() public function getPaginationFilters() { $paginationFilter = []; - $page = !is_null($this->request->getParam('page')) ? + $page = $this->request->getParam('page') !== null ? (int) $this->request->getParam('page') - 1 : 0; $paginationFilter['page'] = $page; @@ -84,7 +84,7 @@ public function getCategoryFilters($storeId) $categoryId = $category->getEntityId(); } - if (!is_null($categoryId)) { + if ($categoryId !== null) { $categoryFilter['facetFilters'][] = 'categoryIds:' . $categoryId; } @@ -234,13 +234,13 @@ public function getPriceFilters($storeId) $prices = []; // Instantsearch price facet compatibility - if (!is_null($this->request->getParam($paramPriceSlider))) { + if ($this->request->getParam($paramPriceSlider) !== null) { $pricesFilter = $this->request->getParam($paramPriceSlider); $prices = explode(':', $pricesFilter); } // Native Magento price facet compatibility - if (!$this->config->isInstantEnabled($storeId) && !is_null($this->request->getParam('price'))) { + if (!$this->config->isInstantEnabled($storeId) && $this->request->getParam('price') !== null) { $pricesFilter = $this->request->getParam('price'); $prices = explode('-', $pricesFilter); } diff --git a/Helper/MerchandisingHelper.php b/Helper/MerchandisingHelper.php index e8bd00882..a31f14e58 100755 --- a/Helper/MerchandisingHelper.php +++ b/Helper/MerchandisingHelper.php @@ -53,11 +53,11 @@ public function saveQueryRule(int $storeId, ], ]; - if (!is_null($query) && $query != '') { + if ($query !== null && $query != '') { $condition['pattern'] = $query; } - if (!is_null($banner)) { + if ($banner !== null) { $rule['consequence']['userData']['banner'] = $banner; } diff --git a/Model/Job.php b/Model/Job.php index 8661dc56f..69205ab38 100644 --- a/Model/Job.php +++ b/Model/Job.php @@ -183,7 +183,7 @@ public function getStatus(): string { $status = JobInterface::STATUS_PROCESSING; - if (is_null($this->getPid())) { + if ($this->getPid() === null) { $status = JobInterface::STATUS_NEW; } diff --git a/Model/Observer/CatalogPermissions/CategoryPermissions.php b/Model/Observer/CatalogPermissions/CategoryPermissions.php index 01f1b3c82..78629b579 100644 --- a/Model/Observer/CatalogPermissions/CategoryPermissions.php +++ b/Model/Observer/CatalogPermissions/CategoryPermissions.php @@ -39,9 +39,9 @@ public function execute(Observer $observer) $customerGroupId = $customerGroup->getCustomerGroupId(); $isVisible = (int) $this->permissionsFactory->getCatalogPermissionsHelper()->isAllowedCategoryView($storeId, $customerGroupId); - if (!is_null($category->getData('shared_catalog_permission_' . $customerGroupId))) { + if ($category->getData('shared_catalog_permission_' . $customerGroupId) !== null) { $isVisible = (int) $category->getData('shared_catalog_permission_' . $customerGroupId); - } elseif (!is_null($category->getData('customer_group_permission_' . $customerGroupId))) { + } elseif ($category->getData('customer_group_permission_' . $customerGroupId) !== null) { $isVisible = (int) $category->getData('customer_group_permission_' . $customerGroupId); } diff --git a/Model/Observer/CatalogPermissions/ProductPermissions.php b/Model/Observer/CatalogPermissions/ProductPermissions.php index 799218ff6..40f27f56f 100644 --- a/Model/Observer/CatalogPermissions/ProductPermissions.php +++ b/Model/Observer/CatalogPermissions/ProductPermissions.php @@ -39,9 +39,9 @@ public function execute(Observer $observer) $customerGroupId = $customerGroup->getCustomerGroupId(); $isVisible = (int) $this->permissionsFactory->getCatalogPermissionsHelper()->isAllowedCategoryView($storeId, $customerGroupId); - if (!is_null($product->getData('shared_catalog_permission_' . $customerGroupId))) { + if ($product->getData('shared_catalog_permission_' . $customerGroupId) !== null) { $isVisible = (int) $product->getData('shared_catalog_permission_' . $customerGroupId); - } elseif (!is_null($product->getData('customer_group_permission_' . $customerGroupId))) { + } elseif ($product->getData('customer_group_permission_' . $customerGroupId) !== null) { $isVisible = (int) $product->getData('customer_group_permission_' . $customerGroupId); } diff --git a/Model/ResourceModel/LandingPage.php b/Model/ResourceModel/LandingPage.php index aba5e357c..898557bcb 100644 --- a/Model/ResourceModel/LandingPage.php +++ b/Model/ResourceModel/LandingPage.php @@ -136,12 +136,12 @@ public function checkUrlRewriteTable($urlKey, $storeId = null, $landingPageId = ->where('request_path = ?', $urlKey); // Only check a particular store if specified - if (!is_null($storeId) && $storeId != 0) { + if ($storeId) { $select->where('store_id = ?', $storeId); } // Handle the already existing url rewrite for the landing page - if (!is_null($landingPageId) && $landingPageId != 0) { + if ($landingPageId) { $select->where('!(entity_type = "landing-page" AND entity_id = ?)', $landingPageId); } diff --git a/Service/AlgoliaConnector.php b/Service/AlgoliaConnector.php index cce5c48cc..5b5e46c3c 100755 --- a/Service/AlgoliaConnector.php +++ b/Service/AlgoliaConnector.php @@ -135,7 +135,7 @@ protected function addAlgoliaUserAgent(int $storeId = self::ALGOLIA_DEFAULT_SCOP */ public function getClient(?int $storeId = self::ALGOLIA_DEFAULT_SCOPE): SearchClient { - if (is_null($storeId)) { + if ($storeId === null) { $storeId = self::ALGOLIA_DEFAULT_SCOPE; } @@ -477,7 +477,7 @@ protected function setLastOperationInfo(IndexOptionsInterface $indexOptions, arr $this->lastUsedIndexName = $indexName; $this->lastTaskId = $response[self::ALGOLIA_API_TASK_ID] ?? null; - if (!is_null($storeId)) { + if ($storeId !== null) { $this->lastTaskInfoByStore[$storeId] = [ 'indexName' => $indexName, 'taskId' => $response[self::ALGOLIA_API_TASK_ID] ?? null @@ -642,16 +642,16 @@ public function clearIndex(IndexOptionsInterface $indexOptions): void */ public function waitLastTask(?int $storeId = null, ?string $lastUsedIndexName = null, ?int $lastTaskId = null): void { - if (is_null($lastUsedIndexName)) { - if (!is_null($storeId) && isset($this->lastTaskInfoByStore[$storeId])) { + if ($lastUsedIndexName === null) { + if ($storeId !== null && isset($this->lastTaskInfoByStore[$storeId])) { $lastUsedIndexName = $this->lastTaskInfoByStore[$storeId]['indexName']; } elseif (isset($this->lastUsedIndexName)){ $lastUsedIndexName = $this->lastUsedIndexName; } } - if (is_null($lastTaskId)) { - if (!is_null($storeId) && isset($this->lastTaskInfoByStore[$storeId])) { + if ($lastTaskId === null) { + if ($storeId !== null && isset($this->lastTaskInfoByStore[$storeId])) { $lastTaskId = $this->lastTaskInfoByStore[$storeId]['taskId']; } elseif (isset($this->lastTaskId)){ $lastTaskId = $this->lastTaskId; @@ -662,7 +662,7 @@ public function waitLastTask(?int $storeId = null, ?string $lastUsedIndexName = return; } - if (is_null($storeId)) { + if ($storeId === null) { $storeId = self::ALGOLIA_DEFAULT_SCOPE; } @@ -894,7 +894,7 @@ public function getLastTaskId(?int $storeId = null): int|null { $lastTaskId = null; - if (!is_null($storeId) && isset($this->lastTaskInfoByStore[$storeId])) { + if ($storeId !== null && isset($this->lastTaskInfoByStore[$storeId])) { $lastTaskId = $this->lastTaskInfoByStore[$storeId]['taskId']; } elseif (isset($this->lastTaskId)){ $lastTaskId = $this->lastTaskId; diff --git a/Service/Category/IndexBuilder.php b/Service/Category/IndexBuilder.php index 66c8a5bd3..444342379 100644 --- a/Service/Category/IndexBuilder.php +++ b/Service/Category/IndexBuilder.php @@ -81,7 +81,7 @@ public function buildIndex(int $storeId, ?array $entityIds, ?array $options): vo return; } - if (!is_null($entityIds)) { + if ($entityIds !== null) { $this->rebuildEntityIds($storeId, $entityIds); return; } diff --git a/Service/IndexOptionsBuilder.php b/Service/IndexOptionsBuilder.php index 93c145807..06ccbd784 100644 --- a/Service/IndexOptionsBuilder.php +++ b/Service/IndexOptionsBuilder.php @@ -82,11 +82,11 @@ protected function build(IndexOptionsInterface $indexOptions): IndexOptionsInter */ protected function computeIndexName(IndexOptionsInterface $indexOptions): ?string { - if (!is_null($indexOptions->getIndexName())) { + if ($indexOptions->getIndexName() !== null) { return $indexOptions->getIndexName(); } - if (is_null($indexOptions->getIndexSuffix())) { + if ($indexOptions->getIndexSuffix() === null) { throw new AlgoliaException('Index suffix is mandatory in case no enforced index name is specified.'); } diff --git a/Service/Product/ReplicaManager.php b/Service/Product/ReplicaManager.php index 6121de3b5..e5ddb9ae1 100644 --- a/Service/Product/ReplicaManager.php +++ b/Service/Product/ReplicaManager.php @@ -126,7 +126,7 @@ protected function getReplicaConfigurationFromAlgolia( protected function clearAlgoliaReplicaSettingCache($primaryIndexName = null): void { - if (is_null($primaryIndexName)) { + if ($primaryIndexName === null) { $this->_algoliaReplicaConfig = []; } else { unset($this->_algoliaReplicaConfig[$primaryIndexName]); @@ -565,7 +565,7 @@ public function getUnusedReplicaIndices(int $storeId): array protected function clearUnusedReplicaIndicesCache(?int $storeId = null): void { - if (is_null($storeId)) { + if ($storeId === null) { $this->_unusedReplicaIndices = []; } else { unset($this->_unusedReplicaIndices[$storeId]); diff --git a/Service/Product/SortingTransformer.php b/Service/Product/SortingTransformer.php index 746f1dba7..718f44633 100644 --- a/Service/Product/SortingTransformer.php +++ b/Service/Product/SortingTransformer.php @@ -51,7 +51,7 @@ public function getSortingIndices( ): array { // Selectively cache this result - only cache manipulation of saved settings per store - $useCache = is_null($currentCustomerGroupId) && is_null($attrs); + $useCache = $currentCustomerGroupId === null && $attrs === null; if ($clearCache) { unset($this->_sortingIndices[$storeId]); @@ -78,7 +78,7 @@ public function getSortingIndices( if ($this->configHelper->isCustomerGroupsEnabled($storeId) && $attr[ReplicaManagerInterface::SORT_KEY_ATTRIBUTE_NAME] === ReplicaManagerInterface::SORT_ATTRIBUTE_PRICE) { $websiteId = $this->storeManager->getStore($storeId)->getWebsiteId(); $groupCollection = $this->groupCollection; - if (!is_null($currentCustomerGroupId)) { + if ($currentCustomerGroupId !== null) { $groupCollection->addFilter('customer_group_id', $currentCustomerGroupId); } foreach ($groupCollection as $group) { diff --git a/Test/Integration/Indexing/Product/Traits/ReplicaAssertionsTrait.php b/Test/Integration/Indexing/Product/Traits/ReplicaAssertionsTrait.php index a6f731f1a..85f5d6faa 100644 --- a/Test/Integration/Indexing/Product/Traits/ReplicaAssertionsTrait.php +++ b/Test/Integration/Indexing/Product/Traits/ReplicaAssertionsTrait.php @@ -150,7 +150,7 @@ protected function mockSortUpdate(string $sortAttr, string $sortDir, array $attr $this->setConfig( ConfigHelper::SORTING_INDICES, json_encode($sorting), - !is_null($store) ? $store->getCode() : 'default' + $store?->getCode() ?? 'default' ); } } From 7cf431572fa688ce18e2f6ef1351b6d8d2bdba36 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Thu, 2 Oct 2025 13:02:29 -0400 Subject: [PATCH 105/119] MAGE-1383 Fix typo for strict comparison Co-authored-by: Damien Couchez --- Console/Command/Replica/ReplicaDisableVirtualCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Console/Command/Replica/ReplicaDisableVirtualCommand.php b/Console/Command/Replica/ReplicaDisableVirtualCommand.php index d0fe50a55..35971a226 100644 --- a/Console/Command/Replica/ReplicaDisableVirtualCommand.php +++ b/Console/Command/Replica/ReplicaDisableVirtualCommand.php @@ -151,7 +151,7 @@ protected function disableVirtualReplicasForAllStores(): void public function removeLegacyVirtualReplicaConfig(string $scope = ScopeConfigInterface::SCOPE_TYPE_DEFAULT, int $scopeId = 0): void { $value = $this->scopeConfig->getValue(ConfigHelper::LEGACY_USE_VIRTUAL_REPLICA_ENABLED, $scope, $scopeId); - if ($value == null) { + if ($value === null) { return; } $this->output->writeln("Removing legacy configuration " . ConfigHelper::LEGACY_USE_VIRTUAL_REPLICA_ENABLED . " for $scope scope" . ($scope != ScopeConfigInterface::SCOPE_TYPE_DEFAULT ? " (ID=$scopeId)" : "") . ""); From a9cea943f9615fd233e627223c387d528886f365 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 8 Oct 2025 11:26:33 +0200 Subject: [PATCH 106/119] MAGE-1441: update changelog for 3.17.0-beta.2 --- CHANGELOG.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd9191015..87d518c92 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,12 @@ ## 3.17.0-beta.2 -### Bug fixes +### Updates +- Removed all `is_null` occurrences +### Bug fixes - Fixed 3.17 setup:upgrade on PHP 8.4 +- Fixed many Codacy issues ## 3.17.0-beta.1 @@ -28,11 +31,19 @@ ## 3.16.1 ### Updates +- Add checks on configuration migration processed on data patch - `EventProcessor` now calculates decimal precision on currency based on `Magento\Framework\Locale\FormatInterface` +- Updated various unit/integration tests -### Bug fixes +### Bug fixes +- Fixed Indexing Queue display in backend templates. +- Fixed Indexing Queue merging mechanism, it should now have way better performances with delta indexing (updates) jobs. +- Fixed implicit nullable types for PHP 8.4 - Ensure that only non-redirect URL rewrites are considered when generating product URLs - thank you @fasimana - Apply rounding to insight events revenue values to avoid floating point precision errors - thank you @PromInc +- Fix issue where double conversion occurred during price indexing in case of multi currency stores - thank you @natedawg92 +- Fix issue where non-clickable links where rendered on InstantSearch - thank you @PromInc +- Fixed some Codacy issues ## 3.16.0 From 760f400c6ea35dc96e0773cedc162de343e6a758 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 8 Oct 2025 17:09:02 +0200 Subject: [PATCH 107/119] MAGE-1441: update changelog for 3.17.0-beta.2 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87d518c92..1bd36ef19 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ ### Updates - Add checks on configuration migration processed on data patch - `EventProcessor` now calculates decimal precision on currency based on `Magento\Framework\Locale\FormatInterface` +- Local InstantSearch widgets can now override facet sorting behavior in merchandising rules - Updated various unit/integration tests ### Bug fixes @@ -43,6 +44,7 @@ - Apply rounding to insight events revenue values to avoid floating point precision errors - thank you @PromInc - Fix issue where double conversion occurred during price indexing in case of multi currency stores - thank you @natedawg92 - Fix issue where non-clickable links where rendered on InstantSearch - thank you @PromInc +- Fixed issue where replica setting forwarding can abort if async operation fails to complete in time - Fixed some Codacy issues ## 3.16.0 From a4b2ab8d234039ef22139a85837954b102977016 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Fri, 17 Oct 2025 11:12:32 +0200 Subject: [PATCH 108/119] MAGE-1446: fix product duplication --- Plugin/Cache/CacheCleanProductPlugin.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Plugin/Cache/CacheCleanProductPlugin.php b/Plugin/Cache/CacheCleanProductPlugin.php index 6e5b77777..ef3d9a025 100644 --- a/Plugin/Cache/CacheCleanProductPlugin.php +++ b/Plugin/Cache/CacheCleanProductPlugin.php @@ -29,6 +29,12 @@ public function beforeSave(ProductResource $subject, Product $product): void public function afterSave(ProductResource $subject, ProductResource $result, Product $product): ProductResource { $original = $this->originalData[$product->getSku()] ?? []; + + // In case of a product duplication + if (empty($original)) { + return $result; + } + $storeId = $product->getStoreId(); $shouldClearCache = From 20a5883938ef97e914acfe85ceede63aa0290104 Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Wed, 22 Oct 2025 17:37:58 -0400 Subject: [PATCH 109/119] MAGE-1455 Handle missing price keys gracefully --- .../web/js/template/autocomplete/products.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/view/frontend/web/js/template/autocomplete/products.js b/view/frontend/web/js/template/autocomplete/products.js index 31a47d295..7ffd6e8e3 100644 --- a/view/frontend/web/js/template/autocomplete/products.js +++ b/view/frontend/web/js/template/autocomplete/products.js @@ -45,8 +45,8 @@ define([], function () { getColorHtml: function(item, components, html) { const highlight = this.safeHighlight(components, item, "color"); - - return highlight + + return highlight ? html`color: ${highlight}` : ""; }, @@ -54,31 +54,31 @@ define([], function () { getCategoriesHtml: function(item, components, html) { const highlight = this.safeHighlight(components, item, "categories_without_path", false); - return highlight + return highlight ? html`in ${highlight}` : ""; }, getOriginalPriceHtml: (item, html, priceGroup) => { - if (item['price'][algoliaConfig.currencyCode][priceGroup + '_original_formated'] == null) return ""; + if (item['price']?.[algoliaConfig.currencyCode]?.[priceGroup + '_original_formated'] == null) return ""; return html` ${item['price'][algoliaConfig.currencyCode][priceGroup + '_original_formated']} `; }, getTierPriceHtml: (item, html, priceGroup) => { - if (item['price'][algoliaConfig.currencyCode][priceGroup + '_tier_formated'] == null) return ""; + if (item['price']?.[algoliaConfig.currencyCode]?.[priceGroup + '_tier_formated'] == null) return ""; return html` As low as ${item['price'][algoliaConfig.currencyCode][priceGroup + '_tier_formated']}`; }, getPricingHtml: function(item, html) { - if (item['price'] == undefined) return ""; + if (item['price'] == null) return ""; const priceGroup = algoliaConfig.priceGroup || 'default'; return html `
- - ${item['price'][algoliaConfig.currencyCode][priceGroup + '_formated']} + + ${item['price']?.[algoliaConfig.currencyCode]?.[priceGroup + '_formated']} ${this.getOriginalPriceHtml(item, html, priceGroup)} @@ -87,7 +87,7 @@ define([], function () { }, getFooterSearchCategoryLinks: (html, resultDetails) => { - if (resultDetails.allCategories == undefined || resultDetails.allCategories.length === 0) return ""; + if (resultDetails.allCategories == null || resultDetails.allCategories.length === 0) return ""; return html ` ${algoliaConfig.translations.orIn} ${resultDetails.allCategories.map((list, index) => From bb9dca77a9fa708c9d014f1f7e6c6c94ae000b4f Mon Sep 17 00:00:00 2001 From: Eric Wright Date: Fri, 24 Oct 2025 23:57:36 -0400 Subject: [PATCH 110/119] MAGE-1455 Update change log --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bd36ef19..8c8a16f7d 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGE LOG +## 3.17.0-dev + +### Bug fixes +- Fixed issue where missing pricing keys were not handled gracefully in the Autocomplete product template + ## 3.17.0-beta.2 ### Updates From 4f7d270ee09ebe8570dc49a229c959e0308f40a6 Mon Sep 17 00:00:00 2001 From: Benjamin Volle Date: Thu, 30 Oct 2025 17:55:02 +0100 Subject: [PATCH 111/119] MAGE-1453 FIX : wrong test for category (#1848) Function \Algolia\AlgoliaSearch\Block\Algolia::getCurrentCategory always return a category (see \Algolia\AlgoliaSearch\Registry\CurrentCategory::get), so we have to check is the returned category has an ID. --- Block/Configuration.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Block/Configuration.php b/Block/Configuration.php index f788e5473..8d8be8b1d 100755 --- a/Block/Configuration.php +++ b/Block/Configuration.php @@ -27,7 +27,7 @@ public function isSearchPage(): bool if ($this->getConfigHelper()->replaceCategories() && $request->getControllerName() === 'category') { $category = $this->getCurrentCategory(); - if ($category && $category->getDisplayMode() !== 'PAGE') { + if ($category->getId() && $category->getDisplayMode() !== 'PAGE') { return true; } } @@ -125,7 +125,7 @@ public function getConfiguration() && $request->getControllerName() === 'category') { $category = $this->getCurrentCategory(); - if ($category && $category->getDisplayMode() !== 'PAGE') { + if ($category->getId() && $category->getDisplayMode() !== 'PAGE') { $category->getUrlInstance()->setStore($this->getStoreId()); if (self::IS_CATEGORY_NAVIGATION_ENABLED) { $childCategories = $this->getChildCategoryUrls($category); From 260d13074016586ae1e3cf7a6b24c54f1274987c Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Fri, 31 Oct 2025 11:09:14 +0100 Subject: [PATCH 112/119] MAGE-1453: add unit tests --- Test/Unit/Block/ConfigurationTest.php | 172 ++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 Test/Unit/Block/ConfigurationTest.php diff --git a/Test/Unit/Block/ConfigurationTest.php b/Test/Unit/Block/ConfigurationTest.php new file mode 100644 index 000000000..7d6a057e5 --- /dev/null +++ b/Test/Unit/Block/ConfigurationTest.php @@ -0,0 +1,172 @@ +config = $this->createMock(ConfigHelper::class); + $this->autocompleteConfig = $this->createMock(AutocompleteHelper::class); + $this->instantSearchConfig = $this->createMock(InstantSearchHelper::class); + $this->personalizationHelper = $this->createMock(PersonalizationHelper::class); + $this->catalogSearchHelper = $this->createMock(CatalogSearchHelper::class); + $this->productHelper = $this->createMock(ProductHelper::class); + $this->currency = $this->createMock(Currency::class); + $this->format = $this->createMock(Format::class); + $this->currentProduct = $this->createMock(CurrentProduct::class); + $this->algoliaConnector = $this->createMock(AlgoliaConnector::class); + $this->urlHelper = $this->createMock(UrlHelper::class); + $this->formKey = $this->createMock(FormKey::class); + $this->httpContext = $this->createMock(HttpContext::class); + $this->coreHelper = $this->createMock(CoreHelper::class); + $this->categoryHelper = $this->createMock(CategoryHelper::class); + $this->suggestionHelper = $this->createMock(SuggestionHelper::class); + $this->landingPageHelper = $this->createMock(LandingPageHelper::class); + $this->checkoutSession = $this->createMock(CheckoutSession::class); + $this->date = $this->createMock(DateTime::class); + $this->currentCategory = $this->createMock(CurrentCategory::class); + $this->sortingTransformer = $this->createMock(SortingTransformer::class); + $this->context = $this->createMock(Context::class); + + $this->request = $this->createMock(Http::class); + $this->context->method('getRequest')->willReturn($this->request); + + $this->configurationBlock = new ConfigurationBlock( + $this->config, + $this->autocompleteConfig , + $this->instantSearchConfig , + $this->personalizationHelper , + $this->catalogSearchHelper , + $this->productHelper, + $this->currency , + $this->format , + $this->currentProduct , + $this->algoliaConnector , + $this->urlHelper , + $this->formKey , + $this->httpContext , + $this->coreHelper , + $this->categoryHelper, + $this->suggestionHelper , + $this->landingPageHelper , + $this->checkoutSession, + $this->date , + $this->currentCategory , + $this->sortingTransformer , + $this->context, + ); + } + + /** + * @dataProvider searchPageDataProvider + */ + public function testIsSearchPage($action, $categoryId, $categoryDisplayMode, $expectedResult): void + { + $this->config->method('isInstantEnabled')->willReturn(true); + $this->request->method('getFullActionName')->willReturn($action); + + $controller = explode('_', $action); + $controller = $controller[1]; + + $this->request->method('getControllerName')->willReturn($controller); + $this->config->method('replaceCategories')->willReturn(true); + + $category = $this->createMock(Category::class); + $category->method('getId')->willReturn($categoryId); + $category->method('getDisplayMode')->willReturn($categoryDisplayMode); + $this->currentCategory->method('get')->willReturn($category); + + $this->assertEquals($expectedResult, $this->configurationBlock->isSearchPage()); + } + + public static function searchPageDataProvider(): array + { + return [ + [ // true if category has an ID + 'action' => 'catalog_category_view', + 'categoryId' => 1, + 'categoryDisplayMode' => 'PRODUCT', + 'expectedResult' => true + ], + [ // false if category has no ID + 'action' => 'catalog_category_view', + 'categoryId' => null, + 'categoryDisplayMode' => 'PRODUCT', + 'expectedResult' => false + ], + [ // false if category has a PAGE as display mode + 'action' => 'catalog_category_view', + 'categoryId' => 1, + 'categoryDisplayMode' => 'PAGE', + 'expectedResult' => false + ], + [ // true if catalogsearch + 'action' => 'catalogsearch_result_index', + 'categoryId' => null, + 'categoryDisplayMode' => 'FOO', + 'expectedResult' => true + ], + [ // true if landing page + 'action' => 'algolia_landingpage_view', + 'categoryId' => null, + 'categoryDisplayMode' => 'FOO', + 'expectedResult' => true + ] + ]; + } +} From 35c92ebd23f8f7b11bfa2a198540eb3dfffa8169 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Fri, 31 Oct 2025 14:56:05 +0100 Subject: [PATCH 113/119] MAGE-1453: updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c8a16f7d..bf5766790 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Bug fixes - Fixed issue where missing pricing keys were not handled gracefully in the Autocomplete product template +- Fixed issue where category was not properly checked in the configuration block - thank you @benjamin-volle ## 3.17.0-beta.2 From e7cc59f9084bc4a0a48f960edf20e089d15b0fc6 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Fri, 14 Nov 2025 15:15:08 +0100 Subject: [PATCH 114/119] MAGE-1460: update version files and changelog --- CHANGELOG.md | 22 ++++++---------------- README.md | 2 +- composer.json | 2 +- etc/module.xml | 2 +- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf5766790..b16cdf9cb 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,21 +1,6 @@ # CHANGE LOG -## 3.17.0-dev - -### Bug fixes -- Fixed issue where missing pricing keys were not handled gracefully in the Autocomplete product template -- Fixed issue where category was not properly checked in the configuration block - thank you @benjamin-volle - -## 3.17.0-beta.2 - -### Updates -- Removed all `is_null` occurrences - -### Bug fixes -- Fixed 3.17 setup:upgrade on PHP 8.4 -- Fixed many Codacy issues - -## 3.17.0-beta.1 +## 3.17.0 ### Features - Added an Algolia indexing cache for storing metadata to prevent extra queries. Large collections that run periodic full indexes can benefit from this cache. @@ -30,9 +15,14 @@ - Updated default "Maximum number of records sent per indexing request" to 1000 (previously 300). - Updated `ConfigHelper` class, it now has more methods deprecated and ported to separate helper classes. - Updated Unit and Integration tests. +- Removed all `is_null` occurrences ### Bug fixes - Fixed indexing queue templates escaping. +- Fixed 3.17 setup:upgrade on PHP 8.4 +- Fixed many Codacy issues +- Fixed issue where missing pricing keys were not handled gracefully in the Autocomplete product template +- Fixed issue where category was not properly checked in the configuration block - thank you @benjamin-volle ## 3.16.1 diff --git a/README.md b/README.md index 558854773..da50999dd 100755 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Algolia Search & Discovery extension for Magento 2 ================================================== -![Latest version](https://img.shields.io/badge/latest-3.16.0-green) +![Latest version](https://img.shields.io/badge/latest-3.17.0-green) ![Magento 2](https://img.shields.io/badge/Magento-2.4.7+-orange) ![Beta version](https://img.shields.io/badge/beta-3.17.0--beta.1-purple) diff --git a/composer.json b/composer.json index 506307b03..3ad156dc2 100755 --- a/composer.json +++ b/composer.json @@ -3,7 +3,7 @@ "description": "Algolia Search & Discovery extension for Magento 2", "type": "magento2-module", "license": ["MIT"], - "version": "3.17.0-beta.2", + "version": "3.17.0", "require": { "php": "~8.2|~8.3|~8.4", "magento/framework": "~103.0", diff --git a/etc/module.xml b/etc/module.xml index b22b86847..f450526eb 100755 --- a/etc/module.xml +++ b/etc/module.xml @@ -1,6 +1,6 @@ - + From c3c441fbeb3d4249d23f0b8de9f12462ad93a6bd Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Tue, 18 Nov 2025 14:12:47 +0100 Subject: [PATCH 115/119] MAGE-1460: fix integration tests --- Test/Integration/Indexing/Config/ConfigTest.php | 8 +++++++- Test/Integration/Indexing/Queue/QueueTest.php | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Test/Integration/Indexing/Config/ConfigTest.php b/Test/Integration/Indexing/Config/ConfigTest.php index 2f6af8375..3cc32e90e 100644 --- a/Test/Integration/Indexing/Config/ConfigTest.php +++ b/Test/Integration/Indexing/Config/ConfigTest.php @@ -32,7 +32,13 @@ public function testRenderingContent() { $this->setConfig('algoliasearch_instant/instant_facets/enable_dynamic_facets', '1'); - $this->syncSettingsToAlgolia(); + try { + $this->syncSettingsToAlgolia(); + } catch (AlgoliaException $e) { + // Skip this test if the renderingContent feature isn't enabled on the application + $this->setConfig('algoliasearch_instant/instant_facets/enable_dynamic_facets', '0'); + $this->markTestSkipped($e->getMessage()); + } $indexOptions = $this->indexOptionsBuilder->buildWithEnforcedIndex($this->indexPrefix . 'default_products'); $indexSettings = $this->algoliaConnector->getSettings($indexOptions); diff --git a/Test/Integration/Indexing/Queue/QueueTest.php b/Test/Integration/Indexing/Queue/QueueTest.php index 00559137e..9eb931127 100644 --- a/Test/Integration/Indexing/Queue/QueueTest.php +++ b/Test/Integration/Indexing/Queue/QueueTest.php @@ -188,6 +188,7 @@ public function testSettings() ]); $this->setConfig(QueueHelper::IS_ACTIVE, '1'); + $this->setConfig(ConfigHelper::ENABLE_INDEXER_QUEUE, '1'); $this->connection->query('DELETE FROM algoliasearch_queue'); @@ -221,6 +222,7 @@ public function testSettings() public function testMergeSettings() { $this->setConfig(QueueHelper::IS_ACTIVE, '1'); + $this->setConfig(ConfigHelper::ENABLE_INDEXER_QUEUE, '1'); $this->setConfig(QueueHelper::NUMBER_OF_JOB_TO_RUN, 1); $this->setConfig(ConfigHelper::NUMBER_OF_ELEMENT_BY_PAGE, 300); From 5f62820e716ee6b3bdd0fcd203515df06f2941dd Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Tue, 18 Nov 2025 14:52:58 +0100 Subject: [PATCH 116/119] MAGE-1460: changes after review --- CHANGELOG.md | 2 +- README.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b16cdf9cb..95dfd4afa 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ ### Bug fixes - Fixed indexing queue templates escaping. - Fixed 3.17 setup:upgrade on PHP 8.4 -- Fixed many Codacy issues +- Performed code sanitization for Codacy compliance - Fixed issue where missing pricing keys were not handled gracefully in the Autocomplete product template - Fixed issue where category was not properly checked in the configuration block - thank you @benjamin-volle diff --git a/README.md b/README.md index da50999dd..eaead7ab1 100755 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ Algolia Search & Discovery extension for Magento 2 ![Latest version](https://img.shields.io/badge/latest-3.17.0-green) ![Magento 2](https://img.shields.io/badge/Magento-2.4.7+-orange) -![Beta version](https://img.shields.io/badge/beta-3.17.0--beta.1-purple) ![PHP](https://img.shields.io/badge/PHP-8.1%2C8.2%2C8.3%2C8.4-blue) From ea3ef4c3b66a061f1c862d99f70501af8459c838 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 19 Nov 2025 13:31:32 +0100 Subject: [PATCH 117/119] MAGE-1468: add array check on the plugin logic --- Plugin/Cache/CacheCleanProductPlugin.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Plugin/Cache/CacheCleanProductPlugin.php b/Plugin/Cache/CacheCleanProductPlugin.php index ef3d9a025..f75ba9ee4 100644 --- a/Plugin/Cache/CacheCleanProductPlugin.php +++ b/Plugin/Cache/CacheCleanProductPlugin.php @@ -106,6 +106,12 @@ protected function hasStockChanged(array $orig, array $new, int $storeId): bool $key = 'quantity_and_stock_status'; $oldStock = $orig[$key]; $newStock = $new[$key]; + + // In case of a product duplication on second save + if (!is_array($newStock)) { + $newStock = ['is_in_stock' => $newStock]; + } + return $this->canCompareValues($oldStock, $newStock, 'is_in_stock') && (bool) $oldStock['is_in_stock'] !== (bool) $newStock['is_in_stock'] || $this->canCompareValues($oldStock, $newStock, 'qty') From c728fb5615e91814054490427506d92149f10c28 Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 19 Nov 2025 14:23:59 +0100 Subject: [PATCH 118/119] MAGE-1468: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95dfd4afa..d6e565f61 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Performed code sanitization for Codacy compliance - Fixed issue where missing pricing keys were not handled gracefully in the Autocomplete product template - Fixed issue where category was not properly checked in the configuration block - thank you @benjamin-volle +- Fixed issue on duplicated product save ## 3.16.1 From 804fdebfa7a29dce71cb6dd00a43f24251e33cce Mon Sep 17 00:00:00 2001 From: Damien Couchez Date: Wed, 19 Nov 2025 15:43:25 +0100 Subject: [PATCH 119/119] MAGE-1468: add comment --- Plugin/Cache/CacheCleanProductPlugin.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Plugin/Cache/CacheCleanProductPlugin.php b/Plugin/Cache/CacheCleanProductPlugin.php index f75ba9ee4..c57a9a8b3 100644 --- a/Plugin/Cache/CacheCleanProductPlugin.php +++ b/Plugin/Cache/CacheCleanProductPlugin.php @@ -107,7 +107,7 @@ protected function hasStockChanged(array $orig, array $new, int $storeId): bool $oldStock = $orig[$key]; $newStock = $new[$key]; - // In case of a product duplication on second save + // In case of a product duplication on second save (for some reason, Magento returns a different data structure in that case). if (!is_array($newStock)) { $newStock = ['is_in_stock' => $newStock]; }