feat: reworked how indexation works to have separate indexes

per store and type of stored data. This is a first working implementation for a full reindexation of the fulltext indexer. refs #7
front-commerce · Oct 28, 2019 · 9903800 · 9903800
1 parent b11d696
commit 9903800
Show file tree

Hide file tree

Showing 14 changed files with 515 additions and 368 deletions.
diff --git a/src/app/code/community/Smile/ElasticSearch/Helper/Data.php b/src/app/code/community/Smile/ElasticSearch/Helper/Data.php
@@ -22,7 +22,7 @@ class Smile_ElasticSearch_Helper_Data extends Mage_Core_Helper_Abstract
      * Temporary feature flag to disable multi-store indexation in a single index
      * while https://github.com/front-commerce/magento1-elasticsuite-indexer/issues/7 is not implemented
      */
-    const FEATURE_MULTI_STORE_ENABLED = false;
+    const FEATURE_MULTI_STORE_ENABLED = true;
 
     /**
      * Allowed languages.
@@ -32,34 +32,37 @@ class Smile_ElasticSearch_Helper_Data extends Mage_Core_Helper_Abstract
      */
     protected $_languageCodes = array();
 
-    public function getIndexedStores()
+    public function getIndexScopes()
+    {
+        return array_values(array_map(
+            ['Smile_ElasticSearch_Model_Scope', 'fromMagentoStore'],
+            $this->getIndexedStores())
+        );
+    }
+
+    private function getIndexedStores()
     {
         $stores = Mage::app()->getStores();
         if (static::FEATURE_MULTI_STORE_ENABLED) {
-            throw new LogicException('Multi-store is not available yet. See https://github.com/front-commerce/magento1-elasticsuite-indexer/issues/7');
+            return $stores;
         }
+
         $firstKey = array_shift(array_keys($stores));
         return [
             $firstKey => $stores[$firstKey]
         ];
     }
 
-    public function getIndexedStoreIds($ids)
-    {
-        if (static::FEATURE_MULTI_STORE_ENABLED) {
-            throw new LogicException('Multi-store is not available yet. See https://github.com/front-commerce/magento1-elasticsuite-indexer/issues/7');
-        }
-        return [array_shift($ids)];
-    }
-
     public function getIndexedStoreIdsFromWebsiteIds($ids)
     {
         if (static::FEATURE_MULTI_STORE_ENABLED) {
+            // TODO Get rid of usages of this method
             throw new LogicException('Multi-store is not available yet. See https://github.com/front-commerce/magento1-elasticsuite-indexer/issues/7');
         }
 
         $websiteId = current($ids);
-        return $this->getIndexedStoreIds(Mage::app()->getWebsite($websiteId)->getStoreIds());
+        $storeIds = Mage::app()->getWebsite($websiteId)->getStoreIds();
+        return [array_shift($storeIds)];
     }
 
     /**

diff --git a/src/app/code/community/Smile/ElasticSearch/Model/Indexer/Fulltext.php b/src/app/code/community/Smile/ElasticSearch/Model/Indexer/Fulltext.php
@@ -187,12 +187,22 @@ public function getCurrentIndex()
      */
     public function reindexAll()
     {
-        $index = $this->getCurrentIndex();
+        $indexes = $this->getAllIndexes();
 
-        $index->prepareNewIndex();
-        foreach ($index->getAllMappings() as $mapping) {
-            $mapping->rebuildIndex();
+        foreach ($indexes as $index) {
+            $index->prepareNewIndex();
+            $index->rebuildIndex();
+            $index->installNewIndex();
         }
-        $index->installNewIndex();
+    }
+
+    /**
+     * @return Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch_Index[]
+     */
+    public function getAllIndexes()
+    {
+        return Mage::helper('catalogsearch')->getEngine()->getCurrentIndexesForScopes(
+            Mage::helper('smile_elasticsearch')->getIndexScopes()
+        );
     }
 }
diff --git a/src/app/code/community/Smile/ElasticSearch/Model/Resource/Engine/Elasticsearch.php b/src/app/code/community/Smile/ElasticSearch/Model/Resource/Engine/Elasticsearch.php
@@ -97,7 +97,7 @@ class Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch
     protected $_client = null;
 
     /**
-     *
+     * @deprecated Index must be passed as parameters to use the one matching the correct scope
      * @var Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch_Index
      */
     protected $_currentIndex = null;
@@ -115,6 +115,20 @@ class Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch
      */
     protected $_dateFormats = array();
 
+    /**
+     * Base alias for all indexes
+     *
+     * @var string
+     */
+    protected $_aliasBase;
+
+    /**
+     * Types mappings.
+     *
+     * @var array
+     */
+    protected $_mappings = [];
+
     /**
      * Initializes search engine config and index name.
      *
@@ -135,6 +149,15 @@ public function __construct($params = false)
         if (!isset($config['alias'])) {
             Mage::throwException('Alias must be defined for search engine client.');
         }
+        $this->_aliasBase = $config['alias'];
+
+        $mappingConfig = Mage::getConfig()->getNode(Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch_Index::MAPPING_CONF_ROOT_NODE)->asArray();
+        foreach ($mappingConfig as $type => $config) {
+            if ($type === "product") {
+                $this->_mappings[$type] = Mage::getResourceSingleton($config['model']);
+                $this->_mappings[$type]->setType($type);
+            }
+        }
 
         $this->_currentIndex = Mage::getResourceModel('smile_elasticsearch/engine_elasticsearch_index');
         $this->_currentIndex->setAdapter($this)->setCurrentName($config['alias']);
@@ -153,13 +176,60 @@ public function getClient()
     /**
      * Return the current index instance
      *
+     * @deprecated Use getCurrentIndexesForScope
      * @return Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch_Index
      */
     public function getCurrentIndex()
     {
         return $this->_currentIndex;
     }
 
+    /**
+     * @param array $scopes
+     * @return Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch_Index[]
+     */
+    public function getCurrentIndexesForScopes(array $scopes)
+    {
+        return array_reduce(
+            array_map([$this, 'getCurrentIndexesForScope'], $scopes),
+            'array_merge',
+            []
+        );
+    }
+
+    /**
+     * Return the current index instance for a given scope
+     *
+     * @return Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch_Index[]
+     */
+    public function getCurrentIndexesForScope(Smile_ElasticSearch_Model_Scope $scope)
+    {
+        $indexes = [];
+        foreach ($this->_mappings as $type => $mapping) {
+            /** @var Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch_Index $index */
+            $index = Mage::getResourceModel('smile_elasticsearch/engine_elasticsearch_index');
+            $indexName = $this->aliasForScopedType($scope, $type);
+            $index
+                ->setAdapter($this)
+                ->setCurrentName($indexName)
+                ->setBaseName($indexName)
+                ->setScope($scope)
+                ->setMapping($mapping);
+            $indexes[] = $index;
+        }
+
+        return $indexes;
+    }
+
+    private function aliasForScopedType(Smile_ElasticSearch_Model_Scope $scope, $type)
+    {
+        return implode('_', [
+            $this->_aliasBase,
+            $scope->getIdentifier(),
+            $type
+        ]);
+    }
+
     /**
      * Cleans caches.
      *
@@ -190,21 +260,26 @@ public function cleanIndex($storeId = null, $id = null, $type = 'product')
         }
 
         if (is_null($storeId)) {
-            $storeId = Mage::helper('smile_elasticsearch')->getIndexedStoreIds(
-                array_keys(Mage::app()->getStores())
-            );
+             $scopes = Mage::helper('smile_elasticsearch')->getIndexScopes();
         } else if (!is_array($storeId)) {
-            $storeId = array($storeId);
+            $scopes = array(Smile_ElasticSearch_Model_Scope::fromMagentoStoreId($storeId));
+        } else {
+            $scopes = array_map(['Smile_ElasticSearch_Model_Scope', 'fromMagentoStoreId'], $storeId);
         }
+        $indexes = array_filter(
+            $this->getCurrentIndexesForScopes($scopes),
+            function (Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch_Index $index) use ($type) {
+                return $index->isForType($type);
+            }
+        );
 
         $bulk = array('body' => array());
 
         foreach ($id as $currentId) {
-            foreach ($storeId as $currentStoreId) {
+            foreach ($indexes as $index) {
                 $bulk['body'][] = array(
                     'delete' => array(
-                        '_index' => $this->getCurrentIndex()->getCurrentName(),
-                        '_type'  => $type,
+                        '_index' => $index->getCurrentName(),
                         '_id'    => $currentId
                     )
                 );
@@ -390,29 +465,26 @@ protected function _getDate($storeId, $date = null)
     /**
      * Perpare document to be indexed
      *
-     * @param array  $docsData Source document data to be indexed
-     * @param string $type     Document type
-     *
+     * @param Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch_Index $index
+     * @param array $docsData Source document data to be indexed
      * @return array
      */
-    protected function _prepareDocs($docsData, $type)
+    protected function _prepareDocs(Smile_ElasticSearch_Model_Resource_Engine_Elasticsearch_Index $index, $docsData)
     {
-        if (! is_array($docsData) || empty($docsData)) {
+        if (!is_array($docsData) || empty($docsData)) {
             return array();
         }
 
         $docs = array();
-        foreach ($docsData as $entityId => $index) {
-            $document = $this->getCurrentIndex()->createDocument($index[self::UNIQUE_KEY], $index, $type);
+        foreach ($docsData as $entityId => $data) {
+            $document = $index->createDocument($data[self::UNIQUE_KEY], $data);
             array_push($docs, $document[0]);
             array_push($docs, $document[1]);
         }
 
         return $docs;
     }
 
-
-
     /**
      * Indicates if connection to the search engine is up or not
      *