Merge remote-tracking branch 'github/develop'

Author: tkostuch

CHANGELOG.md

@@ -19,14 +19,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Update to `storefront-query-builder` version `1.0.0` - @cewald (#429)
 - Add `composeError` in './magento2/o2m.js' - @flancer64 (#422)
 - Explicit data extraction from 'Error' objects - @flancer64 (#424)
+- Merge scripts: db7 => db, elastic7=>elastic and add support for es7 in yarn mage2vs import - @gibkigonzo (#436)
 
 ### Fixed
 - add es7 support for map url module and fixed default index for es config - @gibkigonzo
 - Add correct paths for production build - @cewald (#407)
 - Fix MSI default stock id value
 - Add outputFormatter to response from cache - @gibkigonzo (#428)
 - disable showing stack for invalid requests - @gibkigonzo (#431)
-
+- Improve `_outputFormatter` on cache catalog-response to prevent exception - @cewald (#432)
 
 ## [1.11.1] - 2020.03.17
 

README.md

@@ -111,22 +111,7 @@ Starting from [Elasitc 6 and 7](https://www.elastic.co/guide/en/elasticsearch/re
 
 From now on, we're using the separate indexes per each entity type. The convention is: `${indexName}_${entityType}`. If your' **logical index name** is `vue_storefront_catalog` then it will be mapped to the **physical indexes** of: `vue_storefront_catalog_product`, `vue_storefront_catalog_category` ...
 
-To take the advantage of this new logical->physical index distinction we've provided new Elastic tools: `db7`, `migrate7`, `dump7`, `restore7` tools. They can be used exactly the same way [like the old tools](https://docs.vuestorefront.io/guide/data/database-tool.html) were. The only distinction is that they work on separate indexes.
-
-**Create new index**
-
-Before restoreing or importing data you might want to create a new Elastic index with the proper data types/schema applied. You can run just the `yarn db7 new` command in order to do that.
-
-**Restore the data**
-
-After you ran the docker file and have Elastic7 up and running you might want to:
-
-a) restore the demo data by running `yarn restore7` and `yarn restore7_it; yarn restore7_de` for default multistores. The data is restored from `var/catalog_product.json`, `var/catalog_category.json` and so on...
-
-b) import the data from Magento to proper physical indexes. To do so, currently you can do this only with [mage2vuestorefront](https://github.com/DivanteLtd/mage2vuestorefront/pull/96).
-
-**Note:** After 1.11 stable release (around November, 2019) we'll **replace** the standard tools: `db`, `migrate`, `dump`, `restore` with the Elastic 7 equivalents and it will become the default.
-
+[Tools](https://docs.vuestorefront.io/guide/data/database-tool.html) are adjusted to ES7. You can use `yarn db new`, `yarn restore`, `yarn mage2vs import`. Just make sure that you have set up `config.elasticsearch.apiVersion` to `7.1`.
 
 ## API access
 Catalog API calls are compliant with ElasticSearch (it works like a filtering proxy to ES). More on ES queries: [ElasticSearch queries tutorial](http://okfnlabs.org/blog/2013/07/01/elasticsearch-query-tutorial.html)

config/elastic.schema.product.json

@@ -76,6 +76,25 @@
        "tax_class_id": {"type": "integer"},
        "gender": {"type": "integer"},
        "material": {"type": "integer"},
-       "category_gear": {"type": "integer"}
+       "category_gear": {"type": "integer"},
+       "attributes_metadata": {
+            "properties": {
+                "id": {"type": "integer"},
+                "attribute_id": {"type": "integer"},
+                "default_frontend_label": {"type": "text"},
+                "is_visible_on_front": {"type": "text"},
+                "is_visible" : {"type": "boolean"},
+                "frontend_input": {"type": "text"},
+                "is_user_defined": {"type": "boolean"},
+                "is_comparable": {"type": "text"},
+                "attribute_code": {"type": "text"},
+                "options": {
+                    "properties": {
+                        "value": {"type": "text"},
+                        "label": {"type": "text"}
+                    }
+                }
+            }
+        }
     }
 }

package.json

@@ -16,22 +16,15 @@
     "build:copy:schema": "cpx 'src/**/*.schema*.json' dist/src",
     "start": "pm2 start ecosystem.json $PM2_ARGS",
     "db": "node scripts/db.js",
-    "db7": "node scripts/db7.js",
     "cache": "node ./scripts/cache",
     "mage2vs": "node scripts/mage2vs.js",
     "restore": "node scripts/elastic.js restore && npm run db rebuild",
-    "restore7": "node scripts/elastic7.js restore --output-index=vue_storefront_catalog && npm run db7 rebuild --indexName=vue_storefront_catalog",
-    "restore7_it": "npm run restore7 -- --input-file=var/catalog_it.json --output-index=vue_storefront_catalog_it && npm run db7 rebuild -- --indexName=vue_storefront_catalog_it",
-    "restore7_de": "npm run restore7 -- --input-file=var/catalog_de.json --output-index=vue_storefront_catalog_de && npm run db7 rebuild -- --indexName=vue_storefront_catalog_de",
     "restore_it": "npm run restore -- --input-file=var/catalog_it.json --output-index=vue_storefront_catalog_it && npm run db rebuild -- --indexName=vue_storefront_catalog_it",
     "restore_de": "npm run restore -- --input-file=var/catalog_de.json --output-index=vue_storefront_catalog_de && npm run db rebuild -- --indexName=vue_storefront_catalog_de",
     "restore2main": "npm run restore -- --output-index=vue_storefront_catalog",
     "dump": "node scripts/elastic.js dump",
-    "dump_it": "npm run dump -- --input-index=vue_storefront_catalog_it --output-file=var/catalog_it.json",
-    "dump_de": "npm run dump -- --input-index=vue_storefront_catalog_de --output-file=var/catalog_de.json",
-    "dump7": "node scripts/elastic7.js dump",
-    "dump7_it": "npm run dump7 -- --input-index=vue_storefront_catalog_it --output-file=var/catalog_it.json",
-    "dump7_de": "npm run dump7 -- --input-index=vue_storefront_catalog_de --output-file=var/catalog_de.json",
+    "dump_it": "npm run dump -- --input-index=vue_storefront_catalog --output-file=var/catalog_it.json",
+    "dump_de": "npm run dump -- --input-index=vue_storefront_catalog --output-file=var/catalog_de.json",
     "kue": "node scripts/kue.js",
     "kue-dashboard": "npm run kue dashboard -- --port=3050",
     "o2m": "node src/worker/order_to_magento2.js start",

scripts/db.js

@@ -3,75 +3,176 @@ const config = require('config')
 const common = require('../migrations/.common')
 const es = require('../src/lib/elastic')
 
-program
-  .command('rebuild')
-  .option('-i|--indexName <indexName>', 'name of the Elasticsearch index', config.elasticsearch.indices[0])
-  .action((cmd) => { // TODO: add parallel processing
-    if (!cmd.indexName) {
-      console.error('error: indexName must be specified');
-      process.exit(1);
-    }
+/**
+ * REBUILD DB
+ */
+const es7RebuildCommand = (cmd) => { // TODO: add parallel processing
+  if (!cmd.indexName) {
+    console.error('error: indexName must be specified');
+    process.exit(1);
+  }
 
+  let waitingCounter = 0
+  for (var indexTypeIterator in config.elasticsearch.indexTypes) {
+    var collectionName = config.elasticsearch.indexTypes[indexTypeIterator]
+    console.log(config.elasticsearch.indexTypes);
     console.log('** Hello! I am going to rebuild EXISTING ES index to fix the schema')
-    const originalIndex = cmd.indexName
+    const originalIndex = cmd.indexName + '_' + collectionName;
     const tempIndex = originalIndex + '_' + Math.round(+new Date() / 1000)
 
     console.log(`** Creating temporary index ${tempIndex}`)
-    es.createIndex(common.db, tempIndex, '', (err) => {
+    es.createIndex(common.db, tempIndex, collectionName, (err) => {
       if (err) {
         console.log(err)
       }
 
-      console.log(`** Putting the mappings on top of ${tempIndex}`)
-      es.putMappings(common.db, tempIndex, (err) => {
+      console.log(`** We will reindex ${originalIndex} with the current schema`)
+      es.reIndex(common.db, originalIndex, tempIndex, (err) => {
         if (err) {
-          console.error(err.meta ? err.meta : err)
+          console.log(err)
         }
 
-        console.log(`** We will reindex ${originalIndex} with the current schema`)
-        es.reIndex(common.db, originalIndex, tempIndex, (err) => {
+        console.log('** Removing the original index')
+        es.deleteIndex(common.db, originalIndex, (err) => {
           if (err) {
             console.log(err)
           }
 
-          console.log('** Removing the original index')
-          es.deleteIndex(common.db, originalIndex, (err) => {
-            if (err) {
-              console.log(err)
-            }
-
-            console.log('** Creating alias')
-            es.putAlias(common.db, tempIndex, originalIndex, (err) => {
-              console.log('Done! Bye!')
-              process.exit(0)
-            })
+          console.log('** Creating alias')
+          es.putAlias(common.db, tempIndex, originalIndex, (err) => {
+            waitingCounter++
           })
         })
       })
     })
-  })
+  }
+  setInterval(() => {
+    if (waitingCounter === config.elasticsearch.indexTypes.length) process.exit(0)
+  }, 1000)
+}
 
-program
-  .command('new')
-  .option('-i|--indexName <indexName>', 'name of the Elasticsearch index', config.elasticsearch.indices[0])
-  .action((cmd) => { // TODO: add parallel processing
-    if (!cmd.indexName) {
-      console.error('error: indexName must be specified');
-      process.exit(1);
+const es5RebuildCommand = (cmd) => { // TODO: add parallel processing
+  console.warn(`es5 is deprecated and will be removed in 1.13`)
+  if (!cmd.indexName) {
+    console.error('error: indexName must be specified');
+    process.exit(1);
+  }
+
+  console.log('** Hello! I am going to rebuild EXISTING ES index to fix the schema')
+  const originalIndex = cmd.indexName
+  const tempIndex = originalIndex + '_' + Math.round(+new Date() / 1000)
+
+  console.log(`** Creating temporary index ${tempIndex}`)
+  es.createIndex(common.db, tempIndex, '', (err) => {
+    if (err) {
+      console.log(err)
     }
 
-    console.log('** Hello! I am going to create NEW ES index')
-    const indexName = cmd.indexName
-    es.createIndex(common.db, indexName, '', (err) => {
+    console.log(`** Putting the mappings on top of ${tempIndex}`)
+    es.putMappings(common.db, tempIndex, (err) => {
       if (err) {
-        console.log(err)
+        console.error(err.meta ? err.meta : err)
       }
 
-      console.log('Done! Bye!')
-      process.exit(0)
+      console.log(`** We will reindex ${originalIndex} with the current schema`)
+      es.reIndex(common.db, originalIndex, tempIndex, (err) => {
+        if (err) {
+          console.log(err)
+        }
+
+        console.log('** Removing the original index')
+        es.deleteIndex(common.db, originalIndex, (err) => {
+          if (err) {
+            console.log(err)
+          }
+
+          console.log('** Creating alias')
+          es.putAlias(common.db, tempIndex, originalIndex, (err) => {
+            console.log('Done! Bye!')
+            process.exit(0)
+          })
+        })
+      })
     })
   })
+}
+
+program
+  .command('rebuild')
+  .option('-i|--indexName <indexName>', 'name of the Elasticsearch index', config.elasticsearch.indices[0])
+  .action((cmd) => {
+    if (parseInt(config.elasticsearch.apiVersion) < 6) {
+      return es5RebuildCommand(cmd)
+    } else {
+      return es7RebuildCommand(cmd)
+    }
+  })
+
+/**
+ * CREATE INDEX
+ */
+const asyncCreateIndex = (es, common, indexName, collectionName) => new Promise((resolve, reject) => {
+  es.createIndex(common.db, indexName + '_' + collectionName, collectionName, (err) => {
+    if (err) {
+      reject(err)
+    } else {
+      resolve()
+    }
+  })
+})
+
+const es7NewCommand = async (cmd) => { // TODO: add parallel processing
+  if (!cmd.indexName) {
+    console.error('error: indexName must be specified');
+    process.exit(1);
+  }
+
+  console.log('** Hello! I am going to create NEW ES index')
+  const indexName = cmd.indexName
+
+  for (let collectionName of config.elasticsearch.indexTypes) {
+    try {
+      await asyncCreateIndex(es, common, indexName, collectionName)
+    } catch (err) {
+      console.log(JSON.stringify(err, null, 2))
+    }
+  }
+  process.exit(0)
+}
+
+const es5NewCommand = (cmd) => { // TODO: add parallel processing
+  console.warn(`es5 is deprecated and will be removed in 1.13`)
+  if (!cmd.indexName) {
+    console.error('error: indexName must be specified');
+    process.exit(1);
+  }
+
+  console.log('** Hello! I am going to create NEW ES index')
+  const indexName = cmd.indexName
+  es.createIndex(common.db, indexName, '', (err) => {
+    if (err) {
+      console.log(err)
+    }
+
+    console.log('Done! Bye!')
+    process.exit(0)
+  })
+}
+
+program
+  .command('new')
+  .option('-i|--indexName <indexName>', 'name of the Elasticsearch index', config.elasticsearch.indices[0])
+  .action((cmd) => {
+    if (parseInt(config.elasticsearch.apiVersion) < 6) {
+      return es5NewCommand(cmd)
+    } else {
+      return es7NewCommand(cmd)
+    }
+  })
 
+/**
+ * OTHERS COMMANDS
+ */
 program
   .on('command:*', () => {
     console.error('Invalid command: %s\nSee --help for a list of available commands.', program.args.join(' '));