update var files and remove elastic7

Author: tkostuch

README.md

@@ -111,22 +111,7 @@ Starting from [Elasitc 6 and 7](https://www.elastic.co/guide/en/elasticsearch/re
 
 From now on, we're using the separate indexes per each entity type. The convention is: `${indexName}_${entityType}`. If your' **logical index name** is `vue_storefront_catalog` then it will be mapped to the **physical indexes** of: `vue_storefront_catalog_product`, `vue_storefront_catalog_category` ...
 
-To take the advantage of this new logical->physical index distinction we've provided new Elastic tools: `db7`, `migrate7`, `dump7`, `restore7` tools. They can be used exactly the same way [like the old tools](https://docs.vuestorefront.io/guide/data/database-tool.html) were. The only distinction is that they work on separate indexes.
-
-**Create new index**
-
-Before restoreing or importing data you might want to create a new Elastic index with the proper data types/schema applied. You can run just the `yarn db7 new` command in order to do that.
-
-**Restore the data**
-
-After you ran the docker file and have Elastic7 up and running you might want to:
-
-a) restore the demo data by running `yarn restore7` and `yarn restore7_it; yarn restore7_de` for default multistores. The data is restored from `var/catalog_product.json`, `var/catalog_category.json` and so on...
-
-b) import the data from Magento to proper physical indexes. To do so, currently you can do this only with [mage2vuestorefront](https://github.com/DivanteLtd/mage2vuestorefront/pull/96).
-
-**Note:** After 1.11 stable release (around November, 2019) we'll **replace** the standard tools: `db`, `migrate`, `dump`, `restore` with the Elastic 7 equivalents and it will become the default.
-
+[Tools](https://docs.vuestorefront.io/guide/data/database-tool.html) are adjusted to ES7. You can use `yarn db new`, `yarn restore`, `yarn mage2vs import`. Just make sure that you have set up `config.elasticsearch.apiVersion` to `7.1`.
 
 ## API access
 Catalog API calls are compliant with ElasticSearch (it works like a filtering proxy to ES). More on ES queries: [ElasticSearch queries tutorial](http://okfnlabs.org/blog/2013/07/01/elasticsearch-query-tutorial.html)

config/elastic.schema.product.json

@@ -82,11 +82,11 @@
                 "id": {"type": "integer"},
                 "attribute_id": {"type": "integer"},
                 "default_frontend_label": {"type": "text"},
-                "is_visible_on_front": {"type": "boolean"},
+                "is_visible_on_front": {"type": "text"},
                 "is_visible" : {"type": "boolean"},
                 "frontend_input": {"type": "text"},
                 "is_user_defined": {"type": "boolean"},
-                "is_comparable": {"type": "boolean"},
+                "is_comparable": {"type": "text"},
                 "attribute_code": {"type": "text"},
                 "options": {
                     "properties": {

package.json

@@ -19,18 +19,12 @@
     "cache": "node ./scripts/cache",
     "mage2vs": "node scripts/mage2vs.js",
     "restore": "node scripts/elastic.js restore && npm run db rebuild",
-    "restore7": "node scripts/elastic7.js restore --output-index=vue_storefront_catalog && npm run db rebuild --indexName=vue_storefront_catalog",
-    "restore7_it": "npm run restore7 -- --input-file=var/catalog_it.json --output-index=vue_storefront_catalog_it && npm run db rebuild -- --indexName=vue_storefront_catalog_it",
-    "restore7_de": "npm run restore7 -- --input-file=var/catalog_de.json --output-index=vue_storefront_catalog_de && npm run db rebuild -- --indexName=vue_storefront_catalog_de",
     "restore_it": "npm run restore -- --input-file=var/catalog_it.json --output-index=vue_storefront_catalog_it && npm run db rebuild -- --indexName=vue_storefront_catalog_it",
     "restore_de": "npm run restore -- --input-file=var/catalog_de.json --output-index=vue_storefront_catalog_de && npm run db rebuild -- --indexName=vue_storefront_catalog_de",
     "restore2main": "npm run restore -- --output-index=vue_storefront_catalog",
     "dump": "node scripts/elastic.js dump",
-    "dump_it": "npm run dump -- --input-index=vue_storefront_catalog_it --output-file=var/catalog_it.json",
-    "dump_de": "npm run dump -- --input-index=vue_storefront_catalog_de --output-file=var/catalog_de.json",
-    "dump7": "node scripts/elastic7.js dump",
-    "dump7_it": "npm run dump7 -- --input-index=vue_storefront_catalog_it --output-file=var/catalog_it.json",
-    "dump7_de": "npm run dump7 -- --input-index=vue_storefront_catalog_de --output-file=var/catalog_de.json",
+    "dump_it": "npm run dump -- --input-index=vue_storefront_catalog --output-file=var/catalog_it.json",
+    "dump_de": "npm run dump -- --input-index=vue_storefront_catalog --output-file=var/catalog_de.json",
     "kue": "node scripts/kue.js",
     "kue-dashboard": "npm run kue dashboard -- --port=3050",
     "o2m": "node src/worker/order_to_magento2.js start",

scripts/db.js

@@ -52,6 +52,7 @@ const es7RebuildCommand = (cmd) => { // TODO: add parallel processing
 }
 
 const es5RebuildCommand = (cmd) => { // TODO: add parallel processing
+  console.warn(`es5 is deprecated and will be removed in 1.13`)
   if (!cmd.indexName) {
     console.error('error: indexName must be specified');
     process.exit(1);
@@ -140,6 +141,7 @@ const es7NewCommand = async (cmd) => { // TODO: add parallel processing
 }
 
 const es5NewCommand = (cmd) => { // TODO: add parallel processing
+  console.warn(`es5 is deprecated and will be removed in 1.13`)
   if (!cmd.indexName) {
     console.error('error: indexName must be specified');
     process.exit(1);

scripts/elastic.js

@@ -7,34 +7,98 @@ function stdOutErr (stdout, stderr) {
   if (stderr.length > 0) { console.error(stderr.toString('utf8')) }
 }
 
+/**
+ * DUMP COMMAND
+ */
+const es5DumpCommand = (cmd) => {
+  console.warn(`es5 is deprecated and will be removed in 1.13`)
+  const input = `http://${config.host}:${config.port}/${cmd.inputIndex}`
+
+  const child = spawnSync('node', [
+    'node_modules/elasticdump/bin/elasticdump',
+    `--input=${input}`,
+    `--output=${cmd.outputFile}`
+  ])
+  stdOutErr(child.stdout, child.stderr)
+}
+
+const es7DumpCommand = (cmd) => {
+  if (!cmd.outputFile.indexOf('.json')) {
+    console.error('Please provide the file name ending with .json ext.')
+  }
+  for (var indexTypeIterator in config.indexTypes) {
+    var collectionName = config.indexTypes[indexTypeIterator]
+    var inputIndex = `${cmd.inputIndex}_${collectionName}`
+    var outputFile = cmd.outputFile.replace('.json', `_${collectionName}.json`)
+    const input = `http://${config.host}:${config.port}/${inputIndex}`
+
+    const child = spawnSync('node', [
+      'node_modules/elasticdump/bin/elasticdump',
+      `--input=${input}`,
+      `--output=${outputFile}`
+    ])
+    stdOutErr(child.stdout, child.stderr)
+  }
+}
+
 program
   .command('dump')
   .option('--input-index <inputIndex>', 'index to dump', 'vue_storefront_catalog')
   .option('--output-file <outputFile>', 'path to the output file', 'var/catalog.json')
   .action((cmd) => {
-    const input = `http://${config.host}:${config.port}/${cmd.inputIndex}`
+    if (parseInt(config.apiVersion) < 6) {
+      return es5DumpCommand(cmd)
+    } else {
+      return es7DumpCommand(cmd)
+    }
+  })
+
+/**
+ * RESTORE COMMAND
+ */
+
+const es5RestoreCommand = (cmd) => {
+  console.warn(`es5 is deprecated and will be removed in 1.13`)
+  const output = `http://${config.host}:${config.port}/${cmd.outputIndex}`
+
+  const child = spawnSync('node', [
+    'node_modules/elasticdump/bin/elasticdump',
+    `--input=${cmd.inputFile}`,
+    `--output=${output}`
+  ])
+  stdOutErr(child.stdout, child.stderr)
+}
+
+const es7RestoreCommand = (cmd) => {
+  if (!cmd.inputFile.indexOf('.json')) {
+    console.error('Please provide the file name ending with .json ext.')
+  }
+  for (var indexTypeIterator in config.indexTypes) {
+    var collectionName = config.indexTypes[indexTypeIterator]
+    var outputIndex = `${cmd.outputIndex}_${collectionName}`
+    var inputFile = cmd.inputFile.replace('.json', `_${collectionName}.json`)
+
+    const output = `http://${config.host}:${config.port}/${outputIndex}`
 
     const child = spawnSync('node', [
       'node_modules/elasticdump/bin/elasticdump',
-      `--input=${input}`,
-      `--output=${cmd.outputFile}`
+      `--input=${inputFile}`,
+      `--output=${output}`
     ])
     stdOutErr(child.stdout, child.stderr)
-  })
+  }
+}
 
 program
   .command('restore')
   .option('--output-index <outputIndex>', 'index to restore', 'vue_storefront_catalog')
   .option('--input-file <inputFile>', 'path to the input file', 'var/catalog.json')
   .action((cmd) => {
-    const output = `http://${config.host}:${config.port}/${cmd.outputIndex}`
-
-    const child = spawnSync('node', [
-      'node_modules/elasticdump/bin/elasticdump',
-      `--input=${cmd.inputFile}`,
-      `--output=${output}`
-    ])
-    stdOutErr(child.stdout, child.stderr)
+    if (parseInt(config.apiVersion) < 6) {
+      return es5RestoreCommand(cmd)
+    } else {
+      return es7RestoreCommand(cmd)
+    }
   })
 
 program