Merge pull request #390 from DivanteLtd/feature/2167

feature/2167

Author: andrzejewsky

CHANGELOG.md

@@ -9,11 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Add url module - @gibkigonzo (#3942)
-
-### Fixed
-
-### Changed / Improved
-
+- The `response_format` query parameter to the `/api/catalog` endpoint. Currently there is just one additional format supported: `response_format=compact`. When used, the response format got optimized by: a) remapping the results, removing the `_source` from the `hits.hits`; b) compressing the JSON fields names according to the `config.products.fieldsToCompact`; c) removing the JSON fields from the `product.configurable_children` when their values === parent product values; overall response size reduced over -70% - @pkarw
+- The support for `SearchQuery` instead of the ElasticSearch DSL as for the input to `/api/catalog` - using `storefront-query-builder` package - @pkarw - https://github.com/DivanteLtd/vue-storefront/issues/2167
 
 ## [1.11.0] - 2019.12.20
 

config/default.json

@@ -36,7 +36,83 @@
       "taxrule",
       "review"
     ],
-    "apiVersion": "5.6"
+    "apiVersion": "5.6",
+
+    "searchScoring": {
+      "attributes": {
+        "attribute_code": {
+          "scoreValues": { "attribute_value": { "weight": 1 } }
+        }
+      },
+      "fuzziness": 2,
+      "cutoff_frequency":  0.01,
+      "max_expansions": 3,
+      "minimum_should_match": "75%",
+      "prefix_length": 2,
+      "boost_mode": "multiply",
+      "score_mode": "multiply",
+      "max_boost": 100,
+      "function_min_score": 1
+    },
+    "searchableAttributes": {
+      "name": {
+        "boost": 4
+      },
+      "sku": {
+        "boost": 2
+      },
+      "category.name": {
+        "boost": 1
+      }
+    }
+  },
+  "products": {
+    "fieldsToCompress": ["max_regular_price", "max_price", "max_regular_price", "minimal_regular_price", "final_price", "price", "special_price", "original_final_price", "original_price", "original_special_price", "final_price_incl_tax", "price_incl_tax", "special_price_incl_tax", "final_price_tax", "price_tax", "special_price_tax", "image", "small_image", "thumbnail"],
+    "fieldsToCompact": {
+      "minimal_price": "mp",
+      "has_options": "ho",
+      "url_key": "u",
+      "status": "s",
+      "required_options": "ro",
+      "name": "nm",
+      "tax_class_id": "tci",
+      "description": "desc",
+      "minimal_regular_price": "mrp",
+      "final_price": "fp", 
+      "price": "p", 
+      "special_price": "sp",
+      "original_final_price": "ofp",
+      "original_price": "op",
+      "original_special_price": "osp",
+      "final_price_incl_tax": "fpit", 
+      "original_price_incl_tax": "opit",
+      "price_incl_tax": "pit",
+      "special_price_incl_tax": "spit", 
+      "final_price_tax": "fpt",
+      "price_tax": "pt",
+      "special_price_tax": "spt",
+      "original_price_tax": "opt",
+      "image": "i", 
+      "small_image": "si",
+      "thumbnail": "t"
+    },
+    "filterFieldMapping": {
+      "category.name": "category.name.keyword"
+    },
+    "filterAggregationSize": {
+      "default": 10,
+      "size": 10,
+      "color": 10
+    },    
+    "priceFilterKey": "final_price",
+    "priceFilters": {
+      "ranges": [
+        { "from": 0, "to": 50 },
+        { "from": 50, "to": 100 },
+        { "from": 100, "to": 150 },
+        { "from": 150 }
+      ]
+    }    
   },
   "redis": {
     "host": "localhost",
@@ -76,7 +152,7 @@
       "tax": {
         "defaultCountry": "DE",
         "defaultRegion": "",
-        "deprecatedPriceFieldsSupport": true,
+        "deprecatedPriceFieldsSupport": false,
         "calculateServerSide": true,
         "sourcePriceIncludesTax": false,
         "finalPriceIncludesTax": true,
@@ -113,7 +189,7 @@
         "defaultRegion": "",
         "calculateServerSide": true,
         "sourcePriceIncludesTax": false,
-        "deprecatedPriceFieldsSupport": true,
+        "deprecatedPriceFieldsSupport": false,
         "finalPriceIncludesTax": true,
         "userGroupId": null,
         "useOnlyDefaultUserGroupId": false
@@ -143,7 +219,7 @@
     "usePlatformTotals": true,
     "setConfigurableProductOptions": true,
     "sourcePriceIncludesTax": false,
-    "deprecatedPriceFieldsSupport": true,
+    "deprecatedPriceFieldsSupport": false,
     "finalPriceIncludesTax": false,
     "userGroupId": null,
     "useOnlyDefaultUserGroupId": false

nodemon.json

@@ -1,7 +1,7 @@
 {
   "verbose": true,
   "debug": false,
-  "exec": "ts-node src",
+  "exec": "node -r ts-node/register src/",
   "watch": ["./src"],
   "ext": "ts, js",
   "inspect": true

package.json

@@ -96,6 +96,7 @@
     "resource-router-middleware": "^0.6.0",
     "sharp": "^0.23.4",
     "soap": "^0.25.0",
+    "storefront-query-builder": "^0.0.9",
     "syswide-cas": "latest",
     "winston": "^2.4.2"
   },
@@ -122,7 +123,7 @@
     "ts-jest": "^24.0.2",
     "ts-node": "^8.1.0",
     "tslib": "^1.9.3",
-    "typescript": "3.3.*"
+    "typescript": "3.7.*"
   },
   "bugs": {
     "url": "https://github.com/DivanteLtd/vue-storefront-api/issues"

src/api/cart.js renamed to src/api/cart.ts

@@ -4,6 +4,8 @@ import ProcessorFactory from '../processor/factory';
 import { adjustBackendProxyUrl } from '../lib/elastic'
 import cache from '../lib/cache-instance'
 import { sha3_224 } from 'js-sha3'
+import bodybuilder from 'bodybuilder'
+import { elasticsearch, SearchQuery } from 'storefront-query-builder'
 
 function _cacheStorageHandler (config, result, hash, tags) {
   if (config.server.useOutputCache && cache) {
@@ -17,7 +19,23 @@ function _cacheStorageHandler (config, result, hash, tags) {
   }
 }
 
-export default ({config, db}) => function (req, res, body) {
+function _outputFormatter (responseBody, format = 'standard') {
+  if (format === 'compact') { // simple formatter
+    delete responseBody.took
+    delete responseBody.timed_out
+    delete responseBody._shards
+    if (responseBody.hits) {
+      delete responseBody.hits.max_score
+      responseBody.total = responseBody.hits.total
+      responseBody.hits = responseBody.hits.hits.map(hit => {
+        return Object.assign(hit._source, { _score: hit._score })
+      })
+    }
+  }
+  return responseBody
+}
+
+export default ({config, db}) => async function (req, res, body) {
   let groupId = null
 
   // Request method handling: exit if not GET or POST
@@ -26,15 +44,19 @@ export default ({config, db}) => function (req, res, body) {
     throw new Error('ERROR: ' + req.method + ' request method is not supported.')
   }
 
-  let requestBody = {}
+  let responseFormat = 'standard'
+  let requestBody = req.body
   if (req.method === 'GET') {
     if (req.query.request) { // this is in fact optional
       requestBody = JSON.parse(decodeURIComponent(req.query.request))
     }
-  } else {
-    requestBody = req.body
   }
 
+  if (req.query.request_format === 'search-query') { // search query and not Elastic DSL - we need to translate it
+    requestBody = await elasticsearch.buildQueryBodyFromSearchQuery({ config, queryChain: bodybuilder(), searchQuery: new SearchQuery(requestBody) })
+  }
+  if (req.query.response_format) responseFormat = req.query.response_format
+
   const urlSegments = req.url.split('/');
 
   let indexName = ''
@@ -108,15 +130,15 @@ export default ({config, db}) => function (req, res, body) {
           resultProcessor.process(_resBody.hits.hits, groupId).then((result) => {
             _resBody.hits.hits = result
             _cacheStorageHandler(config, _resBody, reqHash, tagsArray)
-            res.json(_resBody);
+            res.json(_outputFormatter(_resBody, responseFormat));
           }).catch((err) => {
             console.error(err)
           })
         } else {
           resultProcessor.process(_resBody.hits.hits).then((result) => {
             _resBody.hits.hits = result
             _cacheStorageHandler(config, _resBody, reqHash, tagsArray)
-            res.json(_resBody);
+            res.json(_outputFormatter(_resBody, responseFormat));
           }).catch((err) => {
             console.error(err)
           })

src/api/catalog.js renamed to src/api/catalog.ts

@@ -38,7 +38,7 @@ export default ({ config, db }) => {
   api.use('/sync', sync({ config, db }))
 
   // mount the url resource
-  api.use('/url', url({ config, db }))
+  api.use('/url', url({ config }))
 
   // perhaps expose some API metadata at the root
   api.get('/', (req, res) => {

src/api/img.js renamed to src/api/img.ts

@@ -4,23 +4,23 @@ import cache from '../lib/cache-instance'
 import request from 'request'
 
 function invalidateCache (req, res) {
-  if (config.server.useOutputCache) {
+  if (config.get('server.useOutputCache')) {
     if (req.query.tag && req.query.key) { // clear cache pages for specific query tag
-      if (req.query.key !== config.server.invalidateCacheKey) {
+      if (req.query.key !== config.get('server.invalidateCacheKey')) {
         console.error('Invalid cache invalidation key')
         apiStatus(res, 'Invalid cache invalidation key', 500)
         return
       }
       console.log(`Clear cache request for [${req.query.tag}]`)
       let tags = []
       if (req.query.tag === '*') {
-        tags = config.server.availableCacheTags
+        tags = config.get('server.availableCacheTags')
       } else {
         tags = req.query.tag.split(',')
       }
       const subPromises = []
       tags.forEach(tag => {
-        if (config.server.availableCacheTags.indexOf(tag) >= 0 || config.server.availableCacheTags.find(t => {
+        if ((config.get('server.availableCacheTags') as [string]).indexOf(tag) >= 0 || (config.get('server.availableCacheTags') as [string]).find(t => {
           return tag.indexOf(t) === 0
         })) {
           subPromises.push(cache.invalidate(tag).then(() => {
@@ -36,9 +36,9 @@ function invalidateCache (req, res) {
         apiStatus(res, error, 500)
         console.error(error)
       })
-      if (config.server.invalidateCacheForwarding) { // forward invalidate request to the next server in the chain
-        if (!req.query.forwardedFrom && config.server.invalidateCacheForwardUrl) { // don't forward forwarded requests
-          request(config.server.invalidateCacheForwardUrl + req.query.tag + '&forwardedFrom=vs', {}, (err, res, body) => {
+      if (config.get('server.invalidateCacheForwarding')) { // forward invalidate request to the next server in the chain
+        if (!req.query.forwardedFrom && config.get('server.invalidateCacheForwardUrl')) { // don't forward forwarded requests
+          request(config.get('server.invalidateCacheForwardUrl') + req.query.tag + '&forwardedFrom=vs', {}, (err, res, body) => {
             if (err) { console.error(err); }
             try {
               if (body && JSON.parse(body).code !== 200) console.log(body);

src/api/index.js renamed to src/api/index.ts

@@ -43,7 +43,7 @@ export default ({ config, db }) => resource({
     console.log(JSON.stringify(incomingOrder))
 
     for (let product of req.body.products) {
-      let key = config.tax.calculateServerSide ? { priceInclTax: product.priceInclTax } : { price: product.price }
+      let key: { id?: string, sku?: string, priceInclTax?: number, price?: number } = config.tax.calculateServerSide ? { priceInclTax: product.priceInclTax } : { price: product.price }
       if (config.tax.alwaysSyncPlatformPricesOver) {
         key.id = product.id
       } else {