Skip to content
This repository was archived by the owner on May 28, 2023. It is now read-only.

Commit d3d0e78

Browse files
authored
Merge pull request #332 from DivanteLtd/feature/3367_output_cache
Output cache with tagging and cache forwarding added
2 parents 8cb7894 + 8648ff2 commit d3d0e78

File tree

12 files changed

+395
-137
lines changed

12 files changed

+395
-137
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77
## [1.11.0-rc.1] - UNRELEASED
88

99
### Added
10+
- Output cache support with tagging and cache invalidate requests forwarding - @pkarw (https://github.com/DivanteLtd/vue-storefront/issues/3367)
1011
- Constant for Mailchimp subscription status - @KonstantinSoelch (#294)
1112
- mage2vs import now has optional `--generate-unique-url-keys` parameter which defaults to `false` to enable/disable the url key generation with name and id for categories - @rain2o (#232)
1213
- `extensions/elastic-stock` module added which is a drop-in replacement for `stock`; the difference is that it's getting the stock information from Elastic, not from e-Commerce backend directly; to use it - please just make sure your `config/local.json` file has `elastic-stock` in the `registeredExtensions` collection; then please make sure in the `vue-storefront` to change the `config.stock.ednpoint` from `http://<your-api-host>/api/stock` to `http://<your-api-host>/api/ext/elastic-stock`

README.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,18 @@ NOTE: `npm` users will still have to install the dependencies individually in th
9595
## Reviews
9696
To use review feature you need to install custom module for Magento 2: [Divante ReviewApi](https://github.com/DivanteLtd/magento2-review-api)
9797

98+
## Output Cache
99+
Vue Storefront API supports output cache for catalog operations. Cache is tagged and can by dynamically invalidated. Please find the details how to configure it [in our docs](https://docs.vuestorefront.io/guide/basics/ssr-cache.html).
100+
101+
You can manually clear the Redis cache for specific tags by running the following command:
102+
103+
```bash
104+
npm run cache clear
105+
npm run cache clear -- --tag=product,category
106+
npm run cache clear -- --tag=P198
107+
npm run cache clear -- --tag=*
108+
```
109+
98110
## Running initial Magento2 import
99111

100112
Magento2 data import is now integrated into `vue-storefront-api` for simplicity. It's still managed by the [mage2vuestorefront](https://github.com/DivanteLtd/mage2vuestorefront) - added as a dependency to `vue-storefront-api`.

config/default.json

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,14 @@
22
"server": {
33
"host": "localhost",
44
"port": 8080,
5-
"searchEngine": "elasticsearch"
5+
"searchEngine": "elasticsearch",
6+
"useOutputCacheTagging": false,
7+
"useOutputCache": false,
8+
"outputCacheDefaultTtl": 86400,
9+
"availableCacheTags": ["P", "C", "T", "A", "product", "category", "attribute", "taxrule"],
10+
"invalidateCacheKey": "aeSu7aip",
11+
"invalidateCacheForwarding": false,
12+
"invalidateCacheForwardUrl": "http://localhost:3000/invalidate?key=aeSu7aip&tag="
613
},
714
"orders": {
815
"useServerQueue": false

package.json

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"start": "pm2 start ecosystem.json $PM2_ARGS",
1515
"db": "node scripts/db.js",
1616
"seo": "node scripts/seo.js",
17+
"cache": "node ./scripts/cache",
1718
"mage2vs": "node scripts/mage2vs.js",
1819
"restore": "node scripts/elastic.js restore",
1920
"restore_it": "npm run restore -- --input-file=var/catalog_it.json --output-index=vue_storefront_catalog_it && npm run db rebuild -- --indexName=vue_storefront_catalog_it",
@@ -62,6 +63,7 @@
6263
"graphql": "^0.13.1",
6364
"graphql-tools": "^1.2.1",
6465
"humps": "^1.1.0",
66+
"js-sha3": "^0.8.0",
6567
"jsonfile": "^4.0.0",
6668
"jwa": "^1.1.5",
6769
"jwt-simple": "^0.5.1",
@@ -78,6 +80,7 @@
7880
"nodemailer": "^4.6.8",
7981
"oauth-1.0a": "^1.0.1",
8082
"pm2": "^2.10.4",
83+
"redis-tag-cache": "^1.2.1",
8184
"request": "^2.85.0",
8285
"request-promise-native": "^1.0.5",
8386
"resource-router-middleware": "^0.6.0",
@@ -96,17 +99,17 @@
9699
"@typescript-eslint/parser": "^1.7.1-alpha.17",
97100
"apollo-server-express": "^1.3.6",
98101
"cpx": "^1.5.0",
99-
"jest": "^24.8.0",
100102
"eslint": "^5.0.0",
101103
"eslint-config-standard": "^11.0.0",
102104
"eslint-plugin-import": "^2.13.0",
103105
"eslint-plugin-node": "^6.0.1",
104106
"eslint-plugin-promise": "^3.7.0",
105107
"eslint-plugin-standard": "^3.1.0",
106108
"eslint-plugin-vue-storefront": "^0.0.1",
109+
"jest": "^24.8.0",
107110
"nodemon": "^1.18.7",
108-
"ts-jest": "^24.0.2",
109111
"pre-commit": "^1.2.2",
112+
"ts-jest": "^24.0.2",
110113
"ts-node": "^8.1.0",
111114
"tslib": "^1.9.3",
112115
"typescript": "3.3.*"

scripts/cache.js

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
const program = require('commander')
2+
const config = require('config')
3+
const cache = require('../src/lib/cache-instance')
4+
5+
program
6+
.command('clear')
7+
.option('-t|--tag <tag>', 'tag name, available tags: ' + config.server.availableCacheTags.join(', '), '*')
8+
.action((cmd) => { // TODO: add parallel processing
9+
if (!cmd.tag) {
10+
console.error('error: tag must be specified')
11+
process.exit(1)
12+
} else {
13+
console.log(`Clear cache request for [${cmd.tag}]`)
14+
let tags = []
15+
if (cmd.tag === '*') {
16+
tags = config.server.availableCacheTags
17+
} else {
18+
tags = cmd.tag.split(',')
19+
}
20+
const subPromises = []
21+
tags.forEach(tag => {
22+
if (config.server.availableCacheTags.indexOf(tag) >= 0 || config.server.availableCacheTags.find(t => {
23+
return tag.indexOf(t) === 0
24+
})) {
25+
subPromises.push(cache.invalidate(tag).then(() => {
26+
console.log(`Tags invalidated successfully for [${tag}]`)
27+
}))
28+
} else {
29+
console.error(`Invalid tag name ${tag}`)
30+
}
31+
})
32+
Promise.all(subPromises).then(r => {
33+
console.log(`All tags invalidated successfully [${cmd.tag}]`)
34+
process.exit(0)
35+
}).catch(error => {
36+
console.error(error)
37+
})
38+
}
39+
})
40+
41+
program.parse(process.argv)

src/api/catalog.js

Lines changed: 78 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,20 @@
11
import jwt from 'jwt-simple';
22
import request from 'request';
33
import ProcessorFactory from '../processor/factory';
4+
import cache from '../lib/cache-instance'
5+
import { sha3_224 } from 'js-sha3'
6+
7+
function _cacheStorageHandler (config, result, hash, tags) {
8+
if (config.server.useOutputCache && cache) {
9+
cache.set(
10+
'api:' + hash,
11+
result,
12+
tags
13+
).catch((err) => {
14+
console.error(err)
15+
})
16+
}
17+
}
418

519
function _updateQueryStringParameter (uri, key, value) {
620
var re = new RegExp('([?&])' + key + '=.*?(&|#|$)', 'i');
@@ -87,37 +101,71 @@ export default ({config, db}) => function (req, res, body) {
87101
pass: config.elasticsearch.password
88102
};
89103
}
104+
const s = Date.now()
105+
const reqHash = sha3_224(JSON.stringify(requestBody))
106+
const dynamicRequestHandler = () => {
107+
request({ // do the elasticsearch request
108+
uri: url,
109+
method: req.method,
110+
body: requestBody,
111+
json: true,
112+
auth: auth
113+
}, (_err, _res, _resBody) => { // TODO: add caching layer to speed up SSR? How to invalidate products (checksum on the response BEFORE processing it)
114+
if (_resBody && _resBody.hits && _resBody.hits.hits) { // we're signing up all objects returned to the client to be able to validate them when (for example order)
115+
const factory = new ProcessorFactory(config)
116+
const tagsArray = []
117+
if (config.server.useOutputCache && cache) {
118+
const tagPrefix = entityType[0].toUpperCase() // first letter of entity name: P, T, A ...
119+
tagsArray.push(entityType)
120+
_resBody.hits.hits.map(item => {
121+
if (item._source.id) { // has common identifier
122+
tagsArray.push(`${tagPrefix}${item._source.id}`)
123+
}
124+
})
125+
}
126+
127+
let resultProcessor = factory.getAdapter(entityType, indexName, req, res)
128+
129+
if (!resultProcessor) { resultProcessor = factory.getAdapter('default', indexName, req, res) } // get the default processor
130+
131+
if (entityType === 'product') {
132+
resultProcessor.process(_resBody.hits.hits, groupId).then((result) => {
133+
_resBody.hits.hits = result
134+
_cacheStorageHandler(config, _resBody, reqHash, tagsArray)
135+
res.json(_resBody);
136+
}).catch((err) => {
137+
console.error(err)
138+
})
139+
} else {
140+
resultProcessor.process(_resBody.hits.hits).then((result) => {
141+
_resBody.hits.hits = result
142+
_cacheStorageHandler(config, _resBody, reqHash, tagsArray)
143+
res.json(_resBody);
144+
}).catch((err) => {
145+
console.error(err)
146+
})
147+
}
148+
} else { // no cache storage if no results from Elastic
149+
res.json(_resBody);
150+
}
151+
});
152+
}
90153

91-
request({ // do the elasticsearch request
92-
uri: url,
93-
method: req.method,
94-
body: requestBody,
95-
json: true,
96-
auth: auth
97-
}, (_err, _res, _resBody) => { // TODO: add caching layer to speed up SSR? How to invalidate products (checksum on the response BEFORE processing it)
98-
if (_resBody && _resBody.hits && _resBody.hits.hits) { // we're signing up all objects returned to the client to be able to validate them when (for example order)
99-
const factory = new ProcessorFactory(config)
100-
let resultProcessor = factory.getAdapter(entityType, indexName, req, res)
101-
102-
if (!resultProcessor) { resultProcessor = factory.getAdapter('default', indexName, req, res) } // get the default processor
103-
104-
if (entityType === 'product') {
105-
resultProcessor.process(_resBody.hits.hits, groupId).then((result) => {
106-
_resBody.hits.hits = result
107-
res.json(_resBody);
108-
}).catch((err) => {
109-
console.error(err)
110-
})
154+
if (config.server.useOutputCache && cache) {
155+
cache.get(
156+
'api:' + reqHash
157+
).then(output => {
158+
if (output !== null) {
159+
res.setHeader('X-VS-Cache', 'Hit')
160+
res.json(output)
161+
console.log(`cache hit [${req.url}], cached request: ${Date.now() - s}ms`)
111162
} else {
112-
resultProcessor.process(_resBody.hits.hits).then((result) => {
113-
_resBody.hits.hits = result
114-
res.json(_resBody);
115-
}).catch((err) => {
116-
console.error(err)
117-
})
163+
res.setHeader('X-VS-Cache', 'Miss')
164+
console.log(`cache miss [${req.url}], request: ${Date.now() - s}ms`)
165+
dynamicRequestHandler()
118166
}
119-
} else {
120-
res.json(_resBody);
121-
}
122-
});
167+
}).catch(err => console.error(err))
168+
} else {
169+
dynamicRequestHandler()
170+
}
123171
}

0 commit comments

Comments
 (0)