Skip to content
This repository was archived by the owner on May 28, 2023. It is now read-only.

Commit 05998f8

Browse files
authored
Merge pull request #399 from gibkigonzo/task/3948
fetch attributes based on aggregates
2 parents 530f420 + 0fd7379 commit 05998f8

File tree

7 files changed

+204
-4
lines changed

7 files changed

+204
-4
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1111
- Add url module - @gibkigonzo (#3942)
1212
- The `response_format` query parameter to the `/api/catalog` endpoint. Currently there is just one additional format supported: `response_format=compact`. When used, the response format got optimized by: a) remapping the results, removing the `_source` from the `hits.hits`; b) compressing the JSON fields names according to the `config.products.fieldsToCompact`; c) removing the JSON fields from the `product.configurable_children` when their values === parent product values; overall response size reduced over -70% - @pkarw
1313
- The support for `SearchQuery` instead of the ElasticSearch DSL as for the input to `/api/catalog` - using `storefront-query-builder` package - @pkarw - https://github.com/DivanteLtd/vue-storefront/issues/2167
14+
- Create attribute service that allows to fetch attributes with specific options - used for products aggregates - @gibkigonzo (https://github.com/DivanteLtd/vue-storefront/pull/4001, https://github.com/DivanteLtd/mage2vuestorefront/pull/99)
1415
- Add ElasticSearch client support for HTTP authentication - @cewald (#397)
1516

1617
### Fixed

config/default.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,8 @@
333333
"includeFields": [ "children_data", "id", "children_count", "sku", "name", "is_active", "parent_id", "level", "url_key" ]
334334
},
335335
"attribute": {
336-
"includeFields": [ "attribute_code", "id", "entity_type_id", "options", "default_value", "is_user_defined", "frontend_label", "attribute_id", "default_frontend_label", "is_visible_on_front", "is_visible", "is_comparable" ]
336+
"includeFields": [ "attribute_code", "id", "entity_type_id", "options", "default_value", "is_user_defined", "frontend_label", "attribute_id", "default_frontend_label", "is_visible_on_front", "is_visible", "is_comparable" ],
337+
"loadByAttributeMetadata": false
337338
},
338339
"productList": {
339340
"sort": "",

src/api/attribute/service.ts

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
2+
import TagCache from 'redis-tag-cache'
3+
import get from 'lodash/get';
4+
import cache from '../../lib/cache-instance'
5+
import { adjustQuery, getClient as getElasticClient } from './../../lib/elastic'
6+
import bodybuilder from 'bodybuilder'
7+
8+
export interface AttributeListParam {
9+
[key: string]: number[]
10+
}
11+
12+
/**
13+
* Transforms ES aggregates into valid format for AttributeService - {[attribute_code]: [bucketId1, bucketId2]}
14+
* @param body - products response body
15+
* @param config - global config
16+
* @param indexName - current indexName
17+
*/
18+
function transformAggsToAttributeListParam (aggregations): AttributeListParam {
19+
const attributeListParam: AttributeListParam = Object.keys(aggregations)
20+
.filter(key => aggregations[key].buckets.length) // leave only buckets with values
21+
.reduce((acc, key) => {
22+
const attributeCode = key.replace(/^(agg_terms_|agg_range_)|(_options)$/g, '')
23+
const bucketsIds = aggregations[key].buckets.map(bucket => bucket.key)
24+
25+
if (!acc[attributeCode]) {
26+
acc[attributeCode] = []
27+
}
28+
29+
// there can be more then one attributes for example 'agg_terms_color' and 'agg_terms_color_options'
30+
// we need to get buckets from both
31+
acc[attributeCode] = [...new Set([...acc[attributeCode], ...bucketsIds])]
32+
33+
return acc
34+
}, {})
35+
36+
return attributeListParam
37+
}
38+
39+
/**
40+
* Returns attributes from cache
41+
*/
42+
async function getAttributeFromCache (attributeCode: string, config) {
43+
if (config.server.useOutputCache && cache) {
44+
try {
45+
const res = await (cache as TagCache).get(
46+
'api:attribute-list' + attributeCode
47+
)
48+
return res
49+
} catch (err) {
50+
console.error(err)
51+
return null
52+
}
53+
}
54+
}
55+
56+
/**
57+
* Save attributes in cache
58+
*/
59+
async function setAttributeInCache (attributeList, config) {
60+
if (config.server.useOutputCache && cache) {
61+
try {
62+
await Promise.all(
63+
attributeList.map(attribute => (cache as TagCache).set(
64+
'api:attribute-list' + attribute.attribute_code,
65+
attribute
66+
))
67+
)
68+
} catch (err) {
69+
console.error(err)
70+
}
71+
}
72+
}
73+
74+
/**
75+
* Returns attribute with only needed options
76+
* @param attribute - attribute object
77+
* @param optionsIds - list of only needed options ids
78+
*/
79+
function clearAttributeOpitons (attribute, optionsIds: number[]) {
80+
const stringOptionsIds = optionsIds.map(String)
81+
return {
82+
...attribute,
83+
options: (attribute.options || []).filter(option => stringOptionsIds.includes(String(option.value)))
84+
}
85+
}
86+
87+
async function list (attributesParam: AttributeListParam, config, indexName: string): Promise<any[]> {
88+
// we start with all attributeCodes that are requested
89+
let attributeCodes = Object.keys(attributesParam)
90+
91+
// here we check if some of attribute are in cache
92+
const rawCachedAttributeList = await Promise.all(
93+
attributeCodes.map(attributeCode => getAttributeFromCache(attributeCode, config))
94+
)
95+
96+
const cachedAttributeList = rawCachedAttributeList
97+
.map((cachedAttribute, index) => {
98+
if (cachedAttribute) {
99+
const attributeOptionsIds = attributesParam[cachedAttribute.attribute_code]
100+
101+
// side effect - we want to reduce starting 'attributeCodes' if some of them are in cache
102+
attributeCodes.splice(index, 1)
103+
104+
// clear unused options
105+
return clearAttributeOpitons(cachedAttribute, attributeOptionsIds)
106+
}
107+
})
108+
// remove empty results from cache.get
109+
// this needs to be after .map because we want to have same indexes as are in attributeCodes
110+
.filter(Boolean)
111+
112+
// if all requested attributes are in cache then we can return here
113+
if (!attributeCodes.length) {
114+
return cachedAttributeList
115+
}
116+
117+
// fetch attributes for rest attributeCodes
118+
try {
119+
const query = adjustQuery({
120+
index: indexName,
121+
type: 'attribute',
122+
body: bodybuilder().filter('terms', 'attribute_code', attributeCodes).build()
123+
}, 'attribute', config)
124+
const response = await getElasticClient(config).search(query)
125+
const fetchedAttributeList = get(response.body, 'hits.hits', []).map(hit => hit._source)
126+
127+
// save atrributes in cache
128+
await setAttributeInCache(fetchedAttributeList, config)
129+
130+
// cached and fetched attributes
131+
const allAttributes = [
132+
...cachedAttributeList,
133+
...fetchedAttributeList.map(fetchedAttribute => {
134+
const attributeOptionsIds = attributesParam[fetchedAttribute.attribute_code]
135+
136+
// clear unused options
137+
return clearAttributeOpitons(fetchedAttribute, attributeOptionsIds)
138+
})
139+
]
140+
141+
return allAttributes
142+
} catch (err) {
143+
console.error(err)
144+
return []
145+
}
146+
}
147+
148+
/**
149+
* Returns only needed data for filters in vsf
150+
*/
151+
function transformToMetadata ({
152+
is_visible_on_front,
153+
is_visible,
154+
default_frontend_label,
155+
attribute_id,
156+
entity_type_id,
157+
id,
158+
is_user_defined,
159+
is_comparable,
160+
attribute_code,
161+
slug,
162+
options = []
163+
}) {
164+
return {
165+
is_visible_on_front,
166+
is_visible,
167+
default_frontend_label,
168+
attribute_id,
169+
entity_type_id,
170+
id,
171+
is_user_defined,
172+
is_comparable,
173+
attribute_code,
174+
slug,
175+
options
176+
}
177+
}
178+
179+
export default {
180+
list,
181+
transformToMetadata,
182+
transformAggsToAttributeListParam
183+
}

src/api/catalog.ts

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import ProcessorFactory from '../processor/factory';
44
import { adjustBackendProxyUrl } from '../lib/elastic'
55
import cache from '../lib/cache-instance'
66
import { sha3_224 } from 'js-sha3'
7+
import AttributeService from './attribute/service'
78
import bodybuilder from 'bodybuilder'
89
import { elasticsearch, SearchQuery } from 'storefront-query-builder'
910

@@ -125,11 +126,16 @@ export default ({config, db}) => async function (req, res, body) {
125126
let resultProcessor = factory.getAdapter(entityType, indexName, req, res)
126127

127128
if (!resultProcessor) { resultProcessor = factory.getAdapter('default', indexName, req, res) } // get the default processor
128-
129129
if (entityType === 'product') {
130-
resultProcessor.process(_resBody.hits.hits, groupId).then((result) => {
130+
resultProcessor.process(_resBody.hits.hits, groupId).then(async (result) => {
131131
_resBody.hits.hits = result
132132
_cacheStorageHandler(config, _resBody, reqHash, tagsArray)
133+
if (_resBody.aggregations && config.entities.attribute.loadByAttributeMetadata) {
134+
const attributeListParam = AttributeService.transformAggsToAttributeListParam(_resBody.aggregations)
135+
// find attribute list
136+
const attributeList = await AttributeService.list(attributeListParam, config, indexName)
137+
_resBody.attribute_metadata = attributeList.map(AttributeService.transformToMetadata)
138+
}
133139
res.json(_outputFormatter(_resBody, responseFormat));
134140
}).catch((err) => {
135141
console.error(err)

src/graphql/elasticsearch/catalog/resolver.js

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { buildQuery } from '../queryBuilder';
44
import esResultsProcessor from './processor'
55
import { getIndexName } from '../mapping'
66
import { adjustQuery } from './../../../lib/elastic'
7+
import AttributeService from './../../../api/attribute/service'
78

89
const resolver = {
910
Query: {
@@ -67,6 +68,13 @@ async function list (filter, sort, currentPage, pageSize, search, context, rootV
6768
}
6869

6970
response.aggregations = esResponse.aggregations
71+
72+
if (response.aggregations && config.entities.attribute.loadByAttributeMetadata) {
73+
const attributeListParam = AttributeService.transformAggsToAttributeListParam(response.aggregations)
74+
const attributeList = await AttributeService.list(attributeListParam, config, esIndex)
75+
response.attribute_metadata = attributeList.map(AttributeService.transformToMetadata)
76+
}
77+
7078
response.sort_fields = {}
7179
if (sortOptions.length > 0) {
7280
response.sort_fields.options = sortOptions

src/graphql/elasticsearch/catalog/schema.graphqls

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,15 @@ type Products @doc(description: "The Products object is the top-level object ret
77
total_count: Int @doc(description: "The number of products returned")
88
# filters: [LayerFilter] @doc(description: "Layered navigation filters array") // @TODO: add filters to response instead of aggregations
99
aggregations: JSON @doc(description: "Layered navigation filters array as aggregations")
10+
attribute_metadata: JSON @doc(description: "Transformed aggregations into attributes - you need to allow 'config.entities.attribute.loadByAttributeMetadata'")
1011
sort_fields: SortFields @doc(description: "An object that includes the default sort field and all available sort fields")
1112
}
1213

1314
type ESResponse {
1415
hits: JSON
1516
suggest: JSON
1617
aggregations: JSON
18+
attribute_metadata: JSON
1719
}
1820

1921
type Query {

src/graphql/elasticsearch/queryBuilder.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,5 @@ export function buildQuery ({
1919
if (search !== '') {
2020
builtQuery['min_score'] = config.get('elasticsearch.min_score')
2121
}
22-
2322
return builtQuery;
2423
}

0 commit comments

Comments
 (0)