Skip to content

Commit b27b334

Browse files
huangqunparthshah
huangqun
authored and
parthshah
committed
changes to enhance project search using postgres sql text search (#26)
* changes to enhance project search using postgres sql text search * move the code that creates index to a separate migration sql script
1 parent e2ba44b commit b27b334

File tree

5 files changed

+576
-546
lines changed

5 files changed

+576
-546
lines changed

‎src/index.js‎

Lines changed: 44 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,44 @@
1-
'use strict'
2-
3-
// include newrelic
4-
if (process.env.NODE_ENV !== 'test' && process.env.NODE_ENV !== 'local') {
5-
require('newrelic')
6-
}
7-
8-
const app = require('./app')
9-
10-
11-
/**
12-
* Handle server shutdown gracefully
13-
*/
14-
function gracefulShutdown() {
15-
app.services.pubsub.disconnect()
16-
.then(()=> {
17-
app.logger.info('Gracefully shutting down server')
18-
process.exit()
19-
}).catch((err) => {
20-
app.logger.error(err)
21-
})
22-
// if after
23-
setTimeout(function() {
24-
app.logger.error("Could not close connections in time, forcefully shutting down");
25-
process.exit()
26-
}, 10*1000);
27-
}
28-
process.on('SIGTERM', gracefulShutdown)
29-
process.on('SIGINT', gracefulShutdown)
30-
31-
// =======================
32-
// start the server ======
33-
// =======================
34-
var port = process.env.PORT || 3000 // used to create, sign, and verify tokens
35-
36-
var server = app.listen(port, () => {
37-
app.logger.info("Starting server on PORT: %d", port)
38-
let authz = require('tc-core-library-js').Authorizer
39-
app.logger.info("Registered Policies", authz.getRegisteredPolicies())
40-
require('express-list-routes')({prefix: '', spacer: 7}, 'APIs:', app.routerRef)
41-
})
42-
43-
module.exports = server
1+
'use strict'
2+
3+
import models from './models'
4+
5+
// include newrelic
6+
if (process.env.NODE_ENV !== 'test' && process.env.NODE_ENV !== 'local') {
7+
require('newrelic')
8+
}
9+
10+
const app = require('./app')
11+
12+
/**
13+
* Handle server shutdown gracefully
14+
*/
15+
function gracefulShutdown() {
16+
app.services.pubsub.disconnect()
17+
.then(()=> {
18+
app.logger.info('Gracefully shutting down server')
19+
process.exit()
20+
}).catch((err) => {
21+
app.logger.error(err)
22+
})
23+
// if after
24+
setTimeout(function() {
25+
app.logger.error("Could not close connections in time, forcefully shutting down");
26+
process.exit()
27+
}, 10*1000);
28+
}
29+
process.on('SIGTERM', gracefulShutdown)
30+
process.on('SIGINT', gracefulShutdown)
31+
32+
// =======================
33+
// start the server ======
34+
// =======================
35+
var port = process.env.PORT || 3000 // used to create, sign, and verify tokens
36+
37+
var server = app.listen(port, () => {
38+
app.logger.info("Starting server on PORT: %d", port)
39+
let authz = require('tc-core-library-js').Authorizer
40+
app.logger.info("Registered Policies", authz.getRegisteredPolicies())
41+
require('express-list-routes')({prefix: '', spacer: 7}, 'APIs:', app.routerRef)
42+
})
43+
44+
module.exports = server

‎src/models/project.js‎

Lines changed: 155 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,100 +1,155 @@
1-
'use strict'
2-
import { PROJECT_TYPE, PROJECT_STATUS, PROJECT_MEMBER_ROLE } from '../constants'
3-
import _ from 'lodash'
4-
5-
module.exports = function(sequelize, DataTypes) {
6-
var Project = sequelize.define('Project', {
7-
id: { type: DataTypes.BIGINT, primaryKey: true, autoIncrement: true },
8-
directProjectId: DataTypes.BIGINT,
9-
billingAccountId: DataTypes.BIGINT,
10-
name: { type: DataTypes.STRING, allowNull: false },
11-
description: DataTypes.TEXT,
12-
external: DataTypes.JSON,
13-
bookmarks: DataTypes.JSON,
14-
utm: { type: DataTypes.JSON, allowNull: true },
15-
estimatedPrice: { type: DataTypes.DECIMAL(10,2), allowNull: true },
16-
actualPrice: { type: DataTypes.DECIMAL(10,2), allowNull: true},
17-
terms: {
18-
type: DataTypes.ARRAY(DataTypes.INTEGER),
19-
allowNull: false,
20-
defaultValue: []
21-
},
22-
type: {
23-
type: DataTypes.STRING,
24-
allowNull: false,
25-
validate: {
26-
isIn: [_.values(PROJECT_TYPE)]
27-
}
28-
},
29-
status: {
30-
type: DataTypes.STRING,
31-
allowNull: false,
32-
validate: {
33-
isIn: [_.values(PROJECT_STATUS)]
34-
}
35-
},
36-
details: { type: DataTypes.JSON },
37-
challengeEligibility: DataTypes.JSON,
38-
deletedAt: { type: DataTypes.DATE, allowNull: true },
39-
createdAt: { type: DataTypes.DATE, defaultValue: DataTypes.NOW },
40-
updatedAt: { type: DataTypes.DATE, defaultValue: DataTypes.NOW },
41-
createdBy: { type: DataTypes.INTEGER, allowNull: false },
42-
updatedBy: { type: DataTypes.INTEGER, allowNull: false }
43-
}, {
44-
tableName: 'projects',
45-
timestamps: true,
46-
updatedAt: 'updatedAt',
47-
createdAt: 'createdAt',
48-
deletedAt: 'deletedAt',
49-
indexes: [
50-
{ fields: ['createdAt'] },
51-
{ fields: ['name'] },
52-
{ fields: ['type'] },
53-
{ fields: ['status'] },
54-
{ fields: ['directProjectId'] }
55-
],
56-
classMethods: {
57-
/*
58-
* @Co-pilots should be able to view projects any of the following conditions are met:
59-
* a. they are registered active project members on the project
60-
* b. any project that is in 'reviewed' state AND does not yet have a co-pilot assigned
61-
* @param userId the id of user
62-
*/
63-
getProjectIdsForCopilot: function(userId) {
64-
return this.findAll({
65-
where: {
66-
$or: [
67-
['EXISTS(SELECT * FROM "project_members" WHERE "deletedAt" IS NULL AND "projectId" = "Project".id AND "userId" = ? )', userId],
68-
['"Project".status=? AND NOT EXISTS(SELECT * FROM "project_members" WHERE "deletedAt" IS NULL AND "projectId" = "Project".id AND "role" = ? )',
69-
PROJECT_STATUS.REVIEWED, PROJECT_MEMBER_ROLE.COPILOT]
70-
]
71-
},
72-
attributes:['id'],
73-
raw: true
74-
})
75-
.then((res) => {
76-
return _.map(res, 'id')
77-
})
78-
},
79-
/**
80-
* Get direct project id
81-
* @param id the id of project
82-
*/
83-
getDirectProjectId: function(id) {
84-
return this.findById(id, {
85-
attributes:['directProjectId'],
86-
raw: true
87-
})
88-
.then((res) => {
89-
return res.directProjectId
90-
})
91-
},
92-
associate: (models) => {
93-
Project.hasMany(models.ProjectMember, { as : 'members', foreignKey: 'projectId' })
94-
Project.hasMany(models.ProjectAttachment, { as : 'attachments', foreignKey: 'projectId' })
95-
}
96-
}
97-
})
98-
99-
return Project
100-
}
1+
'use strict'
2+
import { PROJECT_TYPE, PROJECT_STATUS, PROJECT_MEMBER_ROLE } from '../constants'
3+
import _ from 'lodash'
4+
5+
module.exports = function(sequelize, DataTypes) {
6+
var Project = sequelize.define('Project', {
7+
id: { type: DataTypes.BIGINT, primaryKey: true, autoIncrement: true },
8+
directProjectId: DataTypes.BIGINT,
9+
billingAccountId: DataTypes.BIGINT,
10+
name: { type: DataTypes.STRING, allowNull: false },
11+
description: DataTypes.TEXT,
12+
external: DataTypes.JSON,
13+
bookmarks: DataTypes.JSON,
14+
utm: { type: DataTypes.JSON, allowNull: true },
15+
estimatedPrice: { type: DataTypes.DECIMAL(10,2), allowNull: true },
16+
actualPrice: { type: DataTypes.DECIMAL(10,2), allowNull: true},
17+
terms: {
18+
type: DataTypes.ARRAY(DataTypes.INTEGER),
19+
allowNull: false,
20+
defaultValue: []
21+
},
22+
type: {
23+
type: DataTypes.STRING,
24+
allowNull: false,
25+
validate: {
26+
isIn: [_.values(PROJECT_TYPE)]
27+
}
28+
},
29+
status: {
30+
type: DataTypes.STRING,
31+
allowNull: false,
32+
validate: {
33+
isIn: [_.values(PROJECT_STATUS)]
34+
}
35+
},
36+
details: { type: DataTypes.JSON },
37+
challengeEligibility: DataTypes.JSON,
38+
deletedAt: { type: DataTypes.DATE, allowNull: true },
39+
createdAt: { type: DataTypes.DATE, defaultValue: DataTypes.NOW },
40+
updatedAt: { type: DataTypes.DATE, defaultValue: DataTypes.NOW },
41+
createdBy: { type: DataTypes.INTEGER, allowNull: false },
42+
updatedBy: { type: DataTypes.INTEGER, allowNull: false }
43+
}, {
44+
tableName: 'projects',
45+
timestamps: true,
46+
updatedAt: 'updatedAt',
47+
createdAt: 'createdAt',
48+
deletedAt: 'deletedAt',
49+
indexes: [
50+
{ fields: ['createdAt'] },
51+
{ fields: ['name'] },
52+
{ fields: ['type'] },
53+
{ fields: ['status'] },
54+
{ fields: ['directProjectId'] }
55+
],
56+
classMethods: {
57+
/*
58+
* @Co-pilots should be able to view projects any of the following conditions are met:
59+
* a. they are registered active project members on the project
60+
* b. any project that is in 'reviewed' state AND does not yet have a co-pilot assigned
61+
* @param userId the id of user
62+
*/
63+
getProjectIdsForCopilot: function(userId) {
64+
return this.findAll({
65+
where: {
66+
$or: [
67+
['EXISTS(SELECT * FROM "project_members" WHERE "deletedAt" IS NULL AND "projectId" = "Project".id AND "userId" = ? )', userId],
68+
['"Project".status=? AND NOT EXISTS(SELECT * FROM "project_members" WHERE "deletedAt" IS NULL AND "projectId" = "Project".id AND "role" = ? )',
69+
PROJECT_STATUS.REVIEWED, PROJECT_MEMBER_ROLE.COPILOT]
70+
]
71+
},
72+
attributes:['id'],
73+
raw: true
74+
})
75+
.then((res) => {
76+
return _.map(res, 'id')
77+
})
78+
},
79+
/**
80+
* Get direct project id
81+
* @param id the id of project
82+
*/
83+
getDirectProjectId: function(id) {
84+
return this.findById(id, {
85+
attributes:['directProjectId'],
86+
raw: true
87+
})
88+
.then((res) => {
89+
return res.directProjectId
90+
})
91+
},
92+
associate: (models) => {
93+
Project.hasMany(models.ProjectMember, { as : 'members', foreignKey: 'projectId' })
94+
Project.hasMany(models.ProjectAttachment, { as : 'attachments', foreignKey: 'projectId' })
95+
},
96+
/**
97+
* Search keyword in name, description, details.utm.code
98+
* @param parameters the parameters
99+
* - filters: the filters contains keyword
100+
* - order: the order
101+
* - limit: the limit
102+
* - offset: the offset
103+
* - attributes: the attributes to get
104+
* @param log the request log
105+
* @return the result rows and count
106+
*/
107+
searchText: function(parameters, log) {
108+
// special handling for keyword filter
109+
var query = '1=1 ';
110+
if (_.has(parameters.filters, 'id')) {
111+
if (_.isObject(parameters.filters.id)) {
112+
if (parameters.filters.id['$in'].length === 0) {
113+
parameters.filters.id['$in'].push(-1)
114+
}
115+
query += `AND id IN (${parameters.filters.id['$in']}) `;
116+
} else if(_.isString(parameters.filters.id) || _.isNumber(parameters.filters.id)){
117+
query += `AND id = ${parameters.filters.id} `;
118+
}
119+
}
120+
if (_.has(parameters.filters, 'status')) {
121+
query += `AND status = '${parameters.filters.status}' `;
122+
}
123+
if (_.has(parameters.filters, 'type')) {
124+
query += `AND type = '${parameters.filters.type}' `;
125+
}
126+
if (_.has(parameters.filters, 'keyword')) {
127+
query += `AND "projectFullText" ~ lower('${parameters.filters.keyword}')`;
128+
}
129+
130+
let attributesStr = '"' + parameters.attributes.join('","') + '"';
131+
let orderStr = '"' + parameters.order[0][0] + '" ' + parameters.order[0][1];
132+
133+
// select count of projects
134+
return sequelize.query(`SELECT COUNT(1) FROM projects WHERE ${query}`,
135+
{ type: sequelize.QueryTypes.SELECT,
136+
logging: (str) => { log.debug(str); },
137+
raw: true
138+
})
139+
.then(function(count) {
140+
// select project attributes
141+
return sequelize.query(`SELECT ${attributesStr} FROM projects WHERE ${query} ORDER BY ${orderStr} LIMIT ${parameters.limit} OFFSET ${parameters.offset}`,
142+
{ type: sequelize.QueryTypes.SELECT,
143+
logging: (str) => { log.debug(str); },
144+
raw: true
145+
})
146+
.then(function(projects) {
147+
return {rows: projects, count: count.count};
148+
});
149+
});
150+
}
151+
}
152+
})
153+
154+
return Project
155+
}

‎src/routes/projects/list.js‎

Lines changed: 7 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -41,39 +41,13 @@ var _retrieveProjects = (req, criteria, sort, fields) => {
4141
let retrieveAttachments = !req.query.fields || req.query.fields.indexOf('attachments') > -1
4242
let retrieveMembers = !req.query.fields || !!fields.project_members.length
4343

44-
// special handling for keyword filter
45-
if (_.has(criteria.filters, 'keyword')) {
46-
criteria.filters.$or = [
47-
{
48-
name: {
49-
$ilike: `%${criteria.filters.keyword}%`
50-
},
51-
}, {
52-
description: {
53-
$ilike: `%${criteria.filters.keyword}%`
54-
},
55-
}, {
56-
details: {
57-
utm: {
58-
code: {
59-
$ilike: `%${criteria.filters.keyword}%`
60-
}
61-
}
62-
}
63-
}
64-
]
65-
delete criteria.filters.keyword
66-
}
67-
68-
return models.Project.findAndCountAll({
69-
logging: (str) => { req.log.debug(str)},
70-
where: criteria.filters,
71-
order,
72-
limit : criteria.limit,
73-
offset: criteria.offset,
74-
attributes: _.get(fields, 'projects', null),
75-
raw: true,
76-
})
44+
return models.Project.searchText({
45+
filters: criteria.filters,
46+
order,
47+
limit : criteria.limit,
48+
offset: criteria.offset,
49+
attributes: _.get(fields, 'projects', null)
50+
}, req.log)
7751
.then( ({rows, count}) => {
7852
const projectIds = _.map(rows, 'id')
7953
const promises = []

0 commit comments

Comments
 (0)