Skip to content
This repository was archived by the owner on Mar 13, 2025. It is now read-only.

Commit 96e4e9b

Browse files
xxcxyxxcxy
committed
Create db migrations
1 parent ebdd993 commit 96e4e9b

38 files changed

+832
-220
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ Setup your Elasticsearch instance and ensure that it is up and running.
6767

6868
## Working with mock data
6969

70-
You can use the scripts `npm run insert-data` (and `npm run delete-data`) to insert mock data (and delete mock data respectively). The data is inserted into Postgres and Elasticsearch. You need to setup the configurations beforehand and also start the elasticsearch instance before you run these scripts. You can run the script `npm run migrate-db-to-es` to dump data in db into es.
70+
You can use the scripts `npm run migrations up` (and `npm run migrations down`) to insert mock data (and delete mock data respectively). The data is inserted into Postgres and Elasticsearch. You need to setup the configurations beforehand and also start the elasticsearch instance before you run these scripts. You can run the script `npm run migrate-db-to-es` to dump data in db into es.
7171

7272
## Local Deployment with Docker
7373

‎package-lock.json‎

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
"lint": "standard \"**/*.js\"",
99
"insert-data": "node scripts/db/genData.js",
1010
"delete-data": "node scripts/db/dropAll.js",
11-
"migrate-db-to-es": "node scripts/db/dumpDbToEs.js"
11+
"migrate-db-to-es": "node scripts/db/dumpDbToEs.js",
12+
"migrations": "node scripts/db/migrations.js"
1213
},
1314
"repository": {
1415
"type": "git",
@@ -36,6 +37,7 @@
3637
"swagger-ui-express": "^4.1.4",
3738
"tc-bus-api-wrapper": "github:topcoder-platform/tc-bus-api-wrapper",
3839
"tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6.4",
40+
"umzug": "^2.3.0",
3941
"uuid": "^7.0.1",
4042
"winston": "^3.2.1"
4143
},

‎scripts/db/data/ExternalProfile.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
"uri": "http://www.new.com/new-uri"
2525
},
2626
{
27-
"id": "f2d1b567-8ea3-4eec-93b0-32378a19edb7",
27+
"id": "f2d1b567-8eb3-4eec-93b0-32378a19edb7",
2828
"created": "2020-05-13T06:11:21.361Z",
2929
"updated": "2020-05-13T06:46:15.893Z",
3030
"createdBy": "tc-Admin",
@@ -36,7 +36,7 @@
3636
"uri": "http://www.new.com/new-uri"
3737
},
3838
{
39-
"id": "f2d1b567-8ea3-4eec-93b0-32378a19edb7",
39+
"id": "f2d1b567-8ec3-4eec-93b0-32378a19edb7",
4040
"created": "2020-05-13T06:11:21.361Z",
4141
"updated": "2020-05-13T06:46:15.893Z",
4242
"createdBy": "tc-Admin",

‎scripts/db/genData.js‎

Lines changed: 0 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,10 @@
22

33
const _ = require('lodash')
44
const sequelize = require('../../src/models/index')
5-
const dbHelper = require('../../src/common/db-helper')
65
const logger = require('../../src/common/logger')
76
const { getESClient } = require('../../src/common/es-client')
87
const {
98
topResources,
10-
userResources,
119
organizationResources,
1210
modelToESIndexMapping
1311
} = require('../constants')
@@ -32,98 +30,6 @@ const RESOURCES_IN_ORDER = [
3230

3331
const client = getESClient()
3432

35-
async function insertIntoES (modelName, body) {
36-
const esResourceName = modelToESIndexMapping[modelName]
37-
38-
if (!esResourceName) {
39-
logger.error(`Cannot insert data into model ${modelName}. No equivalent elasticsearch index found`)
40-
41-
return
42-
}
43-
44-
if (_.includes(_.keys(topResources), esResourceName)) {
45-
await client.index({
46-
index: topResources[esResourceName].index,
47-
type: topResources[esResourceName].type,
48-
id: body.id,
49-
body,
50-
pipeline: topResources[esResourceName].ingest ? topResources[esResourceName].ingest.pipeline.id : undefined,
51-
refresh: 'wait_for'
52-
})
53-
} else if (_.includes(_.keys(userResources), esResourceName)) {
54-
const userResource = userResources[esResourceName]
55-
56-
const { body: user } = await client.getSource({
57-
index: topResources.user.index,
58-
type: topResources.user.type,
59-
id: body.userId
60-
})
61-
62-
if (userResource.nested === true && userResource.mappingCreated !== true) {
63-
await client.indices.putMapping({
64-
index: topResources.user.index,
65-
type: topResources.user.type,
66-
include_type_name: true,
67-
body: {
68-
properties: {
69-
[userResource.propertyName]: {
70-
type: 'nested'
71-
}
72-
}
73-
}
74-
})
75-
userResource.mappingCreated = true
76-
}
77-
78-
const relateId = body[userResource.relateKey]
79-
80-
if (!user[userResource.propertyName]) {
81-
user[userResource.propertyName] = []
82-
}
83-
84-
if (_.some(user[userResource.propertyName], [userResource.relateKey, relateId])) {
85-
logger.error(`Can't create existing ${esResourceName} with the ${userResource.relateKey}: ${relateId}, userId: ${body.userId}`)
86-
} else {
87-
user[userResource.propertyName].push(body)
88-
await client.index({
89-
index: topResources.user.index,
90-
type: topResources.user.type,
91-
id: body.userId,
92-
body: user,
93-
pipeline: topResources.user.pipeline.id,
94-
refresh: 'wait_for'
95-
})
96-
}
97-
} else if (_.includes(_.keys(organizationResources), esResourceName)) {
98-
const orgResource = organizationResources[esResourceName]
99-
100-
const { body: organization } = await client.getSource({
101-
index: topResources.organization.index,
102-
type: topResources.organization.type,
103-
id: body.organizationId
104-
})
105-
106-
const relateId = body[orgResource.relateKey]
107-
108-
if (!organization[orgResource.propertyName]) {
109-
organization[orgResource.propertyName] = []
110-
}
111-
112-
if (_.some(organization[orgResource.propertyName], [orgResource.relateKey, relateId])) {
113-
logger.error(`Can't create existing ${esResourceName} with the ${orgResource.relateKey}: ${relateId}, organizationId: ${body.organizationId}`)
114-
} else {
115-
organization[orgResource.propertyName].push(body)
116-
await client.index({
117-
index: topResources.organization.index,
118-
type: topResources.organization.type,
119-
id: body.organizationId,
120-
body: organization,
121-
refresh: 'wait_for'
122-
})
123-
}
124-
}
125-
}
126-
12733
/**
12834
* Creates and executes the enrich policy for the provided model
12935
* @param {String} modelName The model name
@@ -219,10 +125,6 @@ async function createEnrichProcessor (modelName) {
219125
* @return {Promise<void>}
220126
*/
221127
async function main () {
222-
await dbHelper.createDb()
223-
await sequelize.sync()
224-
dbHelper.addRelationships(sequelize)
225-
226128
let keys = Object.keys(sequelize.models)
227129

228130
// Sort the models in the order of insertion (for correct enrichment)
@@ -238,20 +140,6 @@ async function main () {
238140

239141
for (let i = 0; i < keys.length; i++) {
240142
const key = keys[i]
241-
try {
242-
const data = require(`./data/${key}.json`)
243-
for (let i = 0; i < data.length; i++) {
244-
logger.info(`Inserting data ${i + 1} of ${data.length}`)
245-
await dbHelper.create(sequelize.models[key], data[i], null)
246-
// await insertIntoES(key, data[i])
247-
}
248-
logger.info('import data for ' + key + ' done')
249-
} catch (e) {
250-
logger.error(e)
251-
logger.warn('import data for ' + key + ' failed')
252-
continue
253-
}
254-
255143
try {
256144
await createAndExecuteEnrichPolicy(key)
257145
logger.info('create and execute enrich policy for ' + key + ' done')

‎scripts/db/migrations.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
const sequelize = require('../../src/models/index')
2+
const path = require('path')
3+
const Umzug = require('umzug')
4+
const { createDb } = require('../../src/common/db-helper')
5+
6+
const umzug = new Umzug({
7+
migrations: {
8+
// indicates the folder containing the migration .js files
9+
path: path.join(__dirname, './migrations'),
10+
// inject sequelize's QueryInterface in the migrations
11+
params: [
12+
sequelize.getQueryInterface()
13+
]
14+
},
15+
// indicates that the migration data should be store in the database
16+
// itself through sequelize. The default configuration creates a table
17+
// named `SequelizeMeta`.
18+
storage: 'sequelize',
19+
storageOptions: {
20+
sequelize: sequelize
21+
}
22+
});
23+
24+
(async () => {
25+
if (process.argv[2] === 'up') {
26+
createDb()
27+
await umzug.up()
28+
console.log('All migrations performed successfully')
29+
} else if (process.argv[2] === 'down') {
30+
await umzug.down()
31+
console.log('The last executed migration reverted successfully')
32+
} else {
33+
console.error('You should pass \'up\' or \'down\' as the parameter')
34+
}
35+
})()

‎scripts/db/migrations/00_create-achievement.js‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/**
2+
* Achievement model
3+
*/
4+
const { DataTypes } = require('sequelize')
5+
6+
module.exports = {
7+
up: async (query) => {
8+
await query.createTable('Achievements', {
9+
id: {
10+
primaryKey: true,
11+
type: DataTypes.UUID,
12+
defaultValue: DataTypes.UUIDV4
13+
},
14+
createdBy: {
15+
type: DataTypes.STRING
16+
},
17+
updatedBy: {
18+
type: DataTypes.STRING
19+
},
20+
name: {
21+
type: DataTypes.STRING
22+
},
23+
uri: {
24+
type: DataTypes.STRING
25+
},
26+
certifierId: {
27+
type: DataTypes.STRING
28+
},
29+
certifiedDate: {
30+
type: DataTypes.DATE
31+
},
32+
created: {
33+
type: DataTypes.DATE,
34+
allowNull: false
35+
},
36+
updated: {
37+
type: DataTypes.DATE
38+
}
39+
})
40+
},
41+
down: async (query) => {
42+
await query.dropTable('Achievements')
43+
}
44+
}

‎scripts/db/migrations/01_create-achievementsProvider.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/**
2+
* AchievementsProvider model
3+
*/
4+
const { DataTypes } = require('sequelize')
5+
6+
module.exports = {
7+
up: async (query) => {
8+
await query.createTable('AchievementsProviders', {
9+
id: {
10+
primaryKey: true,
11+
type: DataTypes.UUID,
12+
defaultValue: DataTypes.UUIDV4
13+
},
14+
createdBy: {
15+
type: DataTypes.STRING
16+
},
17+
updatedBy: {
18+
type: DataTypes.STRING
19+
},
20+
name: {
21+
type: DataTypes.STRING
22+
},
23+
created: {
24+
type: DataTypes.DATE,
25+
allowNull: false
26+
},
27+
updated: {
28+
type: DataTypes.DATE
29+
}
30+
})
31+
},
32+
down: async (query) => {
33+
await query.dropTable('AchievementsProviders')
34+
}
35+
}

‎scripts/db/migrations/02_create-attribute.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/**
2+
* Attribute model
3+
*/
4+
const { DataTypes } = require('sequelize')
5+
6+
module.exports = {
7+
up: async (query) => {
8+
await query.createTable('Attributes', {
9+
id: {
10+
primaryKey: true,
11+
type: DataTypes.UUID,
12+
defaultValue: DataTypes.UUIDV4
13+
},
14+
createdBy: {
15+
type: DataTypes.STRING
16+
},
17+
updatedBy: {
18+
type: DataTypes.STRING
19+
},
20+
name: {
21+
type: DataTypes.STRING
22+
},
23+
created: {
24+
type: DataTypes.DATE,
25+
allowNull: false
26+
},
27+
updated: {
28+
type: DataTypes.DATE
29+
}
30+
})
31+
},
32+
down: async (query) => {
33+
await query.dropTable('Attributes')
34+
}
35+
}

‎scripts/db/migrations/03_create-attributeGroup.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/**
2+
* AttributeGroup model
3+
*/
4+
const { DataTypes } = require('sequelize')
5+
6+
module.exports = {
7+
up: async (query) => {
8+
await query.createTable('AttributeGroups', {
9+
id: {
10+
primaryKey: true,
11+
type: DataTypes.UUID,
12+
defaultValue: DataTypes.UUIDV4
13+
},
14+
createdBy: {
15+
type: DataTypes.STRING
16+
},
17+
updatedBy: {
18+
type: DataTypes.STRING
19+
},
20+
name: {
21+
type: DataTypes.STRING
22+
},
23+
created: {
24+
type: DataTypes.DATE,
25+
allowNull: false
26+
},
27+
updated: {
28+
type: DataTypes.DATE
29+
}
30+
})
31+
},
32+
down: async (query) => {
33+
await query.dropTable('AttributeGroups')
34+
}
35+
}

0 commit comments

Comments
 (0)