Skip to content

Commit 1e1368d

Browse files
Gunasekar-KGunasekar-K
authored
Merge pull request #7 from topcoder-platform/dev
chr fix with formating
2 parents f9d4f96 + 1977219 commit 1e1368d

20 files changed

+1082
-979
lines changed

‎.circleci/config.yml‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,21 +38,21 @@ builddeploy_steps: &builddeploy_steps
3838
source awsenvconf
3939
# # producer deployment
4040
# ./unsetenv.sh
41-
# ./buildenv.sh -e $DEPLOY_ENV -b ${LOGICAL_ENV}-${APPNAME}-producer-deployvar
42-
# source buildenvvar
43-
# ./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${LOGICAL_ENV}-global-appvar,${LOGICAL_ENV}-${APPNAME}-appvar -i ${APPNAME}
41+
./buildenv.sh -e $DEPLOY_ENV -b ${LOGICAL_ENV}-${APPNAME}-producer-deployvar
42+
source buildenvvar
43+
./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${LOGICAL_ENV}-global-appvar,${LOGICAL_ENV}-${APPNAME}-appvar -i ${APPNAME}
4444
# # notify deployment
4545
# rm -rf buildenvvar
4646
# ./unsetenv.sh
4747
# ./buildenv.sh -e $DEPLOY_ENV -b ${LOGICAL_ENV}-${APPNAME}-notify-deployvar
4848
# source buildenvvar
4949
# ./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${LOGICAL_ENV}-global-appvar,${LOGICAL_ENV}-${APPNAME}-notify-appvar -i ${APPNAME}
5050
# # consumer deployment
51-
# rm -rf buildenvvar
52-
# ./unsetenv.sh
53-
# ./buildenv.sh -e $DEPLOY_ENV -b ${LOGICAL_ENV}-${APPNAME}-consumer-deployvar
54-
# source buildenvvar
55-
# ./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${LOGICAL_ENV}-global-appvar,${LOGICAL_ENV}-${APPNAME}-appvar -i ${APPNAME}
51+
rm -rf buildenvvar
52+
./unsetenv.sh
53+
./buildenv.sh -e $DEPLOY_ENV -b ${LOGICAL_ENV}-${APPNAME}-consumer-deployvar
54+
source buildenvvar
55+
./master_deploy.sh -d ECS -e $DEPLOY_ENV -t latest -s ${LOGICAL_ENV}-global-appvar,${LOGICAL_ENV}-${APPNAME}-appvar -i ${APPNAME}
5656
# # without kafka dynamodb
5757
# rm -rf buildenvvar
5858
# ./unsetenv.sh
@@ -97,6 +97,7 @@ workflows:
9797
branches:
9898
only:
9999
- dev
100+
- dev-chr-fix
100101

101102
# Production builds are exectuted only on tagged commits to the
102103
# master branch.

‎config/default.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const path = require('path');
22
module.exports = {
3+
DISABLE_LOGGING: false, // If true, logging will be disabled
34
LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
45
LOG_FILE: path.join(__dirname, '../app.log'),
56
PORT: process.env.PORT || 8080,

‎package.json‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,17 @@
2727
"bluebird": "^3.5.1",
2828
"sequelize": "^5.19.0",
2929
"underscore": "^1.9.1",
30-
"winston": "^3.1.0",
30+
"winston": "^2.2.0",
3131
"is-utf8": "^0.2.1",
3232
"aws-sdk": "latest",
3333
"zlib": "latest",
3434
"url": "latest",
3535
"https": "latest",
3636
"nodemon": "latest",
37-
"topcoder-healthcheck-dropin": "^1.0.3"
37+
"topcoder-healthcheck-dropin": "^1.0.3",
38+
"util": "latest",
39+
"get-parameter-names": "^0.3.0",
40+
"lodash": "^4.17.4"
3841
},
3942
"devDependencies": {
4043
"chai": "^4.2.0",

‎src/api/audit.js‎

Lines changed: 54 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
const models = require('../models')
2-
const model=models.auditlog
2+
const model = models.auditlog
33
const Joi = require('joi')
4-
4+
55

66
producerLog.schema = Joi.object().keys({
77
SEQ_ID: Joi.string().required(),
@@ -13,25 +13,34 @@ producerLog.schema = Joi.object().keys({
1313
//add producer_log = used for data update about producer received
1414
function producerLog(payload) {
1515
const result = Joi.validate(payload, producerLog.schema)
16-
if(result.error !== null) {
16+
if (result.error !== null) {
1717
return Promise.resolve().then(function () {
1818
throw new Error('Producer log create ' + result.error)
1919
})
2020
}
21-
return model.producer_log.create(payload)
21+
return model.producer_log.create(payload)
2222
}
2323

2424
//update producer_log = used for failure update in case of success, it will not be executed
2525
function producerLog_update(payload) {
2626
const result = Joi.validate(payload, producerLog.schema)
27-
if(result.error !== null) {
27+
if (result.error !== null) {
2828
return Promise.resolve().then(function () {
2929
throw new Error('Producer log create ' + result.error)
3030
})
3131
}
3232
//return model.producer_log.update(payload)
33-
const { SEQ_ID,PODUCER_PUBLISH_RETRY_COUNT, ...change } = payload
34-
return model.producer_log.update(change, { where: { SEQ_ID: payload.SEQ_ID, PODUCER_PUBLISH_RETRY_COUNT : payload.PODUCER_PUBLISH_RETRY_COUNT }})
33+
const {
34+
SEQ_ID,
35+
PODUCER_PUBLISH_RETRY_COUNT,
36+
...change
37+
} = payload
38+
return model.producer_log.update(change, {
39+
where: {
40+
SEQ_ID: payload.SEQ_ID,
41+
PODUCER_PUBLISH_RETRY_COUNT: payload.PODUCER_PUBLISH_RETRY_COUNT
42+
}
43+
})
3544
}
3645

3746

@@ -40,7 +49,7 @@ pAuditLog.schema = Joi.object().keys({
4049
SEQ_ID: Joi.string().required(),
4150
REQUEST_CREATE_TIME: Joi.date(),
4251
PRODUCER_PAYLOAD: Joi.object(),
43-
PRODUCER_PUBLISH_STATUS: Joi.string().valid('success','failure'),
52+
PRODUCER_PUBLISH_STATUS: Joi.string().valid('success', 'failure'),
4453
PRODUCER_FAILURE_LOG: Joi.object(),
4554
PRODUCER_PUBLISH_TIME: Joi.date(),
4655
PODUCER_PUBLISH_RETRY_COUNT: Joi.number(),
@@ -51,7 +60,7 @@ pAuditLog.schema = Joi.object().keys({
5160
//add audit_log about the producer details
5261
function pAuditLog(payload) {
5362
const result = Joi.validate(payload, pAuditLog.schema)
54-
if(result.error !== null) {
63+
if (result.error !== null) {
5564
return Promise.resolve().then(function () {
5665
throw new Error('Audit' + result.error)
5766
})
@@ -62,14 +71,21 @@ function pAuditLog(payload) {
6271
//updated audit_log about the producer status
6372
function pAuditLog_update(payload) {
6473
const result = Joi.validate(payload, pAuditLog.schema)
65-
if(result.error !== null) {
74+
if (result.error !== null) {
6675
return Promise.resolve().then(function () {
6776
throw new Error('Audit' + result.error)
6877
})
6978
}
7079
//return model.audit_log.create(payload)
71-
const { SEQ_ID, ...change } = payload
72-
return model.audit_log.update(change, { where: { SEQ_ID: payload.SEQ_ID }})
80+
const {
81+
SEQ_ID,
82+
...change
83+
} = payload
84+
return model.audit_log.update(change, {
85+
where: {
86+
SEQ_ID: payload.SEQ_ID
87+
}
88+
})
7389
}
7490

7591
consumerLog.schema = Joi.object().keys({
@@ -82,32 +98,41 @@ consumerLog.schema = Joi.object().keys({
8298
//add consumer_log = Entering received record
8399
function consumerLog(payload) {
84100
const result = Joi.validate(payload, consumerLog.schema)
85-
if(result.error !== null) {
101+
if (result.error !== null) {
86102
return Promise.resolve().then(function () {
87103
throw new Error('Consumer' + result.error)
88104
})
89105
}
90106

91-
return model.consumer_log.create(payload)
107+
return model.consumer_log.create(payload)
92108
}
93109

94110
//update consumer_log = used for failure log update
95111
function consumerLog_update(payload) {
96112
const result = Joi.validate(payload, consumerLog.schema)
97-
if(result.error !== null) {
113+
if (result.error !== null) {
98114
return Promise.resolve().then(function () {
99115
throw new Error('Consumer' + result.error)
100116
})
101117
}
102118
//return model.consumer_log.create(payload)
103-
const { SEQ_ID,CONSUMER_UPDATE_RETRY_COUNT, ...change } = payload
104-
return model.consumer_log.update(change, { where: { SEQ_ID: payload.SEQ_ID, CONSUMER_UPDATE_RETRY_COUNT : payload.CONSUMER_UPDATE_RETRY_COUNT }})
119+
const {
120+
SEQ_ID,
121+
CONSUMER_UPDATE_RETRY_COUNT,
122+
...change
123+
} = payload
124+
return model.consumer_log.update(change, {
125+
where: {
126+
SEQ_ID: payload.SEQ_ID,
127+
CONSUMER_UPDATE_RETRY_COUNT: payload.CONSUMER_UPDATE_RETRY_COUNT
128+
}
129+
})
105130
}
106131

107132
cAuditLog.schema = Joi.object().keys({
108133
SEQ_ID: Joi.string().required(),
109134
CONSUMER_PAYLOAD: Joi.object(),
110-
CONSUMER_DEPLOY_STATUS: Joi.string().valid('success','failure'),
135+
CONSUMER_DEPLOY_STATUS: Joi.string().valid('success', 'failure'),
111136
CONSUMER_FAILURE_LOG: Joi.object(),
112137
CONSUMER_UPDATE_TIME: Joi.date(),
113138
CONSUMER_UPDATE_RETRY_COUNT: Joi.number(),
@@ -117,14 +142,21 @@ cAuditLog.schema = Joi.object().keys({
117142
//add audit_log = only update is possible
118143
function cAuditLog(payload) {
119144
const result = Joi.validate(payload, cAuditLog.schema)
120-
if(result.error !== null) {
145+
if (result.error !== null) {
121146
return Promise.resolve().then(function () {
122147
throw new Error('Audit' + result.error)
123148
})
124149
}
125150

126-
const { SEQ_ID, ...change } = payload
127-
return model.audit_log.update(change, { where: { SEQ_ID: payload.SEQ_ID }})
151+
const {
152+
SEQ_ID,
153+
...change
154+
} = payload
155+
return model.audit_log.update(change, {
156+
where: {
157+
SEQ_ID: payload.SEQ_ID
158+
}
159+
})
128160
}
129161

130162
module.exports = {
@@ -135,4 +167,4 @@ module.exports = {
135167
consumerLog,
136168
consumerLog_update,
137169
cAuditLog
138-
}
170+
}

‎src/api/consumer_retry.js‎

Lines changed: 32 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,42 @@
11
const config = require('config');
2-
const {
3-
create_producer_app_log,
4-
producerpost_success_log,
5-
producerpost_failure_log
6-
} = require('../common/app_log')
2+
const logger = require('../common/logger')
3+
const app_log = require('../common/app_log')
74
const pushToKafka = require('./pushToKafka')
8-
const {
9-
postMessage,
10-
validateMsgPosted
11-
} = require('./postslackinfo')
12-
async function consumerretry(producer, payload)
13-
{
14-
payload['retryCount'] = payload.retryCount + 1;
15-
//add producer_log
16-
try {
17-
await create_producer_app_log(payload, "ConsumerRetry")
18-
} catch (error) {
19-
console.log(error)
20-
}
21-
kafka_error = await pushToKafka(producer, config.topic.NAME, payload)
5+
const slack = require('./postslackinfo')
6+
async function consumerretry(producer, payload) {
7+
payload['retryCount'] = payload.retryCount + 1;
8+
//add producer_log
9+
try {
10+
await app_log.create_producer_app_log(payload, "ConsumerRetry")
11+
} catch (error) {
12+
logger.logFullError(error)
13+
}
14+
kafka_error = await pushToKafka(producer, config.topic.NAME, payload)
15+
//add auditlog
16+
if (!kafka_error) {
17+
await app_log.producerpost_success_log(payload, "ConsumerReposted")
18+
} else {
2219
//add auditlog
20+
await app_log.producerpost_failure_log(payload, kafka_error, 'ConsumerRepostFailed')
21+
msgValue = {
22+
...kafka_error,
23+
SEQ_ID: payload.SEQ_ID,
24+
recipients: config.topic_error.EMAIL,
25+
msgoriginator: "consumer-producer"
26+
}
27+
//send error message to kafka
28+
kafka_error = await pushToKafka(producer, config.topic_error.NAME, msgValue)
2329
if (!kafka_error) {
24-
await producerpost_success_log(payload, "ConsumerReposted")
25-
//res.send('done')
26-
//res.send('done')
30+
logger.info("Kafka Message posted successfully to the topic : " + config.topic_error.NAME)
2731
} else {
28-
//add auditlog
29-
await producerpost_failure_log(payload, kafka_error, 'ConsumerRepostFailed')
30-
msgValue = {
31-
...kafka_error,
32-
SEQ_ID: payload.SEQ_ID,
33-
recipients: config.topic_error.EMAIL,
34-
msgoriginator: "consumer-producer"
35-
}
36-
//send error message to kafka
37-
kafka_error = await pushToKafka(producer, config.topic_error.NAME, msgValue)
38-
if (!kafka_error) {
39-
console.log("Kafka Message posted successfully to the topic : " + config.topic_error.NAME)
40-
} else {
41-
if (config.SLACK.SLACKNOTIFY === 'true') {
42-
await postMessage("consumer repost failed - But unable to post message in kafka error topic due to errors", async (response) => {
43-
await validateMsgPosted(response.statusCode, response.statusMessage)
44-
});
45-
}
32+
if (config.SLACK.SLACKNOTIFY === 'true') {
33+
await slack.postMessage("consumer repost failed - But unable to post message in kafka error topic due to errors", async (response) => {
34+
await slack.validateMsgPosted(response.statusCode, response.statusMessage)
35+
});
4636
}
47-
4837
}
38+
39+
}
4940
}
5041

5142
module.exports = consumerretry

‎src/api/migratedynamodb.js‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@ const _ = require('lodash')
44
var AWS = require("aws-sdk");
55
async function pushToDynamoDb(payload) {
66
try {
7-
console.log('----Inside DynomoDB code -------');
7+
logger.debug('----Inside DynomoDB code -------');
88
let seqID = payload.TIME + "_" + payload.TABLENAME
9-
// console.log(payload)
109
var params = {
1110
TableName: config.DYNAMODB.TABLENAME,
1211
Item: {
@@ -25,12 +24,16 @@ async function pushToDynamoDb(payload) {
2524
convertEmptyValues: true
2625
});
2726
docClient.put(params, function (err, data) {
28-
if (err) console.log('DynamoDB error : ', err);
29-
else console.log('DynamoDB Success : ', data);
27+
if (err) {
28+
logger.error('DynamoDB error : ')
29+
logger.logFullError(err);
30+
} else {
31+
logger.info('DynamoDB Success : ', data);
32+
}
3033
});
3134

3235
} catch (e) {
33-
console.log(e)
36+
logger.logFullError(e);
3437
}
3538
}
3639
module.exports = pushToDynamoDb

0 commit comments

Comments
 (0)