Topcoder Project Service - Import and Export Data - Final Fixes

AliGebily · AliGebily · commit ee649cf9b24b · 2020-04-13T08:47:09.000+02:00
diff --git a/package.json b/package.json
@@ -27,8 +27,8 @@
     "seed": "babel-node src/tests/seed.js --presets es2015",
     "demo-data": "babel-node local/seed",
     "es-db-compare": "babel-node scripts/es-db-compare",
-    "data:export": "babel-node scripts/data/export",
-    "data:import": "babel-node scripts/data/import"
+    "data:export": "LOG_LEVEL=info babel-node scripts/data/export",
+    "data:import": "LOG_LEVEL=info babel-node scripts/data/import"
   },
   "repository": {
     "type": "git",
diff --git a/scripts/data/export/exportData.js b/scripts/data/export/exportData.js
@@ -10,9 +10,9 @@ import { dataModels, validateDataModels } from '../dataModels';
  * @return {Promise}              Returns a promise
  */
 function saveExportedData(filePath, data, logger) {
-  logger.log('Start Saving data to file....');
+  logger.info('Start Saving data to file....');
   fs.writeFileSync(filePath, JSON.stringify(data));
-  logger.log('End Saving data to file....');
+  logger.info('End Saving data to file....');
 }
 /**
  * loads data  from database and export it to specified file path
@@ -38,7 +38,7 @@ async function exportDatabaseToJson(filePath, logger) {
     const modelName = dataModels[index];
     const modelRecords = results[index][0];
     allModelsRecords[modelName] = modelRecords;
-    logger.log(
+    logger.info(
       `Records loaded for model: ${modelName} = ${modelRecords.length}`,
     );
   }
diff --git a/scripts/data/export/index.js b/scripts/data/export/index.js
@@ -1,6 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import * as readline from 'readline';
+import Promise from 'bluebird';
 import util from '../../../src/util';
 import { exportData } from './exportData';
 /**
@@ -12,24 +13,24 @@ import { exportData } from './exportData';
 function runExportData(filePath, logger) {
   exportData(filePath, logger)
     .then(() => {
-      logger.log('Successfully exported data');
+      logger.info('Successfully exported data');
       process.exit(0);
     })
     .catch((err) => {
       logger.error('Failed to export data, ERROR:', err.message || err);
       process.exit(1);
     });
 }
-
-setTimeout(() => {
-  const logger = console;
+  const logger = util.getAppLogger();
   const filePath =
     process.argv[2] === '--file' && process.argv[3]
       ? process.argv[3]
       : 'data/demo-data.json';
-  logger.log('\nScript will export data to file:', filePath);
+  logger.info('Script will export data to file:', filePath);
   // check if file exists
   if (fs.existsSync(filePath)) {
+  // We delay question for overwrite file, because the question overlaps with a warning message from sequelize module
+  Promise.delay(1).then(() => {
     const rl = readline.createInterface({
       input: process.stdin,
       output: process.stdout,
@@ -40,19 +41,19 @@ setTimeout(() => {
       (answer) => {
         rl.close();
         if (answer.toLowerCase() === 'y') {
-          logger.log('File will be overwritten.');
+          logger.info('File will be overwritten.');
           runExportData(filePath, logger);
         } else {
-          logger.log('Exit without exporting any data');
+          logger.info('Exit without exporting any data');
           process.exit(0);
         }
       },
     ); // question()
+  });
   } else {
     // get base directory of the file
     const baseDir = path.resolve(filePath, '..');
     // create directory recursively if it does not exist
     util.mkdirSyncRecursive(baseDir);
     runExportData(filePath, logger);
   }
-});
diff --git a/scripts/data/import/importData.js b/scripts/data/import/importData.js
@@ -16,6 +16,7 @@ async function writeDataToDatabase(filePath, logger) {
     // Start a transaction
     transaction = await models.sequelize.transaction();
     const jsonData = JSON.parse(fs.readFileSync(filePath).toString());
+    // we disable no-await-in-loop because we need to run insert operations sequentially to avoid FK constraints errors
     /* eslint-disable no-await-in-loop */
     for (let index = 0; index < dataModels.length; index += 1) {
       const modelName = dataModels[index];
@@ -25,21 +26,21 @@ async function writeDataToDatabase(filePath, logger) {
         await models[modelName].bulkCreate(modelRecords, {
           transaction,
         });
-        logger.log(
+        logger.info(
           `Records to save for model: ${modelName} = ${modelRecords.length}`,
         );
       } else {
-        logger.log(`No records to save for model: ${modelName}`);
+        logger.info(`No records to save for model: ${modelName}`);
       }
     }
     // commit transaction only if all things went ok
-    logger.log('committing transaction to database...');
+    logger.info('committing transaction to database...');
     await transaction.commit();
   } catch (error) {
     logger.error('Error while writing data of model:', currentModelName);
     // rollback all insert operations
     if (transaction) {
-      logger.log('rollback database transaction...');
+      logger.info('rollback database transaction...');
       transaction.rollback();
     }
     if (error.name && error.errors && error.fields) {
@@ -65,10 +66,10 @@ async function writeDataToDatabase(filePath, logger) {
  * @return {Promise}              Returns a promise
  */
 async function indexDataToES(logger) {
-  logger.log('Indexing metatdata...');
+  logger.info('Indexing metatdata...');
   await indexMetadata();
 
-  logger.log('Indexing projects data...');
+  logger.info('Indexing projects data...');
   const req = {
     logger,
     projectIdStart: 1,
@@ -78,19 +79,7 @@ async function indexDataToES(logger) {
     fields: null,
     id: 0,
   };
-  await new Promise((resolve, reject) => {
-    indexProjectsRange(
-      req,
-      null,
-      (error) => {
-        if (error) {
-          reject(error);
-        } else {
-          resolve();
-        }
-      },
-    );
-  });
+  await indexProjectsRange(req);
 }
 
 /**
diff --git a/scripts/data/import/index.js b/scripts/data/import/index.js
@@ -1,17 +1,18 @@
 import * as fs from 'fs';
+import util from '../../../src/util';
 import { importData } from './importData';
 
-const logger = console;
+const logger = util.getAppLogger();
 const filePath = (process.argv[2] === '--file' && process.argv[3]) ? process.argv[3] : 'data/demo-data.json';
 // check if file exists
 if (!fs.existsSync(filePath)) {
   logger.error('File is not existing:', filePath);
   process.exit(1);
 } else {
-  logger.log('Script will import data from file:', filePath);
+  logger.info('Script will import data from file:', filePath);
   importData(filePath, logger)
     .then(() => {
-      logger.log('Successfully imported data');
+      logger.info('Successfully imported data');
       process.exit(0);
     })
     .catch((err) => {
diff --git a/src/middlewares/performanceRequestLogger.js b/src/middlewares/performanceRequestLogger.js
@@ -11,8 +11,7 @@ module.exports = function logRequest(logger) {
   }
 
   // Use the logger with memory usage info
-  return (request, response, next) => {
-    const req = request;
+  return (req, response, next) => {
     const res = response;
     const startOpts = {
       method: req.method,
diff --git a/src/middlewares/userIdAuth.js b/src/middlewares/userIdAuth.js
@@ -9,13 +9,12 @@ const whitelistedOrigins = JSON.parse(config.get('whitelistedOriginsForUserIdAut
 
 /**
  * The userId authentication middleware.
- * @param {Object} request the request
+ * @param {Object} req the request
  * @param {Object} res the response
  * @param {Function} next the next middleware
  * @returns {Promise<void>} void
  */
-module.exports = function userIdAuth(request, res, next) { // eslint-disable-line consistent-return
-  const req = request;
+module.exports = function userIdAuth(req, res, next) { // eslint-disable-line consistent-return
   req.log.debug('Enter userIdAuth middleware');
 
   const bearerUserId = 'Bearer userId_';
diff --git a/src/middlewares/validateMilestoneTemplate.js b/src/middlewares/validateMilestoneTemplate.js
@@ -40,8 +40,7 @@ const validateMilestoneTemplate = {
    * @param {Function} next the express next middleware
    */
   // eslint-disable-next-line valid-jsdoc
-  validateRequestBody: (request, res, next) => {
-    const req = request;
+  validateRequestBody: (req, res, next) => {
     validateReference(req.body, req)
       .then(() => {
         if (req.body.sourceReference) {
@@ -110,13 +109,12 @@ const validateMilestoneTemplate = {
    * The middleware to validate milestoneTemplateId from request
    * path parameter, and set to the request params. This should be called after the validate()
    * middleware, and before the permissions() middleware.
-   * @param {Object} request the express request instance
+   * @param {Object} req the express request instance
    * @param {Object} res the express response instance
    * @param {Function} next the express next middleware
    */
   // eslint-disable-next-line valid-jsdoc
-  validateIdParam: (request, res, next) => {
-    const req = request;
+  validateIdParam: (req, res, next) => {
     models.MilestoneTemplate.findByPk(req.params.milestoneTemplateId)
       .then((milestoneTemplate) => {
         if (!milestoneTemplate) {
diff --git a/src/middlewares/validateTimeline.js b/src/middlewares/validateTimeline.js
@@ -11,8 +11,7 @@ import util from '../util';
  * @param {boolean} [validateProjectExists]
  * @returns {Promise}
  */
-async function validateReference(sourceObject, request, validateProjectExists) {
-  const req = request;
+async function validateReference(sourceObject, req, validateProjectExists) {
   // The source object refers to a project
   if (sourceObject.reference === TIMELINE_REFERENCES.PROJECT) {
     // Set projectId to the params so it can be used in the permission check middleware
@@ -138,13 +137,12 @@ const validateTimeline = {
    * The middleware to validate and get the projectId specified by the timelineId from request
    * path parameter, and set to the request params. This should be called after the validate()
    * middleware, and before the permissions() middleware.
-   * @param {Object} request the express request instance
+   * @param {Object} req the express request instance
    * @param {Object} res the express response instance
    * @param {Function} next the express next middleware
    */
   // eslint-disable-next-line valid-jsdoc
-  validateTimelineIdParam: (request, res, next) => {
-    const req = request;
+  validateTimelineIdParam: (req, res, next) => {
     models.Timeline.findByPk(req.params.timelineId)
       .then((timeline) => {
         if (!timeline) {
diff --git a/src/permissions/connectManagerOrAdmin.ops.js b/src/permissions/connectManagerOrAdmin.ops.js
@@ -6,11 +6,10 @@ import models from '../models';
 
 /**
  * Only Connect Manager, Connect Admin, and administrator are allowed to perform the operations
- * @param {Object}    request         the express request instance
+ * @param {Object}    req         the express request instance
  * @return {Promise}              returns a promise
  */
-module.exports = request => new Promise(async (resolve, reject) => {
-  const req = request;
+module.exports = req => new Promise(async (resolve, reject) => {
   const hasAccess = util.hasRoles(req, MANAGER_ROLES);
 
   if (!hasAccess) {
diff --git a/src/permissions/copilotAndAbove.js b/src/permissions/copilotAndAbove.js
@@ -7,11 +7,10 @@ import { PERMISSION } from './constants';
  * Permission to allow copilot and above roles to perform certain operations
  * - User with Topcoder admins roles should be able to perform the operations.
  * - Project members with copilot and manager Project roles should be also able to perform the operations.
- * @param {Object}    request         the express request instance
+ * @param {Object}    req         the express request instance
  * @return {Promise}              returns a promise
  */
-module.exports = request => new Promise((resolve, reject) => {
-  const req = request;
+module.exports = req => new Promise((resolve, reject) => {
   const projectId = _.parseInt(req.params.projectId);
 
   return models.ProjectMember.getActiveProjectMembers(projectId)
diff --git a/src/permissions/project.anyAuthUser.js b/src/permissions/project.anyAuthUser.js
@@ -10,8 +10,7 @@
 import _ from 'lodash';
 import models from '../models';
 
-module.exports = (request) => {
-  const req = request;
+module.exports = (req) => {
   if (_.isUndefined(req.params.projectId)) {
     return Promise.reject(new Error('Policy "project.anyAuthUser" cannot be used for route without "projectId".'));
   }
diff --git a/src/routes/admin/project-index-create.js b/src/routes/admin/project-index-create.js
@@ -1,4 +1,3 @@
-
 import _ from 'lodash';
 import config from 'config';
 import { middleware as tcMiddleware } from 'tc-core-library-js';
@@ -32,24 +31,31 @@ module.exports = [
     const docType = _.get(req, 'body.docType', ES_PROJECT_TYPE);
     const fields = req.query.fields;
     const id = req.id;
-    indexProjectsRange(
-      {
-        logger,
-        projectIdStart,
-        projectIdEnd,
-        indexName,
-        docType,
-        fields,
-        id,
-      },
-      (esIndexingBody) => {
-        res.status(200).json({
-          message: `Reindex request successfully submitted for ${
-            esIndexingBody.length / 2
-          } projects`,
-        });
-      },
-      next,
-    );
+    return indexProjectsRange(
+        {
+          logger,
+          projectIdStart,
+          projectIdEnd,
+          indexName,
+          docType,
+          fields,
+          id,
+        },
+        (esIndexingBody) => {
+          res.status(200).json({
+            message: `Reindex request successfully submitted for ${
+              esIndexingBody.length / 2
+            } projects`,
+          });
+        },
+      ).then((result) => {
+        logger.debug(`project indexed successfully (projectId: ${projectIdStart}-${projectIdEnd})`, result);
+        logger.debug(result);
+      }).catch((error) => {
+        logger.error(
+          `Error in getting project details for indexing (projectId: ${projectIdStart}-${projectIdEnd})`,
+        error);
+        next(error);
+      });
   },
 ];
diff --git a/src/routes/index.js b/src/routes/index.js
@@ -35,8 +35,7 @@ router.all(
 );
 
 router.all(
-  RegExp(`\\/${apiVersion}\\/.*`), (request, res, next) => {
-    const req = request;
+  RegExp(`\\/${apiVersion}\\/.*`), (req, res, next) => {
     // if it is an M2M call, hard code user id to a deafult value to avoid errors
     // Ideally, the m2m token should have unique userId, which may not be an actual user, as well
     const isMachineToken = _.get(req, 'authUser.isMachine', false);
diff --git a/src/routes/projectMemberInvites/update.js b/src/routes/projectMemberInvites/update.js
@@ -27,8 +27,7 @@ module.exports = [
   // handles request validations
   validate(updateMemberValidations),
   permissions('projectMemberInvite.edit'),
-  (request, res, next) => {
-    const req = request;
+  (req, res, next) => {
     const newStatus = req.body.status;
     if (newStatus === INVITE_STATUS.CANCELED) {
       const err = new Error('Cannot change invite status to “canceled”. Please, delete the invite instead.');
diff --git a/src/util.js b/src/util.js
@@ -18,6 +18,7 @@ import elasticsearch from 'elasticsearch';
 import AWS from 'aws-sdk';
 import jp from 'jsonpath';
 import Promise from 'bluebird';
+import coreLib from 'tc-core-library-js';
 import models from './models';
 
 import {
@@ -1343,6 +1344,15 @@ _.assignIn(util, {
       return curDir;
     }, initDir);
   },
+  getAppLogger: () => {
+    const appName = 'tc-projects-service';
+    return coreLib.logger({
+      name: appName,
+      level: _.get(config, 'logLevel', 'debug').toLowerCase(),
+      captureLogs: config.get('captureLogs'),
+      logentriesToken: _.get(config, 'logentriesToken', null),
+    });
+  },
 
 });
 
diff --git a/src/utils/es.js b/src/utils/es.js

Original file line number	Diff line number	Diff line change
`@@ -11,8 +11,7 @@ module.exports = function logRequest(logger) {`
`11`	`11`	`}`
`12`	`12`
`13`	`13`	`// Use the logger with memory usage info`
`14`		`- return (request, response, next) => {`
`15`		`- const req = request;`
	`14`	`+ return (req, response, next) => {`
`16`	`15`	`const res = response;`
`17`	`16`	`const startOpts = {`
`18`	`17`	`method: req.method,`