Merge pull request #520 from topcoder-platform/feature/link-attachments

Links as Attachments + Tags - Migration Scripts Optimisations

Author: maxceem

migrations/bookmarks/migrateBookmarksToLinks.js

@@ -4,6 +4,7 @@
  */
 
 import _ from 'lodash';
+import sequelize from 'sequelize';
 import models from '../../src/models';
 import { ATTACHMENT_TYPES } from '../../src/constants';
 
@@ -14,20 +15,41 @@ console.log('Migrate project.bookmarks to project.attachments for all projects i
  *
  * @returns {Promise} the DB data
  */
-const getAllProjectsFromDB = () => models.Project.findAll({ raw: false });
+const getProjectsWithBookmarks = () => models.Project.findAll({
+  raw: false,
+  attributes: ['id', 'bookmarks'],
+  where: sequelize.where(
+    sequelize.fn('json_array_length', sequelize.col('bookmarks')),
+    { [sequelize.Op.gt]: 0 },
+  ),
+});
 
 /**
  * Executes the bookmarks migration to link attachments
  * @returns {Promise}  resolved when migration is complete
  */
 const migrateBookmarks = async () => {
-  const projects = await getAllProjectsFromDB();
+  const projects = await getProjectsWithBookmarks();
+  let count = 0;
+
+  console.log(`Found ${projects.length} projects.`);
 
   for (const project of projects) {
-    const bookmarks = _.get(project, 'bookmarks');
+    await models.sequelize.transaction(async (tr) => { // eslint-disable-line no-loop-func
+      count += 1;
+      const percentage = Math.round((count / projects.length) * 100);
+
+      console.log(`Processing project id ${project.id}: ${count}/${projects.length} (${percentage}%)...`);
+
+      const bookmarks = _.get(project, 'bookmarks', []);
+      console.log(`Processing project id ${project.id}: found ${bookmarks.length} bookmarks`);
 
-    _.each(bookmarks, async (b) => {
-      await models.ProjectAttachment.create({
+      if (bookmarks.length === 0) {
+        console.log(`Processing project id ${project.id}: skipped.`);
+        return;
+      }
+
+      const attachments = bookmarks.map(b => ({
         projectId: project.id,
         type: ATTACHMENT_TYPES.LINK,
         title: b.title,
@@ -37,17 +59,25 @@ const migrateBookmarks = async () => {
         updatedAt: _.isNil(b.updatedAt) ? project.updatedAt : b.updatedAt,
         updatedBy: _.isNil(b.updatedBy) ? project.updatedBy : b.updatedBy,
         tags: [],
-      });
+      }));
+
+      await models.ProjectAttachment.bulkCreate(attachments, { transaction: tr });
+      console.log(`Processing project id ${project.id}: attachments created.`);
+
+      project.bookmarks = [];
+      await project.save({ transaction: tr });
+      console.log(`Processing project id ${project.id}: bookmarks removed.`);
+
+      console.log(`Processing project id ${project.id}: done.`);
     });
-    project.bookmarks = [];
-    await project.save();
   }
 };
 
 migrateBookmarks().then(() => {
   console.log('Migration of projects bookmarks to project links attachments finished!');
   process.exit();
 }).catch((e) => {
-  console.log(e);
+  console.error('Migration of projects bookmarks to project links attachments failed!');
+  console.error(e);
   process.exit();
 });

migrations/bookmarks/migrateLinksToBookmarks.js

@@ -14,7 +14,10 @@ console.log('Migrate project.attachments of type \'link\' to project.bookmarks f
  *
  * @returns {Promise} the DB data
  */
-const getAllProjectsFromDB = async () => models.Project.findAll({ raw: false });
+const getAllProjectsFromDB = async () => models.Project.findAll({
+  raw: false,
+  attributes: ['id', 'bookmarks'],
+});
 
 /**
  * Gets the active project links (Links that were not deleted) for the given project.
@@ -23,49 +26,72 @@ const getAllProjectsFromDB = async () => models.Project.findAll({ raw: false });
  * @returns {Promise} The active project links promise
  */
 const getActiveProjectLinks = async projectId => models.ProjectAttachment
-                 .findAll({
-                   where: {
-                     projectId,
-                     deletedAt: { $eq: null },
-                     type: ATTACHMENT_TYPES.LINK,
-                   },
-                   raw: false,
-                 });
+  .findAll({
+    where: {
+      projectId,
+      type: ATTACHMENT_TYPES.LINK,
+    },
+    raw: true,
+  });
 
 /**
  * Executes the migration of link attachments to bookmarks for all projects in the database.
  * @returns {Promise} resolved when the migration is complete
  */
 const migrateLinksToBookmarks = async () => {
   const projects = await getAllProjectsFromDB();
+  let count = 0;
+
+  console.log(`Found ${projects.length} projects in total.`);
 
   for (const project of projects) {
-      // get the project links
-    const links = await getActiveProjectLinks(project.id);
-    const bookmarks = [];
+    await models.sequelize.transaction(async (tr) => { // eslint-disable-line no-loop-func
+      count += 1;
+      const percentage = Math.round((count / projects.length) * 100);
+
+      console.log(`Processing project id ${project.id}: ${count}/${projects.length} (${percentage}%)...`);
+
+      const links = await getActiveProjectLinks(project.id);
+      console.log(`Processing project id ${project.id}: found ${links.length} link attachments`);
+
+      if (links.length === 0) {
+        console.log(`Processing project id ${project.id}: skipped.`);
+        return;
+      }
 
-    _.each(links, async (link) => {
-      bookmarks.push({
+      const bookmarks = links.map(link => ({
         title: link.title,
         address: link.path,
         createdAt: link.createdAt,
         createdBy: link.createdBy,
         updatedAt: link.updatedAt,
         updatedBy: link.updatedBy,
+      }));
+
+      project.bookmarks = bookmarks;
+      await project.save({
+        transaction: tr,
       });
+      console.log(`Processing project id ${project.id}: bookmarks created.`);
 
-      await link.destroy();
-    });
+      await models.ProjectAttachment.destroy({
+        where: {
+          id: _.map(links, 'id'),
+        },
+        transaction: tr,
+      });
+      console.log(`Processing project id ${project.id}: attachments removed.`);
 
-    project.bookmarks = bookmarks;
-    await project.save();
+      console.log(`Processing project id ${project.id}: done.`);
+    });
   }
 };
 
 migrateLinksToBookmarks().then(() => {
   console.log('Migration of projects link attachments to project bookmarks finished!');
   process.exit();
 }).catch((e) => {
+  console.error('Migration of projects link attachments to project bookmarks failed!');
   console.log(e);
   process.exit();
 });