Updates for returning large attachments immediately and continuing the S3 copy in the background

Author: jmgasper

src/routes/attachments/create.js

@@ -39,7 +39,7 @@ module.exports = [
    * Add project attachment
    * In development mode we have to mock the ec2 file transfer and file service calls
    */
-  (req, res, next) => {
+  async (req, res, next) => {
     const data = req.body;
     // default values
     const projectId = req.params.projectId;
@@ -53,64 +53,22 @@ module.exports = [
     // extract file name
     const fileName = Path.parse(data.path).base;
     // create file path
-    const path = _.join([
+    const attachmentPath = _.join([
       config.get('projectAttachmentPathPrefix'),
       data.projectId,
       config.get('projectAttachmentPathPrefix'),
       fileName,
     ], '/');
-    let newAttachment = null;
 
     const sourceBucket = data.s3Bucket;
     const sourceKey = data.path;
     const destBucket = config.get('attachmentsS3Bucket');
-    const destKey = path;
+    const destKey = attachmentPath;
 
-    if (data.type === ATTACHMENT_TYPES.LINK) {
-      // We create the record in the db and return (i.e. no need to handle transferring file between S3 buckets)
-      Promise.resolve(models.ProjectAttachment.create({
-        projectId,
-        allowedUsers,
-        createdBy: req.authUser.userId,
-        updatedBy: req.authUser.userId,
-        title: data.title,
-        size: data.size,
-        category: data.category || null,
-        description: data.description,
-        contentType: data.contentType,
-        path: data.path,
-        type: data.type,
-        tags: data.tags,
-      })).then((_link) => {
-        const link = _link.get({ plain: true });
-        req.log.debug('New Link Attachment record: ', link);
-
-        // emit the Kafka event
-        util.sendResourceToKafkaBus(
-          req,
-          EVENT.ROUTING_KEY.PROJECT_ATTACHMENT_ADDED,
-          RESOURCES.ATTACHMENT,
-          link);
-
-        res.status(201).json(link);
-        return Promise.resolve();
-      })
-        .catch((error) => {
-          req.log.error('Error adding link attachment', error);
-          const rerr = error;
-          rerr.status = rerr.status || 500;
-          next(rerr);
-        });
-    } else {
-    // don't actually transfer file in development mode if file uploading is disabled, so we can test this endpoint
-      const fileTransferPromise = (process.env.NODE_ENV !== 'development' || config.get('enableFileUpload') === 'true')
-        ? util.s3FileTransfer(req, sourceBucket, sourceKey, destBucket, destKey)
-        : Promise.resolve();
-
-      fileTransferPromise.then(() => {
-        // file copied to final destination, create DB record
-        req.log.debug('creating db file record');
-        return models.ProjectAttachment.create({
+    try {
+      if (data.type === ATTACHMENT_TYPES.LINK) {
+        // Create the record and return immediately (no file transfer needed)
+        const linkInstance = await models.ProjectAttachment.create({
           projectId,
           allowedUsers,
           createdBy: req.authUser.userId,
@@ -120,60 +78,84 @@ module.exports = [
           category: data.category || null,
           description: data.description,
           contentType: data.contentType,
-          path,
+          path: data.path,
           type: data.type,
           tags: data.tags,
         });
-      }).then((_newAttachment) => {
-        newAttachment = _newAttachment.get({ plain: true });
-        req.log.debug('New Attachment record: ', newAttachment);
-        if (process.env.NODE_ENV !== 'development' || config.get('enableFileUpload') === 'true') {
-          // retrieve download url for the response
-          req.log.debug('retrieving download url');
-          return getDownloadUrl(destBucket, path);
-        }
-        return Promise.resolve();
-      }).then((url) => {
-        if (
-          process.env.NODE_ENV !== 'development' ||
-          config.get('enableFileUpload') === 'true'
-        ) {
-          req.log.debug('Retreiving Presigned Url resp: ', url);
-          let response = _.cloneDeep(newAttachment);
-          response = _.omit(response, ['path', 'deletedAt']);
-
-          response.downloadUrl = url;
-
-          // emit the event
-          util.sendResourceToKafkaBus(
-            req,
-            EVENT.ROUTING_KEY.PROJECT_ATTACHMENT_ADDED,
-            RESOURCES.ATTACHMENT,
-            newAttachment,
-          );
-          res.status(201).json(response);
-          return Promise.resolve();
-        }
-        let response = _.cloneDeep(newAttachment);
-        response = _.omit(response, ['path', 'deletedAt']);
-        // only in development mode
-        response.downloadUrl = path;
-        // emit the event
+        const link = linkInstance.get({ plain: true });
+        req.log.debug('New Link Attachment record: ', link);
+
         util.sendResourceToKafkaBus(
           req,
           EVENT.ROUTING_KEY.PROJECT_ATTACHMENT_ADDED,
           RESOURCES.ATTACHMENT,
-          newAttachment);
-
-        res.status(201).json(response);
-        return Promise.resolve();
-      })
-        .catch((error) => {
-          req.log.error('Error adding file attachment', error);
-          const rerr = error;
-          rerr.status = rerr.status || 500;
-          next(rerr);
-        });
+          link,
+        );
+
+        res.status(201).json(link);
+        return;
+      }
+
+      const shouldTransfer = process.env.NODE_ENV !== 'development' || config.get('enableFileUpload') === 'true';
+      const downloadUrlPromise = shouldTransfer
+        ? getDownloadUrl(destBucket, destKey)
+        : Promise.resolve(destKey);
+
+      req.log.debug('creating db file record');
+      const attachmentInstance = await models.ProjectAttachment.create({
+        projectId,
+        allowedUsers,
+        createdBy: req.authUser.userId,
+        updatedBy: req.authUser.userId,
+        title: data.title,
+        size: data.size,
+        category: data.category || null,
+        description: data.description,
+        contentType: data.contentType,
+        path: destKey,
+        type: data.type,
+        tags: data.tags,
+      });
+
+      const newAttachment = attachmentInstance.get({ plain: true });
+      req.log.debug('New Attachment record: ', newAttachment);
+
+      const downloadUrl = await downloadUrlPromise;
+      req.log.debug('Retrieved presigned url for new attachment');
+
+      let response = _.cloneDeep(newAttachment);
+      response = _.omit(response, ['path', 'deletedAt']);
+      response.downloadUrl = downloadUrl;
+
+      util.sendResourceToKafkaBus(
+        req,
+        EVENT.ROUTING_KEY.PROJECT_ATTACHMENT_ADDED,
+        RESOURCES.ATTACHMENT,
+        newAttachment,
+      );
+
+      res.status(201).json(response);
+
+      if (shouldTransfer) {
+        util.s3FileTransfer(req, sourceBucket, sourceKey, destBucket, destKey)
+          .then(() => {
+            req.log.debug('File attachment copied asynchronously', { attachmentId: newAttachment.id });
+          })
+          .catch((error) => {
+            req.log.error('Async S3 file transfer failed', {
+              error: error.message,
+              stack: error.stack,
+              attachmentId: newAttachment.id,
+              source: `${sourceBucket}/${sourceKey}`,
+              destination: `${destBucket}/${destKey}`,
+            });
+          });
+      }
+    } catch (error) {
+      req.log.error('Error adding attachment', error);
+      const rerr = error;
+      rerr.status = rerr.status || 500;
+      next(rerr);
     }
   },
 ];