Merge pull request #4 from Moesif/add-capture-outgoing-support

Add: Support for capturing outgoing requests

Author: dgilling

README.md

@@ -263,6 +263,10 @@ options.maskContent = function(moesifEvent) {
 Type: `Boolean`
 Set to true to print debug logs if you're having integration issues. 
 
+#### __`promisedBased`__
+Type: `Boolean`
+Set to true while using aws lambda async functions. For more details, please refer to - https://docs.aws.amazon.com/lambda/latest/dg/nodejs-handler.html. 
+
 For more documentation regarding what fields and meaning,
 see below or the [Moesif Node API Documentation](https://www.moesif.com/docs/api?javascript).
 
@@ -291,6 +295,32 @@ user_id | _Recommend_ | Identifies this API call to a permanent user_id
 metadata | false | A JSON Object consisting of any custom metadata to be stored with this event.
 
 
+## Capture Outgoing
+
+If you want to capture all outgoing API calls from your Node.js app to third parties like
+Stripe or to your own dependencies, call `startCaptureOutgoing()` to start capturing.
+
+```javascript
+var moesifMiddleware = moesifExpress(options);
+moesifMiddleware.startCaptureOutgoing();
+```
+
+The same set of above options is also applied to outgoing API calls, with a few key differences:
+
+For options functions that take `req` and `res` as input arguments, the request and response objects passed in
+are not Express or Node.js req or res objects when the request is outgoing, but Moesif does mock
+some of the fields for convenience.
+Only a subset of the Node.js req/res fields are available. Specifically:
+
+- *_mo_mocked*: Set to `true` if it is a mocked request or response object (i.e. outgoing API Call)
+- *headers*: object, a mapping of header names to header values. Case sensitive
+- *url*: string. Full request URL.
+- *method*: string. Method/verb such as GET or POST.
+- *statusCode*: number. Response HTTP status code
+- *getHeader*: function. (string) => string. Reads out a header on the request. Name is case insensitive
+- *get*: function. (string) => string. Reads out a header on the request. Name is case insensitive
+- *body*: JSON object. The request body as sent to Moesif
+
 
 ## Update a Single User
 

app.js

@@ -4,6 +4,8 @@
  */
 
 const moesif = require('./lib');
+var http = require('http');
+var https = require('https');
 console.log('Loading function');
 
 const moesifOptions = {
@@ -15,7 +17,29 @@ const moesifOptions = {
     }
 };
 
+var moesifMiddleware = moesif(moesifOptions);
+moesifMiddleware.startCaptureOutgoing();
+
 exports.handler = function (event, context, callback) {
+    // Outgoing API call to third party
+    https.get(
+        {
+          host: 'jsonplaceholder.typicode.com',
+          path: '/posts/1'
+        },
+        function(res) {
+          var body = '';
+          res.on('data', function(d) {
+            body += d;
+          });
+
+          res.on('end', function() {
+            var parsed = JSON.parse(body);
+            console.log(parsed);
+          });
+        }
+      );
+
     callback(null, {
         statusCode: '200',
         body: JSON.stringify({key: 'hello world'}),
@@ -25,4 +49,17 @@ exports.handler = function (event, context, callback) {
     });
 };
 
+// Async Functions 
+// Please set promisedBased configuration flag to true while using async functions. For more details, please refer to - https://docs.aws.amazon.com/lambda/latest/dg/nodejs-handler.html.
+
+// moesifOptions.promisedBased = true; 
+
+// exports.handler = async (event, context) => {
+//   const response = {
+//     statusCode: 200,
+//     body: JSON.stringify({ message: 'hello world' })
+//   }
+//   return response
+// }
+
 exports.handler = moesif(moesifOptions, exports.handler);

lib/batcher.js

@@ -0,0 +1,33 @@
+
+
+function createBatcher(handleBatch, maxSize, maxTime) {
+    return {
+      dataArray: [],
+      // using closure, so no need to keep as part of the object.
+      // maxSize: maxSize,
+      // maxTime: maxTime,
+      add: function(data) {
+        this.dataArray.push(data);
+        if (this.dataArray.length >= maxSize) {
+          this.flush();
+        } else if (maxTime && this.dataArray.length === 1) {
+          var self = this;
+          this._timeout = setTimeout(function() {
+            self.flush();
+          }, maxTime);
+        }
+      },
+      flush: function() {
+        // note, in case the handleBatch is a
+        // delayed function, then it swaps before
+        // sending the current data.
+        clearTimeout(this._timeout);
+        this._lastFlush = Date.now();
+        var currentDataArray = this.dataArray;
+        this.dataArray = [];
+        handleBatch(currentDataArray);
+      }
+    };
+  }
+  
+  module.exports = createBatcher;

lib/dataUtils.js

@@ -0,0 +1,223 @@
+'use strict';
+
+var url = require('url');
+var hash = require('crypto-js/md5');
+var isCreditCard = require('card-validator');
+var assign = require('lodash/assign');
+
+function logMessage(debug, functionName, message) {
+  if (debug) {
+    console.log('MOESIF: [' + functionName + '] ' + message);
+  }
+};
+
+function _hashSensitive(jsonBody, debug) {
+  if (jsonBody === null) return jsonBody;
+
+  if (Array.isArray(jsonBody)) {
+    return jsonBody.map(function (item) {
+      var itemType = typeof item;
+
+      if (itemType === 'number' || itemType === 'string') {
+        var creditCardCheck = isCreditCard.number('' + item);
+        if (creditCardCheck.isValid) {
+          logMessage(debug, 'hashSensitive', 'looks like a credit card, performing hash.');
+          return hash(item).toString();
+        }
+      }
+
+      return _hashSensitive(item, debug);
+    });
+  }
+
+  if (typeof jsonBody === 'object') {
+    var returnObject = {};
+
+    Object.keys(jsonBody).forEach(function (key) {
+      var innerVal = jsonBody[key];
+      var innerValType = typeof innerVal;
+
+      if (key.toLowerCase().indexOf('password') !== -1 && typeof innerVal === 'string') {
+        logMessage(debug, 'hashSensitive', 'key is password, so hashing the value.');
+        returnObject[key] = hash(jsonBody[key]).toString();
+      } else if (innerValType === 'number' || innerValType === 'string') {
+        var creditCardCheck = isCreditCard.number('' + innerVal);
+        if (creditCardCheck.isValid) {
+          logMessage(debug, 'hashSensitive', 'a field looks like credit card, performing hash.');
+          returnObject[key] = hash(jsonBody[key]).toString();
+        } else {
+          returnObject[key] = _hashSensitive(innerVal, debug);
+        }
+      } else {
+        // recursive test for every value.
+        returnObject[key] = _hashSensitive(innerVal, debug);
+      }
+    });
+
+    return returnObject;
+  }
+
+  return jsonBody;
+}
+
+function _getUrlFromRequestOptions(options, request) {
+  if (typeof options === 'string') {
+    options = url.parse(options);
+  } else {
+    // Avoid modifying the original options object.
+    let originalOptions = options;
+    options = {};
+    if (originalOptions) {
+      Object.keys(originalOptions).forEach((key) => {
+        options[key] = originalOptions[key];
+      });
+    }
+  }
+
+  // Oddly, url.format ignores path and only uses pathname and search,
+  // so create them from the path, if path was specified
+  if (options.path) {
+    var parsedQuery = url.parse(options.path);
+    options.pathname = parsedQuery.pathname;
+    options.search = parsedQuery.search;
+  }
+
+  // Simiarly, url.format ignores hostname and port if host is specified,
+  // even if host doesn't have the port, but http.request does not work
+  // this way. It will use the port if one is not specified in host,
+  // effectively treating host as hostname, but will use the port specified
+  // in host if it exists.
+  if (options.host && options.port) {
+    // Force a protocol so it will parse the host as the host, not path.
+    // It is discarded and not used, so it doesn't matter if it doesn't match
+    var parsedHost = url.parse('http://' + options.host);
+    if (!parsedHost.port && options.port) {
+      options.hostname = options.host;
+      delete options.host;
+    }
+  }
+
+  // Mix in default values used by http.request and others
+  options.protocol = options.protocol || (request.agent && request.agent.protocol) || undefined;
+  options.hostname = options.hostname || 'localhost';
+
+  return url.format(options);
+}
+
+function _bodyToBase64(body) {
+  if (!body) {
+    return body;
+  }
+  if (Buffer.isBuffer(body)) {
+    return body.toString('base64');
+  } else if (typeof body === 'string') {
+    return Buffer.from(body).toString('base64');
+  } else if (typeof body.toString === 'function') {
+    return Buffer.from(body.toString()).toString('base64');
+  } else {
+    return '';
+  }
+}
+
+
+function _safeJsonParse(body) {
+  try {
+    if (!Buffer.isBuffer(body) &&
+      (typeof body === 'object' || Array.isArray(body))) {
+      return {
+        body: body,
+        transferEncoding: undefined
+      }
+    }
+    return {
+      body: JSON.parse(body.toString()),
+      transferEncoding: undefined
+    }
+  } catch (e) {
+    return {
+      body: _bodyToBase64(body),
+      transferEncoding: 'base64'
+    }
+  }
+}
+
+
+function _startWithJson(body) {
+
+  var str;
+  if (body && Buffer.isBuffer(body)) {
+    str = body.slice(0, 1).toString('ascii');
+  } else {
+    str = body;
+  }
+
+  if (str && typeof str === 'string') {
+    var newStr = str.trim();
+    if (newStr.startsWith('{') || newStr.startsWith('[')) {
+      return true;
+    }
+  }
+  return true;
+}
+
+function _getEventModelFromRequestandResponse(
+  requestOptions,
+  request,
+  requestTime,
+  requestBody,
+  response,
+  responseTime,
+  responseBody,
+) {
+  var logData = {};
+  logData.request = {};
+
+  logData.request.verb = typeof requestOptions === 'string' ? 'GET' : requestOptions.method || 'GET';
+  logData.request.uri = _getUrlFromRequestOptions(requestOptions, request);
+  logData.request.headers = requestOptions.headers || {};
+  logData.request.time = requestTime;
+
+  if (requestBody) {
+    var isReqBodyMaybeJson = _startWithJson(requestBody);
+
+    if (isReqBodyMaybeJson) {
+      var parsedReqBody = _safeJsonParse(requestBody);
+
+      logData.request.transferEncoding = parsedReqBody.transferEncoding;
+      logData.request.body = parsedReqBody.body;
+    } else {
+      logData.request.transferEncoding = 'base64';
+      logData.request.body = _bodyToBase64(requestBody);
+    }
+  }
+
+  logData.response = {};
+  logData.response.time = responseTime;
+  logData.response.status = (response && response.statusCode) || 599;
+  logData.response.headers = assign({}, (response && response.headers) || {});
+
+  if (responseBody) {
+    var isResBodyMaybeJson = _startWithJson(responseBody);
+
+    if (isResBodyMaybeJson) {
+      var parsedResBody = _safeJsonParse(responseBody);
+
+      logData.response.transferEncoding = parsedResBody.transferEncoding;
+      logData.response.body = parsedResBody.body;
+    } else {
+      logData.response.transferEncoding = 'base64';
+      logData.response.body = _bodyToBase64(responseBody);
+    }
+  }
+
+  return logData;
+}
+
+module.exports = {
+  getUrlFromRequestOptions: _getUrlFromRequestOptions,
+  getEventModelFromRequestandResponse: _getEventModelFromRequestandResponse,
+  safeJsonParse: _safeJsonParse,
+  startWithJson: _startWithJson,
+  bodyToBase64: _bodyToBase64,
+  hashSensitive: _hashSensitive
+};