Merge pull request #33 from SumoLogic/monty

Cloudtrail s3 to Sumo processing lambda

Author: duchatran

cloudtrail_s3/README.md

@@ -0,0 +1,64 @@
+===========================================
+Cloudtrail S3 to Sumo Logic
+===========================================
+
+Files 
+-----
+*	*cloudtrail_s3_to_sumo.js*:  node.js function to read files from an S3 bucket to a Sumo Logic hosted HTTP collector. Files in the source bucket are gzipped. The function receives S3 notifications on new files uploaded to the source S3 bucket, then reads these files, unzips them, and breakdown the records before finally sends the data to the target Sumo endpoint.
+
+## Lambda Setup 
+For the Sumo collector configuration, do not enable multiline processing or
+one message per request -- the idea is to send as many messages in one request
+as possible to Sumo and let Sumo break them apart as needed.
+
+In the AWS console, use a code entry type of 'Edit code inline' and paste in the
+code. In the Environment variable section, set the following Key to the URL provided from Sumo collector configuration.
+SUMO_ENDPOINT
+
+In configuration specify index.handler as the Handler. Specify a Role that has
+sufficient privileges to read from the *source* bucket, and invoke a lambda
+function. The code provided is tested with node runtime 6.10, Memory setting at 128MB, Timeout 10s. 
+
+Set trigger to S3 bucket create-all events.
+ 
+One can use the AWSLambdaBasicExecution and the AWSS3ReadOnlyAccess role, although it is *strongly* recommended to customize them to restrict to relevant resources in production:  
+
+<pre>
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "arn:aws:logs:*:*:*"
+    }
+  ]
+}
+</pre>
+
+AND 
+
+<pre>
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:Get*",
+        "s3:List*"
+      ],
+      "Resource": "*"
+    }
+  ]
+}
+</pre>
+
+Once the function is created, you can tie it to the source S3 bucket. From the S3 Management console, select the bucket, goto its Properties, select Events and add a Notification. From there, provide a name for the notification, select *ObjectCreated (All)* as the Events, and select *Lambda* as the *Send To* option. Finally, select the Lambda function created above and Save.
+
+KNOWN ISSUE:
+Occassionally, the function will fail with either TypeError or Socket Error. AWS has built-in retries to launch the function again with the same parameters (bucket/filename). There shouldn't be any data loss, but the function log will show those errors. Also, using Sumo to log this Lambda run is highly recommended.

cloudtrail_s3/cloudtrail_s3_to_sumo.js

@@ -0,0 +1,79 @@
+/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+//                                        CloudTrail S3 bucket log to SumoLogic                                    // 
+//               https://github.com/SumoLogic/sumologic-aws-lambda                                                 //
+//                                                                                                                 //
+//        YOU MUST CREATE A SUMO LOGIC ENDPOINT CALLED SUMO_ENDPOINT AND PASTE IN ENVIRONMENTAL VARIABLES BELOW    //
+//            https://help.sumologic.com/Send_Data/Sources/02Sources_for_Hosted_Collectors/HTTP_Source             //
+/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// SumoLogic Endpoint to post logs
+var SumoURL = process.env.SUMO_ENDPOINT;
+
+var AWS = require('aws-sdk');
+var s3 = new AWS.S3();
+var https = require('https');
+var zlib = require('zlib'); 
+var url = require('url');
+
+function s3LogsToSumo(bucket, objKey,context) {
+    var urlObject = url.parse(SumoURL);
+    var options = {
+        'hostname': urlObject.hostname,
+        'path': urlObject.pathname,
+        'method': 'POST'
+    };
+    options.headers = {
+        'X-Sumo-Name': objKey,      
+    };
+    var req = https.request(options, function(res) {
+                var body = '';
+                console.log('Status:', res.statusCode);
+                res.setEncoding('utf8');
+                res.on('data', function(chunk) { body += chunk; });
+                res.on('end', function() {
+                    console.log('Successfully processed HTTPS response');
+                    context.succeed(); 
+                });
+            });
+    var finalData = '';
+
+    if (objKey.match(/CloudTrail-Digest/)) {
+        console.log("digest file are ignored");
+        context.succeed();
+    }    
+
+    var s3Stream = s3.getObject({Bucket: bucket, Key: objKey}).createReadStream();
+    s3Stream.on('error', function() {
+        console.log(
+            'Error getting object "' + objKey + '" from bucket "' + bucket + '".  ' +
+            'Make sure they exist and your bucket is in the same region as this function.');
+        context.fail();
+    });
+   var gunzip = zlib.createGunzip();
+    s3Stream.pipe(gunzip);
+    gunzip.on('data',function(data) {
+        finalData += data.toString();
+    }).on('end',function(end){
+        // READ THE UNZIPPED CloudTrail logs
+        var records = JSON.parse(finalData);
+        console.log(records.Records.length + " cloudtrail records in this file");
+        for (var i = 0, len = records.Records.length; i < len; i++) {
+            req.write(JSON.stringify(records.Records[i]) + '\n'); 
+        } 
+        req.end();
+    }).on('error',function(error) {
+        context.fail(error);
+    });
+}
+
+exports.handler = function(event, context) {
+    //options.agent = new https.Agent(options);
+    // Validate URL has been set
+    var urlObject = url.parse(SumoURL);
+    if (urlObject.protocol != 'https:' || urlObject.host === null || urlObject.path === null) {
+        context.fail('Invalid SUMO_ENDPOINT environment variable: ' + SumoURL);
+    }
+    var bucket = event.Records[0].s3.bucket.name;
+    var objKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
+    console.log('Bucket: '+bucket + ' ObjectKey: ' + objKey);
+    s3LogsToSumo(bucket, objKey, context);
+}