Merge pull request #116 from SumoLogic/hpal_SUMO-136652

SUMO-136652: Added delay in subscribing existing log groups

Author: himanshu219

loggroup-lambda-connector/sam/packaged.yaml

@@ -8,6 +8,25 @@ Globals:
   Function:
     Timeout: 300
     MemorySize: 128
+Metadata:
+  AWS::ServerlessRepo::Application:
+    Author: Sumo Logic
+    Description: Lambda Function for automatic subscription of any Sumo Logic lambda
+      function with loggroups matching an input pattern.
+    HomePageUrl: https://github.com/SumoLogic/sumologic-aws-lambda
+    Labels:
+    - lambda
+    - sumologic
+    - autosubscribe
+    - serverless
+    - loggroups
+    - cloudwatch
+    LicenseUrl: s3://appdevstore/LoggroupConnector/v1.0.2/5122657d5b9a0d3713e24d3a33eae431
+    Name: sumologic-loggroup-connector
+    ReadmeUrl: s3://appdevstore/LoggroupConnector/v1.0.2/0490ecc943c35a5d04b00813554c06ad
+    SemanticVersion: 1.0.2
+    SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/loggroup-lambda-connector
+    SpdxLicenseId: Apache-2.0
 Parameters:
   LambdaARN:
     Type: String
@@ -28,7 +47,7 @@ Resources:
   SumoLogGroupLambdaConnector:
     Type: AWS::Serverless::Function
     Properties:
-      CodeUri: s3://appdevstore/LoggroupConnector/v1.0.1/188fa5e39f6b0e3d1087902cb169327d
+      CodeUri: s3://appdevstore/LoggroupConnector/v1.0.2/06605e5d4a1e561027da885250b6d539
       Handler: loggroup-lambda-connector.handler
       Runtime: nodejs10.x
       Environment:

loggroup-lambda-connector/sam/sam_package.sh

@@ -8,9 +8,11 @@ else
     SAM_S3_BUCKET="cf-templates-5d0x5unchag-us-east-2"
     AWS_REGION="us-east-2"
 fi
-sam package --template-file template.yaml --s3-bucket $SAM_S3_BUCKET  --output-template-file packaged.yaml
+sam package --template-file template.yaml --s3-bucket $SAM_S3_BUCKET  --output-template-file packaged.yaml --s3-prefix LoggroupConnector/v1.0.2
+
+# sam deploy --template-file packaged.yaml --stack-name testingloggrpconnector --capabilities CAPABILITY_IAM --region $AWS_REGION  --parameter-overrides LambdaARN="arn:aws:lambda:us-east-1:956882708938:function:AccessVPCResourcesLambda"
+
+sam publish --template packaged.yaml --region us-east-1
+# aws cloudformation describe-stack-events --stack-name testingloggrpconnector --region $AWS_REGION
+# aws cloudformation get-template --stack-name testingloggrpconnector  --region $AWS_REGION
 
-sam deploy --template-file packaged.yaml --stack-name testingloggrpconnector --capabilities CAPABILITY_IAM --region $AWS_REGION  --parameter-overrides LambdaARN="arn:aws:lambda:us-east-1:956882708938:function:AccessVPCResourcesLambda"
-#aws cloudformation describe-stack-events --stack-name testingloggrpconnector --region $AWS_REGION
-#aws cloudformation get-template --stack-name testingloggrpconnector  --region $AWS_REGION
-# aws serverlessrepo create-application-version --region us-east-1 --application-id arn:aws:serverlessrepo:us-east-1:$AWS_ACCOUNT_ID:applications/sumologic-securityhub-connector --semantic-version 1.0.1 --template-body file://packaged.yaml

loggroup-lambda-connector/sam/template.yaml

@@ -9,6 +9,25 @@ Globals:
         Timeout: 300
         MemorySize: 128
 
+Metadata:
+  AWS::ServerlessRepo::Application:
+    Author: Sumo Logic
+    Description: Lambda Function for automatic subscription of any Sumo Logic lambda function with loggroups matching an input pattern.
+    HomePageUrl: https://github.com/SumoLogic/sumologic-aws-lambda
+    Labels:
+    - lambda
+    - sumologic
+    - autosubscribe
+    - serverless
+    - loggroups
+    - cloudwatch
+    LicenseUrl: ./LICENSE
+    Name: sumologic-loggroup-connector
+    ReadmeUrl: ../README.md
+    SemanticVersion: 1.0.2
+    SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/loggroup-lambda-connector
+    SpdxLicenseId: Apache-2.0
+
 Parameters:
     LambdaARN:
         Type : String

loggroup-lambda-connector/src/loggroup-lambda-connector.js

@@ -1,6 +1,9 @@
 var AWS = require("aws-sdk");
-
 var cwl = new AWS.CloudWatchLogs({apiVersion: '2014-03-28'});
+async function sleep(waitTimeInMs) {
+    console.log("sleeping for " + waitTimeInMs + " ms...");
+    return new Promise((resolve) => setTimeout(resolve, waitTimeInMs));
+}
 
 function subscribeToLambda(lambdaLogGroupName, lambdaArn, errorHandler) {
     var params = {
@@ -23,13 +26,17 @@ function filterLogGroups(event, logGroupRegex) {
     }
 }
 
-function subscribeExistingLogGroups(logGroups) {
+async function subscribeExistingLogGroups(logGroups) {
     var logGroupName;
     var logGroupRegex = new RegExp(process.env.LOG_GROUP_PATTERN, "i");
     var lambdaArn = process.env.LAMBDA_ARN;
     for (var i = logGroups.length - 1; i >= 0; i--) {
         logGroupName = logGroups[i].logGroupName;
         if (logGroupName.match(logGroupRegex)) {
+
+            // sleep time between calls
+            await sleep(1000*(process.env.SUBSCRIBE_DELAY_SECONDS || 2)); // 5 seconds
+
             subscribeToLambda(logGroupName, lambdaArn, (function(inner_logGroupName) { return function (err, data) {
                 if (err) {
                     console.log("Error in subscribing", inner_logGroupName, err);
@@ -47,7 +54,7 @@ function processExistingLogGroups(token, errorHandler) {
 
     var params = {
       limit: 50,
-      // logGroupNamePrefix: 'STRING_VALUE',
+      // logGroupNamePrefix: '',
       nextToken: token
     };
     var p = new Promise(function(resolve, reject) {
@@ -81,12 +88,13 @@ function processEvents(env, event, errorHandler) {
         console.log("Subscribing: ", logGroupName, env.LAMBDA_ARN);
         subscribeToLambda(logGroupName, env.LAMBDA_ARN, errorHandler);
     } else {
-        console.log("Unsubscribed: ", logGroupName, env.LAMBDA_ARN);
+        console.log("Unmatched: ", logGroupName, env.LAMBDA_ARN);
     }
 
 }
 
 exports.handler = function (event, context, callback) {
+    console.log("Invoking Log Group connector function")
     function errorHandler(err, msg) {
         if (err) {
             console.log(err, msg);