Merge pull request #111 from SumoLogic/hpal_install_api

himanshu219 · web-flow · commit 07e7f77d28dc · 2020-02-19T16:45:12.000+05:30
Switching app install method via install API
diff --git a/cloudtrailbenchmark/packaged.yaml b/cloudtrailbenchmark/packaged.yaml
@@ -43,9 +43,9 @@ Metadata:
     - benchmark
     - cloudtrail
     Name: sumologic-aws-cloudtrail-benchmark
-    LicenseUrl: s3://appdevstore/cloudtrailbenchmark/v1.0.6/cac1a6df52c685e0f6ebe4ae72078c80
-    ReadmeUrl: s3://appdevstore/cloudtrailbenchmark/v1.0.6/8a901bb4fbbe82f128fc502dd3077508
-    SemanticVersion: 1.0.6
+    LicenseUrl: s3://appdevstore/cloudtrailbenchmark/v1.0.10/cac1a6df52c685e0f6ebe4ae72078c80
+    ReadmeUrl: s3://appdevstore/cloudtrailbenchmark/v1.0.10/8a901bb4fbbe82f128fc502dd3077508
+    SemanticVersion: 1.0.10
     SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/cloudtrailbenchmark
 Parameters:
   CollectorName:
@@ -71,8 +71,9 @@ Parameters:
     - jp
     - us2
     - us1
-    - nite
-    Description: Enter au, ca, de, eu, jp, us2, or us1
+    - in
+    - fed
+    Description: Enter au, ca, de, eu, jp, us2, in, fed or us1
   RemoveSumoResourcesOnDeleteStack:
     AllowedValues:
     - true
@@ -126,7 +127,7 @@ Resources:
     Properties:
       Location:
         ApplicationId: arn:aws:serverlessrepo:us-east-1:956882708938:applications/sumologic-app-utils
-        SemanticVersion: 1.0.22
+        SemanticVersion: 1.0.24
   SumoRole:
     Condition: SetupSumoResources
     Type: AWS::IAM::Role
@@ -346,11 +347,14 @@ Resources:
       Region:
         Ref: AWS::Region
       AppName: Global Intelligence for AWS CloudTrail
+      AppId: 570bdc0d-f824-4fcb-96b2-3230d4497180
       RemoveOnDeleteStack:
         Ref: RemoveSumoResourcesOnDeleteStack
       AppSources:
-        logsrc:
+        cloudtraillogsource:
           Fn::Sub: _sourceCategory=${SourceCategoryName}
+        indexname: '%rnd%'
+        incrementalindex: '%rnd%'
       SumoAccessID:
         Ref: SumoAccessID
       SumoAccessKey:
diff --git a/cloudtrailbenchmark/template.yaml b/cloudtrailbenchmark/template.yaml
@@ -49,7 +49,7 @@ Metadata:
     Name: sumologic-aws-cloudtrail-benchmark
     LicenseUrl: ./LICENSE
     ReadmeUrl: ./README.md
-    SemanticVersion: 1.0.6
+    SemanticVersion: 1.0.10
     SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/cloudtrailbenchmark
 
 
@@ -77,8 +77,9 @@ Parameters:
       - jp
       - us2
       - us1
-      - nite
-    Description: "Enter au, ca, de, eu, jp, us2, or us1"
+      - in
+      - fed
+    Description: "Enter au, ca, de, eu, jp, us2, in, fed or us1"
   RemoveSumoResourcesOnDeleteStack:
     AllowedValues:
       - true
@@ -122,7 +123,7 @@ Resources:
     Properties:
       Location:
         ApplicationId: arn:aws:serverlessrepo:us-east-1:956882708938:applications/sumologic-app-utils
-        SemanticVersion: 1.0.22
+        SemanticVersion: 1.0.24
 
   SumoRole:
     Condition: SetupSumoResources
@@ -310,9 +311,12 @@ Resources:
       ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction
       Region: !Ref "AWS::Region"
       AppName: "Global Intelligence for AWS CloudTrail"
+      AppId: "570bdc0d-f824-4fcb-96b2-3230d4497180"
       RemoveOnDeleteStack: !Ref RemoveSumoResourcesOnDeleteStack
       AppSources:
-        logsrc: !Sub "_sourceCategory=${SourceCategoryName}"
+        cloudtraillogsource: !Sub "_sourceCategory=${SourceCategoryName}"
+        indexname: "%rnd%"
+        incrementalindex: "%rnd%"
       SumoAccessID: !Ref SumoAccessID
       SumoAccessKey: !Ref SumoAccessKey
       SumoDeployment: !Ref SumoDeployment
diff --git a/cloudtrailbenchmark/testdeploy.sh b/cloudtrailbenchmark/testdeploy.sh
@@ -10,7 +10,7 @@ else
 fi
 uid=$(cat /dev/random | LC_CTYPE=C tr -dc "[:lower:]" | head -c 6)
 
-version="1.0.6"
+version="1.0.10"
 
 sam package --template-file template.yaml --s3-bucket $SAM_S3_BUCKET  --output-template-file packaged.yaml --s3-prefix "cloudtrailbenchmark/v$version"
 
@@ -19,9 +19,8 @@ sam publish --template packaged.yaml --region $AWS_REGION --semantic-version $ve
 
 ACCESS_ID=""
 ACCESS_KEY=""
-
 ORG_ID=""
-SUMO_DEPLOYMENT="us2"
+SUMO_DEPLOYMENT="us1"
 # DEP_TYPE="App-SumoResources-CloudTrail-S3Bucket"
 DEP_TYPE="Only-App"
 # DEP_TYPE="App-SumoResources"
diff --git a/cloudwatchevents/guardduty/cloudwatchevents.json b/cloudwatchevents/guardduty/cloudwatchevents.json
@@ -85,7 +85,7 @@
                         ]
                     },
                     "Timeout": 300,
-                    "Runtime": "nodejs8.10"
+                    "Runtime": "nodejs10.x"
                 }
             },
             "CloudWatchEventFunctionCloudWatchEventTriggerPermission": {
diff --git a/cloudwatchevents/guarddutybenchmark/packaged_v2.yaml b/cloudwatchevents/guarddutybenchmark/packaged_v2.yaml
@@ -53,9 +53,9 @@ Metadata:
     - benchmark
     - guardduty
     Name: sumologic-guardduty-benchmark
-    LicenseUrl: s3://appdevstore/guarddutybenchmark/v1.0.8/cac1a6df52c685e0f6ebe4ae72078c80
-    ReadmeUrl: s3://appdevstore/guarddutybenchmark/v1.0.8/cab012d7fb7887671b751e6f5c0d2062
-    SemanticVersion: 1.0.8
+    LicenseUrl: s3://appdevstore/guarddutybenchmark/v1.0.10/cac1a6df52c685e0f6ebe4ae72078c80
+    ReadmeUrl: s3://appdevstore/guarddutybenchmark/v1.0.10/cab012d7fb7887671b751e6f5c0d2062
+    SemanticVersion: 1.0.10
     SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/cloudwatchevents/guarddutybenchmark
 Parameters:
   CollectorName:
@@ -81,6 +81,8 @@ Parameters:
     - jp
     - us2
     - us1
+    - in
+    - fed
     Description: Enter au, ca, de, eu, jp, us2, or us1
   RemoveSumoResourcesOnDeleteStack:
     AllowedValues:
@@ -93,7 +95,7 @@ Parameters:
 Resources:
   CloudWatchEventFunction:
     Properties:
-      CodeUri: s3://appdevstore/guarddutybenchmark/v1.0.8/0d13904d1fbd69ecd879992058f1aa41
+      CodeUri: s3://appdevstore/guarddutybenchmark/v1.0.10/0d13904d1fbd69ecd879992058f1aa41
       Environment:
         Variables:
           SUMO_ENDPOINT:
@@ -115,7 +117,7 @@ Resources:
     Properties:
       Location:
         ApplicationId: arn:aws:serverlessrepo:us-east-1:956882708938:applications/sumologic-app-utils
-        SemanticVersion: 1.0.22
+        SemanticVersion: 1.0.24
   SumoHostedCollector:
     Type: Custom::Collector
     Properties:
@@ -173,6 +175,7 @@ Resources:
       Region:
         Ref: AWS::Region
       AppName: Amazon GuardDuty Benchmark
+      AppId: 8e7efcb3-040a-4a92-9f8d-922fafb24afb
       RemoveOnDeleteStack:
         Ref: RemoveSumoResourcesOnDeleteStack
       AppSources:
diff --git a/cloudwatchevents/guarddutybenchmark/template_v2.yaml b/cloudwatchevents/guarddutybenchmark/template_v2.yaml
@@ -56,7 +56,7 @@ Metadata:
     Name: sumologic-guardduty-benchmark
     LicenseUrl: ./LICENSE
     ReadmeUrl: ./README.md
-    SemanticVersion: 1.0.8
+    SemanticVersion: 1.0.10
     SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/cloudwatchevents/guarddutybenchmark
 
 Parameters:
@@ -83,6 +83,8 @@ Parameters:
       - jp
       - us2
       - us1
+      - in
+      - fed
     Description: "Enter au, ca, de, eu, jp, us2, or us1"
   RemoveSumoResourcesOnDeleteStack:
     AllowedValues:
@@ -115,7 +117,7 @@ Resources:
     Properties:
       Location:
         ApplicationId: arn:aws:serverlessrepo:us-east-1:956882708938:applications/sumologic-app-utils
-        SemanticVersion: 1.0.22
+        SemanticVersion: 1.0.24
 
   SumoHostedCollector:
     Type: Custom::Collector
@@ -150,6 +152,7 @@ Resources:
       ServiceToken: !GetAtt SumoAppUtils.Outputs.SumoAppUtilsFunction
       Region: !Ref "AWS::Region"
       AppName: "Amazon GuardDuty Benchmark"
+      AppId: "8e7efcb3-040a-4a92-9f8d-922fafb24afb"
       RemoveOnDeleteStack: !Ref RemoveSumoResourcesOnDeleteStack
       AppSources:
         gdbenchmark: !Sub "_sourceCategory=${SourceCategoryName}"
diff --git a/cloudwatchevents/guarddutybenchmark/testdeploy.sh b/cloudwatchevents/guarddutybenchmark/testdeploy.sh
@@ -9,7 +9,7 @@ else
     AWS_REGION="us-east-2"
 fi
 
-version="1.0.8"
+version="1.0.10"
 
 sam package --template-file template_v2.yaml --s3-bucket $SAM_S3_BUCKET  --output-template-file packaged_v2.yaml --s3-prefix "guarddutybenchmark/v$version"
 
@@ -18,7 +18,9 @@ sam publish --template packaged_v2.yaml --region $AWS_REGION --semantic-version
 
 ACCESS_ID=""
 ACCESS_KEY=""
-SUMO_DEPLOYMENT="us2"
+ORG_ID=""
+
+SUMO_DEPLOYMENT="us1"
 
 # sam deploy --template-file packaged_v2.yaml --stack-name testinggdbenchmarknew --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND --region $AWS_REGION --parameter-overrides SumoDeployment="$SUMO_DEPLOYMENT" SumoAccessID="$ACCESS_ID" SumoAccessKey="$ACCESS_KEY" RemoveSumoResourcesOnDeleteStack="true"
 
diff --git a/loggroup-lambda-connector/test/loggroup-lambda-cft.json b/loggroup-lambda-connector/test/loggroup-lambda-cft.json
@@ -69,7 +69,7 @@
                     ]
                 },
                 "Timeout": 300,
-                "Runtime": "nodejs8.10"
+                "Runtime": "nodejs10.x"
             }
         },
         "SumoLogGroupLambdaConnectorRole": {
diff --git a/loggroup-lambda-connector/test/test_loggroup_lambda_connector.py b/loggroup-lambda-connector/test/test_loggroup_lambda_connector.py
@@ -149,7 +149,7 @@ def add_dummy_lambda(self, template_data):
                 "FunctionName": test_lambda_name,
                 "Timeout": 300,
                 "Handler": "index.handler",
-                "Runtime": "nodejs8.10",
+                "Runtime": "nodejs10.x",
                 "MemorySize": 128
             }
         }
diff --git a/sumologic-app-utils/deploy.sh b/sumologic-app-utils/deploy.sh
@@ -24,7 +24,7 @@ if [ ! -f sumo_app_utils.zip ]; then
     cd ..
     rm -r python
 fi
-version="1.0.22"
+version="1.0.24"
 aws s3 cp sumo_app_utils.zip s3://$SAM_S3_BUCKET/sumo_app_utils/v"$version"/sumo_app_utils.zip --region $AWS_REGION --acl public-read
 
 sam package --template-file sumo_app_utils.yaml --s3-bucket $SAM_S3_BUCKET  --output-template-file packaged_sumo_app_utils.yaml --s3-prefix "sumo_app_utils/v"$version
diff --git a/sumologic-app-utils/packaged_sumo_app_utils.yaml b/sumologic-app-utils/packaged_sumo_app_utils.yaml
@@ -20,7 +20,7 @@ Metadata:
     - sumologic
     - serverless
     Name: sumologic-app-utils
-    SemanticVersion: 1.0.22
+    SemanticVersion: 1.0.24
     SourceCodeUrl: https://github.com/SumoLogic/sumologic-aws-lambda/tree/master/sumologic-app-utils
     SpdxLicenseId: Apache-2.0
 Resources:
@@ -29,7 +29,7 @@ Resources:
     Properties:
       Handler: main.handler
       Runtime: python3.7
-      CodeUri: s3://appdevstore/sumo_app_utils/v1.0.22/sumo_app_utils.zip
+      CodeUri: s3://appdevstore/sumo_app_utils/v1.0.24/sumo_app_utils.zip
       MemorySize: 128
       Timeout: 300
       Policies:
diff --git a/sumologic-app-utils/src/api.py b/sumologic-app-utils/src/api.py
@@ -146,7 +146,7 @@ def extract_params(self, event):
     def api_endpoint(self):
         if self.deployment == "us1":
             return "https://api.sumologic.com/api"
-        elif self.deployment in ["ca", "au", "de", "eu", "jp", "us2"]:
+        elif self.deployment in ["ca", "au", "de", "eu", "jp", "us2", "fed", "in"]:
             return "https://api.%s.sumologic.com/api" % self.deployment
         else:
             return 'https://%s-api.sumologic.net/api' % self.deployment
@@ -590,11 +590,11 @@ def _wait_for_folder_creation(self, folder_id, job_id):
 
         print("job status: %s" % response.text)
 
-    def _wait_for_app_install(self, app_id, job_id):
-        print("waiting for app installation app_id %s job_id %s" % (app_id, job_id))
+    def _wait_for_app_install(self, job_id):
+        print("waiting for app installation job_id %s" % job_id)
         waiting = True
         while waiting:
-            response = self.sumologic_cli.check_app_install_status(app_id, job_id)
+            response = self.sumologic_cli.check_app_install_status(job_id)
             waiting = response.json()['status'] == "InProgress"
             time.sleep(5)
         print("job status: %s" % response.text)
@@ -649,7 +649,7 @@ def create_by_install_api(self, appid, appname, source_params, *args, **kwargs):
 
         response = self.sumologic_cli.install_app(appid, content)
         job_id = response.json()["id"]
-        response = self._wait_for_app_install(appid, job_id)
+        response = self._wait_for_app_install(job_id)
 
         json_resp = json.loads(response.content)
         if (json_resp['status'] == 'Success'):
@@ -696,52 +696,56 @@ def extract_params(self, event):
 
 if __name__ == '__main__':
 
-    params = {
-
-        "access_id": "",
-        "access_key": "",
-        "deployment": "us1"
-
+    props = {
+        "SumoAccessID": "",
+        "SumoAccessKey": "",
+        "SumoDeployment": "us1",
     }
-    # app_prefix = "CloudTrail"
-    app_prefix = "GuardDuty"
+    app_prefix = "CloudTrail"
+    # app_prefix = "GuardDuty"
     collector_id = None
     collector_type = "Hosted"
     collector_name = "%sCollector" % app_prefix
     source_name = "%sEvents" % app_prefix
     source_category = "Labs/AWS/%s" % app_prefix
-    appname = "Amazon GuardDuty Benchmark"
-    # appname = "AWS CloudTrail"
+    # appname = "Global Intelligence for Amazon GuardDuty"
+    appname = "Global Intelligence for AWS CloudTrail"
+    appid = "570bdc0d-f824-4fcb-96b2-3230d4497180"
     # appid = "ceb7fac5-1137-4a04-a5b8-2e49190be3d4"
-    appid = None
+    # appid = None
+    # source_params = {
+    #     "logsrc": "_sourceCategory=%s" % source_category
+    # }
     source_params = {
-        "logsrc": "_sourceCategory=%s" % source_category
+        "cloudtraillogsource": "_sourceCategory=%s" % source_category,
+        "indexname": '%rnd%',
+        "incrementalindex": "%rnd%"
     }
-    col = Collector(**params)
-    src = HTTPSource(**params)
-    app = App(**params)
+    # col = Collector(**params)
+    # src = HTTPSource(**params)
+    app = App(props)
 
     # create
     # _, collector_id = col.create(collector_type, collector_name, source_category)
     # _, source_id = src.create(collector_id, source_name, source_category)
 
     _, app_folder_id = app.create(appname, source_params, appid)
-
+    app.delete(app_folder_id, True)
 
     # update
     # _, new_collector_id = col.update(collector_id, collector_type, "%sCollectorNew" % app_prefix, "Labs/AWS/%sNew" % app_prefix, description="%s Collector" % app_prefix)
     # assert(collector_id == new_collector_id)
     # _, new_source_id = src.update(collector_id, source_id, "%sEventsNew" % app_prefix, "Labs/AWS/%sNew" % app_prefix, date_format="yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", date_locator='\"createTime\":(.*),')
     # assert(source_id == new_source_id)
-    new_source_params = {
-        "logsrc": "_sourceCategory=%s" % ("Labs/AWS/%sNew" % app_prefix)
-    }
+    # new_source_params = {
+    #     "logsrc": "_sourceCategory=%s" % ("Labs/AWS/%sNew" % app_prefix)
+    # }
 
-    _, new_app_folder_id = app.update(app_folder_id, appname, new_source_params, appid)
-    assert(app_folder_id != new_app_folder_id)
+    # _, new_app_folder_id = app.update(app_folder_id, appname, new_source_params, appid)
+    # assert(app_folder_id != new_app_folder_id)
 
     # delete
     # src.delete(collector_id, source_id, True)
     # col.delete(collector_id, True)
-    app.delete(new_app_folder_id, True)
+    # app.delete(new_app_folder_id, True)
 
diff --git a/sumologic-app-utils/src/sumologic.py b/sumologic-app-utils/src/sumologic.py
@@ -216,5 +216,5 @@ def install_app(self, app_id, content):
         time.sleep(random.randint(1, 10))
         return self.post('/apps/%s/install' % (app_id), params=content)
 
-    def check_app_install_status(self, app_id, job_id):
-        return self.get('/apps/%s/install/%s/status' % (app_id, job_id))
+    def check_app_install_status(self, job_id):
+        return self.get('/apps/install/%s/status' % job_id)
diff --git a/sumologic-app-utils/sumo_app_utils.yaml b/sumologic-app-utils/sumo_app_utils.yaml

Original file line number	Diff line number	Diff line change
`@@ -85,7 +85,7 @@`
`85`	`85`	`]`
`86`	`86`	`},`
`87`	`87`	`"Timeout": 300,`
`88`		`- "Runtime": "nodejs8.10"`
	`88`	`+ "Runtime": "nodejs10.x"`
`89`	`89`	`}`
`90`	`90`	`},`
`91`	`91`	`"CloudWatchEventFunctionCloudWatchEventTriggerPermission": {`
Original file line number	Diff line number	Diff line change
`@@ -69,7 +69,7 @@`
`69`	`69`	`]`
`70`	`70`	`},`
`71`	`71`	`"Timeout": 300,`
`72`		`- "Runtime": "nodejs8.10"`
	`72`	`+ "Runtime": "nodejs10.x"`
`73`	`73`	`}`
`74`	`74`	`},`
`75`	`75`	`"SumoLogGroupLambdaConnectorRole": {`
Original file line number	Diff line number	Diff line change
`@@ -149,7 +149,7 @@ def add_dummy_lambda(self, template_data):`
`149`	`149`	`"FunctionName": test_lambda_name,`
`150`	`150`	`"Timeout": 300,`
`151`	`151`	`"Handler": "index.handler",`
`152`		`- "Runtime": "nodejs8.10",`
	`152`	`+ "Runtime": "nodejs10.x",`
`153`	`153`	`"MemorySize": 128`
`154`	`154`	`}`
`155`	`155`	`}`