feat(aws): AWSX-1987 Migrate remaining soure identification to logs-backend (#998)

Signed-off-by: Vincent Boutour <vincent.boutour@datadoghq.com>

Author: ViBiOh

aws/logs_monitoring/steps/enrichment.py

@@ -25,7 +25,6 @@ def enrich(events, cache_layer):
         add_metadata_to_lambda_log(event, cache_layer)
         extract_ddtags_from_message(event)
         extract_host_from_cloudtrails(event)
-        extract_host_from_guardduty(event)
 
     return events
 
@@ -206,12 +205,3 @@ def extract_host_from_cloudtrails(event):
                 match = HOST_IDENTITY_REGEXP.match(arn)
                 if match is not None:
                     event[DD_HOST] = match.group("host")
-
-
-def extract_host_from_guardduty(event):
-    if event is not None and event.get(DD_SOURCE) == str(AwsEventSource.GUARDDUTY):
-        host = event.get("detail", {}).get("resource")
-        if isinstance(host, dict):
-            host = host.get("instanceDetails", {}).get("instanceId")
-            if host is not None:
-                event[DD_HOST] = host

aws/logs_monitoring/steps/enums.py

@@ -5,15 +5,8 @@ class AwsEventSource(Enum):
     AWS = "aws"
     CLOUDTRAIL = "cloudtrail"
     CLOUDWATCH = "cloudwatch"
-    ELASTICSEARCH = "elasticsearch"
-    FARGATE = "fargate"
-    GUARDDUTY = "guardduty"
     KINESIS = "kinesis"
     LAMBDA = "lambda"
-    MARIADB = "mariadb"
-    MSK = "msk"
-    MYSQL = "mysql"
-    POSTGRESQL = "postgresql"
     S3 = "s3"
     SNS = "sns"
     STEPFUNCTION = "stepfunction"
@@ -26,9 +19,6 @@ def __str__(self):
     def cloudwatch_sources():
         return [
             AwsEventSource.CLOUDTRAIL,
-            AwsEventSource.ELASTICSEARCH,
-            AwsEventSource.FARGATE,
-            AwsEventSource.MSK,
         ]
 
 
@@ -41,9 +31,7 @@ def __init__(self, string, event_source):
     WAF_0 = ("aws-waf-logs", AwsEventSource.WAF)
     WAF_1 = ("waflogs", AwsEventSource.WAF)
 
-    GUARDDUTY = ("guardduty", AwsEventSource.GUARDDUTY)
     KINESIS = ("amazon_kinesis", AwsEventSource.KINESIS)
-    MSK = ("amazon_msk", AwsEventSource.MSK)
 
     def __str__(self):
         return f"{self.string}"

aws/logs_monitoring/tests/test_enrichment.py

@@ -9,7 +9,6 @@
     add_metadata_to_lambda_log,
     extract_ddtags_from_message,
     extract_host_from_cloudtrails,
-    extract_host_from_guardduty,
 )
 
 
@@ -160,14 +159,6 @@ def test_parse_source_cloudtrail(self):
         extract_host_from_cloudtrails(event)
         self.assertEqual(event["host"], "i-99999999")
 
-    def test_parse_source_guardduty(self):
-        event = {
-            "ddsource": "guardduty",
-            "detail": {"resource": {"instanceDetails": {"instanceId": "i-99999999"}}},
-        }
-        extract_host_from_guardduty(event)
-        self.assertEqual(event["host"], "i-99999999")
-
 
 class TestLambdaMetadataEnrichment(unittest.TestCase):
     def test_empty_event(self):

aws/logs_monitoring/tests/test_parsing.py

@@ -134,15 +134,6 @@ def test_redshift_gov_event(self):
             str(AwsEventSource.S3),
         )
 
-    def test_fargate_event(self):
-        self.assertEqual(
-            parse_event_source(
-                {"awslogs": "logs"},
-                "/ecs/fargate-logs",
-            ),
-            str(AwsEventSource.FARGATE),
-        )
-
     def test_cloudfront_event(self):
         self.assertEqual(
             parse_event_source(
@@ -152,28 +143,6 @@ def test_cloudfront_event(self):
             str(AwsEventSource.S3),
         )
 
-    def test_elasticsearch_event(self):
-        self.assertEqual(
-            parse_event_source({"awslogs": "logs"}, "/elasticsearch/domain"),
-            str(AwsEventSource.ELASTICSEARCH),
-        )
-
-    def test_msk_event(self):
-        self.assertEqual(
-            parse_event_source(
-                {"awslogs": "logs"},
-                "/myMSKLogGroup",
-            ),
-            str(AwsEventSource.MSK),
-        )
-        self.assertEqual(
-            parse_event_source(
-                {"Records": ["logs-from-s3"]},
-                "AWSLogs/amazon_msk/us-east-1/xxxxx.log.gz",
-            ),
-            str(AwsEventSource.MSK),
-        )
-
     def test_cloudwatch_source_if_none_found(self):
         self.assertEqual(
             parse_event_source({"awslogs": "logs"}, ""), str(AwsEventSource.CLOUDWATCH)