feat(aws): AWSX-1591 Move appsync, apigateway, bedrock, batch and codebuild to backend

aws/logs_monitoring/steps/enums.py

@@ -2,16 +2,11 @@
 
 
 class AwsEventSource(Enum):
-    APIGATEWAY = "apigateway"
-    APPSYNC = "appsync"
     AWS = "aws"
-    BATCH = "batch"
-    BEDROCK = "bedrock"
     CARBONBLACK = "carbonblack"
     CLOUDFRONT = "cloudfront"
     CLOUDTRAIL = "cloudtrail"
     CLOUDWATCH = "cloudwatch"
-    CODEBUILD = "codebuild"
     DMS = "dms"
     DOCDB = "docdb"
     EKS = "eks"
@@ -49,7 +44,6 @@ def __str__(self):
     @staticmethod
     def cloudwatch_sources():
         return [
-            AwsEventSource.BEDROCK,
             AwsEventSource.CLOUDFRONT,
             AwsEventSource.CLOUDTRAIL,
             AwsEventSource.ELASTICSEARCH,
@@ -72,12 +66,8 @@ def __init__(self, string, event_source):
     WAF_0 = ("aws-waf-logs", AwsEventSource.WAF)
     WAF_1 = ("waflogs", AwsEventSource.WAF)
 
-    # e.g. 2024/06/12/08/amazon-apigateway-<firehose-ds-name>-2-2024-06-12-08-45-12-796e56c0-7fdf-47b7-9268-38b875bb62d2
-    APIGATEWAY = ("amazon-apigateway", AwsEventSource.APIGATEWAY)
-    BEDROCK = ("bedrock", AwsEventSource.BEDROCK)
     # e.g. carbon-black-cloud-forwarder/alerts/org_key=*****/year=2021/month=7/day=19/hour=18/minute=15/second=41/8436e850-7e78-40e4-b3cd-6ebbc854d0a2.jsonl.gz
     CARBONBLACK = ("carbon-black", AwsEventSource.CARBONBLACK)
-    CODEBUILD = ("amazon_codebuild", AwsEventSource.CODEBUILD)
     DMS = ("amazon_dms", AwsEventSource.DMS)
     DOCDB = ("amazon_documentdb", AwsEventSource.DOCDB)
     # e.g. AWSLogs/123456779121/elasticloadbalancing/us-east-1/2020/10/02/123456779121_elasticloadbalancing_us-east-1_app.alb.xxxxx.xx.xxx.xxx_x.log.gz
@@ -102,20 +92,7 @@ def __init__(self, string, event_source):
         self.string = string
         self.event_source = event_source
 
-    # default location for rest api execution logs
-    APIGATEWAY_0 = ("api-gateway", AwsEventSource.APIGATEWAY)
-    # default location set by serverless framework for rest api access logs
-    APIGATEWAY_1 = ("/aws/api-gateway", AwsEventSource.APIGATEWAY)
-    # default location set by serverless framework for http api logs
-    APIGATEWAY_2 = ("/aws/http-api", AwsEventSource.APIGATEWAY)
-    # WebSocket API Execution Logs, e.g. /aws/apigateway/api-id/stage-name
-    APIGATEWAY_3 = ("/aws/apigateway", AwsEventSource.APIGATEWAY)
-    # e.g. /aws/appsync/yourApiId
-    APPSYNC = ("/aws/appsync", AwsEventSource.APPSYNC)
-    BATCH = ("/aws/batch/job", AwsEventSource.BATCH)
-    BEDROCK = ("aws/bedrock/modelinvocations", AwsEventSource.BEDROCK)
     # e.g. /aws/codebuild/my-project
-    CODEBUILD = ("/aws/codebuild", AwsEventSource.CODEBUILD)
     CLOUDTRAIL = ("_CloudTrail_", AwsEventSource.CLOUDTRAIL)
     # e.g. dms-tasks-test-instance
     DMS = ("dms-tasks", AwsEventSource.DMS)

aws/logs_monitoring/steps/handlers/awslogs_handler.py

@@ -97,8 +97,6 @@ def set_source(self, event, metadata, aws_attributes):
             source = str(AwsEventSource.CLOUDTRAIL)
         if str(AwsCwEventSourcePrefix.TRANSITGATEWAY) in log_stream:
             source = str(AwsEventSource.TRANSITGATEWAY)
-        if str(AwsCwEventSourcePrefix.BEDROCK) in log_stream:
-            source = str(AwsEventSource.BEDROCK)
         metadata[DD_SOURCE] = parse_event_source(event, source)
 
         # Special handling for customized log group of Lambda Functions and Step Functions
@@ -136,8 +134,6 @@ def set_host(self, metadata, aws_attributes):
         match metadata_source:
             case AwsEventSource.CLOUDWATCH:
                 metadata[DD_HOST] = log_group
-            case AwsEventSource.APPSYNC:
-                metadata[DD_HOST] = log_group.split("/")[-1]
             case AwsEventSource.VERIFIED_ACCESS:
                 self.handle_verified_access_source(metadata, aws_attributes)
             case AwsEventSource.STEPFUNCTION:

aws/logs_monitoring/tests/test_parsing.py

@@ -78,22 +78,6 @@ def test_lambda_event(self):
             str(AwsEventSource.LAMBDA),
         )
 
-    def test_apigateway_event(self):
-        self.assertEqual(
-            parse_event_source(
-                {"awslogs": "logs"}, "Api-Gateway-Execution-Logs_a1b23c/test"
-            ),
-            str(AwsEventSource.APIGATEWAY),
-        )
-        self.assertEqual(
-            parse_event_source({"awslogs": "logs"}, "/aws/api-gateway/my-project"),
-            str(AwsEventSource.APIGATEWAY),
-        )
-        self.assertEqual(
-            parse_event_source({"awslogs": "logs"}, "/aws/http-api/my-project"),
-            str(AwsEventSource.APIGATEWAY),
-        )
-
     def test_dms_event(self):
         self.assertEqual(
             parse_event_source({"awslogs": "logs"}, "dms-tasks-test-instance"),
@@ -114,14 +98,6 @@ def test_sns_event(self):
             str(AwsEventSource.SNS),
         )
 
-    def test_codebuild_event(self):
-        self.assertEqual(
-            parse_event_source(
-                {"awslogs": "logs"}, "/aws/codebuild/new-project-sample"
-            ),
-            str(AwsEventSource.CODEBUILD),
-        )
-
     def test_kinesis_event(self):
         self.assertEqual(
             parse_event_source({"awslogs": "logs"}, "/aws/kinesisfirehose/test"),
@@ -231,15 +207,6 @@ def test_fargate_event(self):
             str(AwsEventSource.FARGATE),
         )
 
-    def test_appsync_event(self):
-        self.assertEqual(
-            parse_event_source(
-                {"awslogs": "logs"},
-                "/aws/appsync/apis/",
-            ),
-            str(AwsEventSource.APPSYNC),
-        )
-
     def test_cloudfront_event(self):
         self.assertEqual(
             parse_event_source(

aws/logs_monitoring/tools/integration_tests/tester/test_snapshots.py

@@ -1,10 +1,11 @@
-import unittest
 import base64
-import os
-import urllib.request
+import gzip
 import json
+import os
 import re
-import gzip
+import unittest
+import urllib.request
+
 from deepdiff import DeepDiff
 
 recorder_url = os.environ.get("RECORDER_URL", default="")
@@ -138,11 +139,6 @@ def test_cloudwatch_log(self):
         snapshot_filename = f"{input_filename}~snapshot"
         self.compare_snapshot(input_filename, snapshot_filename)
 
-    def test_cloudwatch_apigateway_log(self):
-        input_filename = f"{snapshot_dir}/cloudwatch_log_apigateway.json"
-        snapshot_filename = f"{input_filename}~snapshot"
-        self.compare_snapshot(input_filename, snapshot_filename)
-
     def test_cloudwatch_cloudtrail_log(self):
         input_filename = f"{snapshot_dir}/cloudwatch_log_cloudtrail.json"
         snapshot_filename = f"{input_filename}~snapshot"