diff --git a/aws/logs_monitoring/steps/enums.py b/aws/logs_monitoring/steps/enums.py index 3c3cf0a0f..a48df641e 100644 --- a/aws/logs_monitoring/steps/enums.py +++ b/aws/logs_monitoring/steps/enums.py @@ -2,16 +2,11 @@ class AwsEventSource(Enum): - APIGATEWAY = "apigateway" - APPSYNC = "appsync" AWS = "aws" - BATCH = "batch" - BEDROCK = "bedrock" CARBONBLACK = "carbonblack" CLOUDFRONT = "cloudfront" CLOUDTRAIL = "cloudtrail" CLOUDWATCH = "cloudwatch" - CODEBUILD = "codebuild" DMS = "dms" DOCDB = "docdb" EKS = "eks" @@ -49,7 +44,6 @@ def __str__(self): @staticmethod def cloudwatch_sources(): return [ - AwsEventSource.BEDROCK, AwsEventSource.CLOUDFRONT, AwsEventSource.CLOUDTRAIL, AwsEventSource.ELASTICSEARCH, @@ -72,12 +66,8 @@ def __init__(self, string, event_source): WAF_0 = ("aws-waf-logs", AwsEventSource.WAF) WAF_1 = ("waflogs", AwsEventSource.WAF) - # e.g. 2024/06/12/08/amazon-apigateway--2-2024-06-12-08-45-12-796e56c0-7fdf-47b7-9268-38b875bb62d2 - APIGATEWAY = ("amazon-apigateway", AwsEventSource.APIGATEWAY) - BEDROCK = ("bedrock", AwsEventSource.BEDROCK) # e.g. carbon-black-cloud-forwarder/alerts/org_key=*****/year=2021/month=7/day=19/hour=18/minute=15/second=41/8436e850-7e78-40e4-b3cd-6ebbc854d0a2.jsonl.gz CARBONBLACK = ("carbon-black", AwsEventSource.CARBONBLACK) - CODEBUILD = ("amazon_codebuild", AwsEventSource.CODEBUILD) DMS = ("amazon_dms", AwsEventSource.DMS) DOCDB = ("amazon_documentdb", AwsEventSource.DOCDB) # e.g. AWSLogs/123456779121/elasticloadbalancing/us-east-1/2020/10/02/123456779121_elasticloadbalancing_us-east-1_app.alb.xxxxx.xx.xxx.xxx_x.log.gz @@ -102,20 +92,7 @@ def __init__(self, string, event_source): self.string = string self.event_source = event_source - # default location for rest api execution logs - APIGATEWAY_0 = ("api-gateway", AwsEventSource.APIGATEWAY) - # default location set by serverless framework for rest api access logs - APIGATEWAY_1 = ("/aws/api-gateway", AwsEventSource.APIGATEWAY) - # default location set by serverless framework for http api logs - APIGATEWAY_2 = ("/aws/http-api", AwsEventSource.APIGATEWAY) - # WebSocket API Execution Logs, e.g. /aws/apigateway/api-id/stage-name - APIGATEWAY_3 = ("/aws/apigateway", AwsEventSource.APIGATEWAY) - # e.g. /aws/appsync/yourApiId - APPSYNC = ("/aws/appsync", AwsEventSource.APPSYNC) - BATCH = ("/aws/batch/job", AwsEventSource.BATCH) - BEDROCK = ("aws/bedrock/modelinvocations", AwsEventSource.BEDROCK) # e.g. /aws/codebuild/my-project - CODEBUILD = ("/aws/codebuild", AwsEventSource.CODEBUILD) CLOUDTRAIL = ("_CloudTrail_", AwsEventSource.CLOUDTRAIL) # e.g. dms-tasks-test-instance DMS = ("dms-tasks", AwsEventSource.DMS) diff --git a/aws/logs_monitoring/steps/handlers/awslogs_handler.py b/aws/logs_monitoring/steps/handlers/awslogs_handler.py index 6790ded15..10935550a 100644 --- a/aws/logs_monitoring/steps/handlers/awslogs_handler.py +++ b/aws/logs_monitoring/steps/handlers/awslogs_handler.py @@ -97,8 +97,6 @@ def set_source(self, event, metadata, aws_attributes): source = str(AwsEventSource.CLOUDTRAIL) if str(AwsCwEventSourcePrefix.TRANSITGATEWAY) in log_stream: source = str(AwsEventSource.TRANSITGATEWAY) - if str(AwsCwEventSourcePrefix.BEDROCK) in log_stream: - source = str(AwsEventSource.BEDROCK) metadata[DD_SOURCE] = parse_event_source(event, source) # Special handling for customized log group of Lambda Functions and Step Functions @@ -136,8 +134,6 @@ def set_host(self, metadata, aws_attributes): match metadata_source: case AwsEventSource.CLOUDWATCH: metadata[DD_HOST] = log_group - case AwsEventSource.APPSYNC: - metadata[DD_HOST] = log_group.split("/")[-1] case AwsEventSource.VERIFIED_ACCESS: self.handle_verified_access_source(metadata, aws_attributes) case AwsEventSource.STEPFUNCTION: diff --git a/aws/logs_monitoring/tests/test_parsing.py b/aws/logs_monitoring/tests/test_parsing.py index 131cad371..20dca207b 100644 --- a/aws/logs_monitoring/tests/test_parsing.py +++ b/aws/logs_monitoring/tests/test_parsing.py @@ -78,22 +78,6 @@ def test_lambda_event(self): str(AwsEventSource.LAMBDA), ) - def test_apigateway_event(self): - self.assertEqual( - parse_event_source( - {"awslogs": "logs"}, "Api-Gateway-Execution-Logs_a1b23c/test" - ), - str(AwsEventSource.APIGATEWAY), - ) - self.assertEqual( - parse_event_source({"awslogs": "logs"}, "/aws/api-gateway/my-project"), - str(AwsEventSource.APIGATEWAY), - ) - self.assertEqual( - parse_event_source({"awslogs": "logs"}, "/aws/http-api/my-project"), - str(AwsEventSource.APIGATEWAY), - ) - def test_dms_event(self): self.assertEqual( parse_event_source({"awslogs": "logs"}, "dms-tasks-test-instance"), @@ -114,14 +98,6 @@ def test_sns_event(self): str(AwsEventSource.SNS), ) - def test_codebuild_event(self): - self.assertEqual( - parse_event_source( - {"awslogs": "logs"}, "/aws/codebuild/new-project-sample" - ), - str(AwsEventSource.CODEBUILD), - ) - def test_kinesis_event(self): self.assertEqual( parse_event_source({"awslogs": "logs"}, "/aws/kinesisfirehose/test"), @@ -231,15 +207,6 @@ def test_fargate_event(self): str(AwsEventSource.FARGATE), ) - def test_appsync_event(self): - self.assertEqual( - parse_event_source( - {"awslogs": "logs"}, - "/aws/appsync/apis/", - ), - str(AwsEventSource.APPSYNC), - ) - def test_cloudfront_event(self): self.assertEqual( parse_event_source( diff --git a/aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_apigateway.json b/aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_apigateway.json deleted file mode 100644 index 0ce921492..000000000 --- a/aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_apigateway.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "messageType": "DATA_MESSAGE", - "owner": "123456789123", - "logGroup": "API-Gateway-Execution-Logs_xxxxxxxxxx/production", - "logStream": "123456789123_API-Gateway_us-east-1", - "subscriptionFilters": [ - "testFilter" - ], - "logEvents": [ - { - "ingestionTime": 1607444452966, - "timestamp": 1607444432899, - "message": "(aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa) Extended Request Id: XXXXXX-XXXXXXXX=", - "eventId": "11111111111111111111111111111111", - "logStreamName": "11111111111111111111111111111111" - }, - { - "ingestionTime": 1607444452966, - "timestamp": 1607444432899, - "message": "(aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa) Verifying Usage Plan for request: a5217acf-1172-4860-b374-8116dbd3e9de. API Key: API Stage: xxxxxxxxxx/production", - "eventId": "11111111111111111111111111111111", - "logStreamName": "11111111111111111111111111111111" - }, - { - "ingestionTime": 1607444452966, - "timestamp": 1607444432900, - "message": "(aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa) API Key authorized because method 'POST /event' does not require API Key. Request will not contribute to throttle or quota limits", - "eventId": "11111111111111111111111111111111", - "logStreamName": "11111111111111111111111111111111" - } - ] -} \ No newline at end of file diff --git a/aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_apigateway.json~snapshot b/aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_apigateway.json~snapshot deleted file mode 100644 index cd5c7f11f..000000000 --- a/aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_apigateway.json~snapshot +++ /dev/null @@ -1,197 +0,0 @@ -{ - "events": [ - { - "data": [ - { - "aws": { - "awslogs": { - "logGroup": "API-Gateway-Execution-Logs_xxxxxxxxxx/production", - "logStream": "123456789123_API-Gateway_us-east-1", - "owner": "123456789123" - }, - "invoked_function_arn": "arn:aws:lambda:us-east-1:012345678912:function:test_function" - }, - "ddsource": "apigateway", - "ddsourcecategory": "aws", - "ddtags": "forwardername:test_function,forwarder_version:", - "eventId": "11111111111111111111111111111111", - "host": "API-Gateway-Execution-Logs_xxxxxxxxxx/production", - "ingestionTime": 1607444452966, - "logStreamName": "11111111111111111111111111111111", - "message": "(aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa) Extended Request Id: XXXXXX-XXXXXXXX=", - "service": "apigateway", - "timestamp": 1607444432899 - }, - { - "aws": { - "awslogs": { - "logGroup": "API-Gateway-Execution-Logs_xxxxxxxxxx/production", - "logStream": "123456789123_API-Gateway_us-east-1", - "owner": "123456789123" - }, - "invoked_function_arn": "arn:aws:lambda:us-east-1:012345678912:function:test_function" - }, - "ddsource": "apigateway", - "ddsourcecategory": "aws", - "ddtags": "forwardername:test_function,forwarder_version:", - "eventId": "11111111111111111111111111111111", - "host": "API-Gateway-Execution-Logs_xxxxxxxxxx/production", - "ingestionTime": 1607444452966, - "logStreamName": "11111111111111111111111111111111", - "message": "(aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa) Verifying Usage Plan for request: . API Key: API Stage: xxxxxxxxxx/production", - "service": "apigateway", - "timestamp": 1607444432899 - }, - { - "aws": { - "awslogs": { - "logGroup": "API-Gateway-Execution-Logs_xxxxxxxxxx/production", - "logStream": "123456789123_API-Gateway_us-east-1", - "owner": "123456789123" - }, - "invoked_function_arn": "arn:aws:lambda:us-east-1:012345678912:function:test_function" - }, - "ddsource": "apigateway", - "ddsourcecategory": "aws", - "ddtags": "forwardername:test_function,forwarder_version:", - "eventId": "11111111111111111111111111111111", - "host": "API-Gateway-Execution-Logs_xxxxxxxxxx/production", - "ingestionTime": 1607444452966, - "logStreamName": "11111111111111111111111111111111", - "message": "(aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa) API Key authorized because method 'POST /event' does not require API Key. Request will not contribute to throttle or quota limits", - "service": "apigateway", - "timestamp": 1607444432900 - } - ], - "headers": { - "Accept": "*/*", - "Accept-Encoding": "gzip, deflate", - "Connection": "keep-alive", - "Content-Length": "", - "Content-type": "application/json", - "DD-API-KEY": "abcdefghijklmnopqrstuvwxyz012345", - "DD-EVP-ORIGIN": "aws_forwarder", - "DD-EVP-ORIGIN-VERSION": "", - "Host": "recorder:8080", - "User-Agent": "", - "traceparent": "", - "tracestate": "", - "x-datadog-parent-id": "", - "x-datadog-sampling-priority": "1", - "x-datadog-tags": "", - "x-datadog-trace-id": "" - }, - "path": "/api/v2/logs", - "verb": "POST" - }, - { - "data": { - "series": [ - { - "device": null, - "host": null, - "interval": 10, - "metric": "aws.dd_forwarder.loggroup_cache_fetch_failure", - "points": "", - "tags": [ - "forwardername:test_function", - "forwarder_memorysize:3008", - "forwarder_version:", - "event_type:awslogs" - ], - "type": "distribution" - }, - { - "device": null, - "host": null, - "interval": 10, - "metric": "aws.dd_forwarder.list_tags_log_group_api_call", - "points": "", - "tags": [ - "forwardername:test_function", - "forwarder_memorysize:3008", - "forwarder_version:", - "event_type:awslogs" - ], - "type": "distribution" - }, - { - "device": null, - "host": null, - "interval": 10, - "metric": "aws.dd_forwarder.loggroup_cache_write_failure", - "points": "", - "tags": [ - "forwardername:test_function", - "forwarder_memorysize:3008", - "forwarder_version:", - "event_type:awslogs" - ], - "type": "distribution" - }, - { - "device": null, - "host": null, - "interval": 10, - "metric": "aws.dd_forwarder.incoming_events", - "points": "", - "tags": [ - "forwardername:test_function", - "forwarder_memorysize:3008", - "forwarder_version:", - "event_type:awslogs" - ], - "type": "distribution" - }, - { - "device": null, - "host": null, - "interval": 10, - "metric": "aws.dd_forwarder.logs_forwarded", - "points": "", - "tags": [ - "forwardername:test_function", - "forwarder_memorysize:3008", - "forwarder_version:", - "event_type:awslogs" - ], - "type": "distribution" - }, - { - "device": null, - "host": null, - "interval": 10, - "metric": "aws.dd_forwarder.metrics_forwarded", - "points": "", - "tags": [ - "forwardername:test_function", - "forwarder_memorysize:3008", - "forwarder_version:", - "event_type:awslogs" - ], - "type": "distribution" - } - ] - }, - "headers": { - "Accept": "*/*", - "Accept-Encoding": "gzip, deflate", - "Connection": "keep-alive", - "Content-Encoding": "deflate", - "Content-Length": "", - "Content-Type": "application/json", - "DD-API-KEY": "abcdefghijklmnopqrstuvwxyz012345", - "Host": "recorder:8080", - "User-Agent": "", - "traceparent": "", - "tracestate": "", - "x-datadog-parent-id": "", - "x-datadog-sampling-priority": "1", - "x-datadog-tags": "", - "x-datadog-trace-id": "" - }, - "path": "/api/v1/distribution_points", - "verb": "POST" - } - ] -} \ No newline at end of file diff --git a/aws/logs_monitoring/tools/integration_tests/tester/test_snapshots.py b/aws/logs_monitoring/tools/integration_tests/tester/test_snapshots.py index a0da73440..87117c49f 100644 --- a/aws/logs_monitoring/tools/integration_tests/tester/test_snapshots.py +++ b/aws/logs_monitoring/tools/integration_tests/tester/test_snapshots.py @@ -1,10 +1,11 @@ -import unittest import base64 -import os -import urllib.request +import gzip import json +import os import re -import gzip +import unittest +import urllib.request + from deepdiff import DeepDiff recorder_url = os.environ.get("RECORDER_URL", default="") @@ -138,11 +139,6 @@ def test_cloudwatch_log(self): snapshot_filename = f"{input_filename}~snapshot" self.compare_snapshot(input_filename, snapshot_filename) - def test_cloudwatch_apigateway_log(self): - input_filename = f"{snapshot_dir}/cloudwatch_log_apigateway.json" - snapshot_filename = f"{input_filename}~snapshot" - self.compare_snapshot(input_filename, snapshot_filename) - def test_cloudwatch_cloudtrail_log(self): input_filename = f"{snapshot_dir}/cloudwatch_log_cloudtrail.json" snapshot_filename = f"{input_filename}~snapshot"