Cleanup branch

.ci/jobs/elastic-serverless-agent-mbp.yml

@@ -2,7 +2,7 @@
 - job:
     name: Library/elastic-serverless-forwarder-mbp
     display-name: Elastic Serverless Forwarder
-    description: Elastic Forwarder for Serverless
+    description: Elastic Serverless Forwarder
     view: BEATS-CI
     project-type: multibranch
     concurrent: true

.internal/aws/cloudformation/application.yaml

@@ -0,0 +1,85 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: >
+  Elastic Serverless Forwarder
+
+  SAM Template for the application, not intended to be deployed on its own
+
+Parameters:
+  ElasticServerlessForwarderS3ConfigFile:
+    Type: String
+  ElasticServerlessForwarderSSMSecrets:
+    Type: CommaDelimitedList
+  ElasticServerlessForwarderKMSKeys:
+    Type: CommaDelimitedList
+  ElasticServerlessForwarderSQSEvents:
+    Type: CommaDelimitedList
+  ElasticServerlessForwarderS3SQSEvents:
+    Type: CommaDelimitedList
+  ElasticServerlessForwarderKinesisEvents:
+    Type: CommaDelimitedList
+  ElasticServerlessForwarderCloudWatchLogsEvents:
+    Type: CommaDelimitedList
+  ElasticServerlessForwarderS3Buckets:
+    Type: CommaDelimitedList
+Resources:
+  ElasticServerlessForwarderContinuingDLQ:
+    Type: AWS::SQS::Queue
+    Properties:
+      DelaySeconds: 0
+      QueueName: !Join [ "-", ["elastic-serverless-forwarder-continuing-dlq", !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref AWS::StackId]]]]]]
+      VisibilityTimeout: 910
+  ElasticServerlessForwarderContinuingQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      DelaySeconds: 0
+      QueueName: !Join [ "-", ["elastic-serverless-forwarder-continuing-queue", !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref AWS::StackId]]]]]]
+      RedrivePolicy: { "deadLetterTargetArn" : !GetAtt ElasticServerlessForwarderContinuingDLQ.Arn, "maxReceiveCount" : 1 }
+      VisibilityTimeout: 910
+  ElasticServerlessForwarderReplayDLQ:
+    Type: AWS::SQS::Queue
+    Properties:
+      DelaySeconds: 0
+      QueueName: !Join [ "-", ["elastic-serverless-forwarder-replay-dlq", !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref AWS::StackId]]]]]]
+      VisibilityTimeout: 910
+  ElasticServerlessForwarderReplayQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      DelaySeconds: 0
+      QueueName: !Join [ "-", ["elastic-serverless-forwarder-replay-queue", !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref AWS::StackId]]]]]]
+      RedrivePolicy: { "deadLetterTargetArn" : !GetAtt ElasticServerlessForwarderReplayDLQ.Arn, "maxReceiveCount" : 3 }
+      VisibilityTimeout: 910
+  ApplicationElasticServerlessForwarder:
+    Type: AWS::Serverless::Function
+    Properties:
+      Timeout: 900
+      MemorySize: 512
+      CodeUri: %codeUri%
+      Runtime: python3.9
+      Architectures:
+        - x86_64
+      Handler: main_aws.handler
+      Environment:
+          Variables:
+              S3_CONFIG_FILE: !Ref ElasticServerlessForwarderS3ConfigFile
+              SQS_CONTINUE_URL: !Ref ElasticServerlessForwarderContinuingQueue
+              SQS_REPLAY_URL: !Ref ElasticServerlessForwarderReplayQueue
+      Events:
+        SQSContinuingEvent:
+          Type: SQS
+          Properties:
+            Queue: !GetAtt ElasticServerlessForwarderContinuingQueue.Arn
+            BatchSize: 10
+            Enabled: true
+  Fn::Transform:
+    Type: AWS::CloudFormation::Macro
+    Name: %sarAppName%-macro
+Metadata:
+  AWS::ServerlessRepo::Application:
+    Name: %sarAppName%-application
+    Description: SAM Template for the application, not intended to be deployed on its own
+    Author: %sarAuthorName%
+    SemanticVersion: %semanticVersion%
+    LicenseUrl: %codeUri%/LICENSE.txt
+    HomePageUrl: https://github.com/elastic/elastic-serverless-forwarder
+    SourceCodeUrl: https://github.com/elastic/elastic-serverless-forwarder

.internal/aws/cloudformation/macro.yaml

@@ -0,0 +1,231 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: >
+  Elastic Serverless Forwarder
+
+  SAM Template for the macro, not intended to be deployed on its own
+
+Resources:
+  MacroElasticServerlessForwarderFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      InlineCode: |
+        import random
+        import string
+
+
+        def random_suffix():
+            return "".join(random.choices(string.ascii_letters + string.digits, k=10))
+
+        def create_events(event):
+            events_fragment = {}
+            parameters = event["templateParameterValues"]
+            if "ElasticServerlessForwarderKinesisEvents" in parameters:
+                for kinesis_event in parameters["ElasticServerlessForwarderKinesisEvents"]:
+                    kinesis_event = kinesis_event.strip()
+                    if len(kinesis_event) == 0:
+                        continue
+
+                    kinesis_event_name = f"KinesisEvent{random_suffix()}"
+                    events_fragment[kinesis_event_name] = {
+                        "Type": "Kinesis",
+                        "Properties": {
+                            "Stream": kinesis_event,
+                            "StartingPosition": "TRIM_HORIZON",
+                            "BatchSize": 100,
+                            "FunctionResponseTypes": ["ReportBatchItemFailures"],
+                            "Enabled": True,
+                        }
+                    }
+
+            if "ElasticServerlessForwarderSQSEvents" in parameters:
+                for sqs_event in parameters["ElasticServerlessForwarderSQSEvents"]:
+                    sqs_event = sqs_event.strip()
+                    if len(sqs_event) == 0:
+                        continue
+
+                    sqs_event_name = f"SQSEvent{random_suffix()}"
+                    events_fragment[sqs_event_name] = {
+                        "Type": "SQS",
+                        "Properties": {
+                            "Queue": sqs_event,
+                            "BatchSize": 10,
+                            "Enabled": True,
+                        }
+                    }
+
+            if "ElasticServerlessForwarderS3SQSEvents" in parameters:
+                for s3_sqs_event in parameters["ElasticServerlessForwarderS3SQSEvents"]:
+                    s3_sqs_event = s3_sqs_event.strip()
+                    if len(s3_sqs_event) == 0:
+                        continue
+
+                    s3_sqs_event_name = f"S3SQSEvent{random_suffix()}"
+                    events_fragment[s3_sqs_event_name] = {
+                        "Type": "SQS",
+                        "Properties": {
+                            "Queue": s3_sqs_event,
+                            "BatchSize": 10,
+                            "Enabled": True,
+                        }
+                    }
+
+            if "ElasticServerlessForwarderCloudWatchLogsEvents" in parameters:
+                for cloudwatch_logs_event in parameters["ElasticServerlessForwarderCloudWatchLogsEvents"]:
+                    cloudwatch_logs_event = cloudwatch_logs_event.strip()
+                    if len(cloudwatch_logs_event) == 0:
+                        continue
+
+                    arn_components = cloudwatch_logs_event.split(":")
+                    cloudwatch_logs_group_name = arn_components[6]
+
+                    cloudwatch_logs_event_name = f"CloudWatchLogsEvent{random_suffix()}"
+                    events_fragment[cloudwatch_logs_event_name] = {
+                        "Type": "CloudWatchLogs",
+                        "Properties": {
+                            "FilterPattern": "",
+                            "LogGroupName": cloudwatch_logs_group_name,
+                        }
+                    }
+
+            return events_fragment
+
+
+        def create_policy(event):
+            policy_fragment = {
+                "Type": "AWS::IAM::Policy",
+                "Properties": {
+                    "PolicyName": {
+                        "Fn::Join": ["-", ["elastic-serverless-forwarder-policy", {
+                            "Fn::Select": [4, {
+                                "Fn::Split": ["-", {
+                                    "Fn::Select": [2, {
+                                        "Fn::Split": ["/", {
+                                            "Ref": "AWS::StackId"
+                                        }]
+                                    }]
+                                }]
+                            }]
+                        }]]
+                    },
+                    "PolicyDocument": {
+                        "Version": "2012-10-17",
+                        "Statement": []
+                    },
+                    "Roles": [{
+                        "Ref": "ApplicationElasticServerlessForwarderRole"
+                    }]
+                }
+            }
+
+            parameters = event["templateParameterValues"]
+            if "ElasticServerlessForwarderS3ConfigFile" in parameters:
+                bucket_name_and_object_key = parameters["ElasticServerlessForwarderS3ConfigFile"].replace("s3://", "")
+                resource = f"arn:aws:s3:::{bucket_name_and_object_key}"
+                if len(resource) > 0:
+                    policy_fragment["Properties"]["PolicyDocument"]["Statement"].append(
+                        {
+                            "Effect": "Allow",
+                            "Action": "s3:GetObject",
+                            "Resource": resource
+                        }
+                    )
+
+            if "ElasticServerlessForwarderSSMSecrets" in parameters:
+                ssm_secrets_arn = [x for x in parameters["ElasticServerlessForwarderSSMSecrets"] if len(x.strip()) > 0]
+
+                if len(ssm_secrets_arn) > 0:
+                    policy_fragment["Properties"]["PolicyDocument"]["Statement"].append(
+                        {
+                            "Effect": "Allow",
+                            "Action": "secretsmanager:GetSecretValue",
+                            "Resource": ssm_secrets_arn
+                        }
+                    )
+
+            if "ElasticServerlessForwarderKMSKeys" in parameters:
+                kms_keys_arn = [x for x in parameters["ElasticServerlessForwarderKMSKeys"] if len(x.strip()) > 0]
+                if len(kms_keys_arn) > 0:
+                    policy_fragment["Properties"]["PolicyDocument"]["Statement"].append(
+                        {
+                            "Effect": "Allow",
+                            "Action": "kms:Decrypt",
+                            "Resource": kms_keys_arn
+                        }
+                    )
+
+            if "ElasticServerlessForwarderCloudWatchLogsEvents" in parameters:
+                cloudwatch_logs_group_arn = [f"{':'.join(x.split(':')[0:-1])}:*:*" for x in parameters["ElasticServerlessForwarderCloudWatchLogsEvents"] if len(x.strip()) > 0]
+                if len(cloudwatch_logs_group_arn) > 0:
+                    policy_fragment["Properties"]["PolicyDocument"]["Statement"].append(
+                        {
+                            "Effect": "Allow",
+                            "Action": "logs:DescribeLogGroups",
+                            "Resource": cloudwatch_logs_group_arn[0]
+                        }
+                    )
+
+            if "ElasticServerlessForwarderS3Buckets" in parameters:
+                s3_buckets_arn = [x for x in parameters["ElasticServerlessForwarderS3Buckets"] if len(x.strip()) > 0]
+                if len(s3_buckets_arn) > 0:
+                    policy_fragment["Properties"]["PolicyDocument"]["Statement"].append(
+                        {
+                            "Effect": "Allow",
+                            "Action": "s3:ListBucket",
+                            "Resource": s3_buckets_arn
+                        }
+                    )
+
+                resources = []
+                for s3_bucket_with_notification in s3_buckets_arn:
+                    resources.append(f"{s3_bucket_with_notification}/*")
+
+                if len(resources) > 0:
+                    policy_fragment["Properties"]["PolicyDocument"]["Statement"].append(
+                        {
+                            "Effect": "Allow",
+                            "Action": "s3:GetObject",
+                            "Resource": resources
+                        }
+                    )
+
+            policy_fragment["Properties"]["PolicyDocument"]["Statement"].append(
+                {
+                    "Effect": "Allow",
+                    "Action": "sqs:SendMessage",
+                    "Resource": [
+                        {"Fn::GetAtt": ["ElasticServerlessForwarderReplayQueue", "Arn"]},
+                        {"Fn::GetAtt": ["ElasticServerlessForwarderContinuingQueue", "Arn"]},
+                    ]
+                }
+            )
+
+            return policy_fragment
+
+
+        def handler(event, context):
+            created_events = create_events(event)
+            for created_event in created_events:
+                event["fragment"]["ApplicationElasticServerlessForwarder"]["Properties"]["Events"][created_event] = created_events[created_event]
+
+            created_policy = create_policy(event)
+            event["fragment"]["ElasticServerlessForwarderPolicy"] = created_policy
+
+            return {"status": "SUCCESS", "requestId": event["requestId"], "fragment": event["fragment"]}
+      Handler: index.handler
+      Runtime: python3.9
+  MacroElasticServerlessForwarder:
+    Type: AWS::CloudFormation::Macro
+    Properties:
+      Description: Expand parameters to Events and Policy for %sarAppName%
+      FunctionName: !GetAtt MacroElasticServerlessForwarderFunction.Arn
+      Name: %sarAppName%-macro
+Metadata:
+  AWS::ServerlessRepo::Application:
+    Name: %sarAppName%-macro
+    Description: SAM Template for the macro, not intended to be deployed on its own
+    Author: %sarAuthorName%
+    SemanticVersion: %semanticVersion%
+    LicenseUrl: %codeUri%/LICENSE.txt
+    HomePageUrl: https://github.com/elastic/elastic-serverless-forwarder
+    SourceCodeUrl: https://github.com/elastic/elastic-serverless-forwarder