fix(DMVP-3029): Cloudwatch and s3 to trigger lambda

modules/cloudwatch-to-lambda/README.md

@@ -0,0 +1,43 @@
+# cloudfront-to-s3-to-cloudwatch
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_lambda"></a> [lambda](#module\_lambda) | terraform-aws-modules/lambda/aws | 4.7.1 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_cloudwatch_log_subscription_filter.subscription_filter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_subscription_filter) | resource |
+| [aws_lambda_permission.cloudwatch_permission_to_trigger_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_cloudwatch_log_group_names"></a> [cloudwatch\_log\_group\_names](#input\_cloudwatch\_log\_group\_names) | Cloudwatch Log group name | `list(string)` | n/a | yes |
+| <a name="input_lambda_configs"></a> [lambda\_configs](#input\_lambda\_configs) | Lambda Configs | <pre>object({<br> environment_variables = optional(object({<br> url = optional(string, "")<br> }), {})<br> })</pre> | `{}` | no |
+| <a name="input_name"></a> [name](#input\_name) | Lambda name | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_name"></a> [name](#output\_name) | n/a |
+| <a name="output_role_arn"></a> [role\_arn](#output\_role\_arn) | n/a |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/cloudwatch-to-lambda/data.tf

@@ -0,0 +1,3 @@
+data "aws_region" "current" {}
+
+data "aws_caller_identity" "current" {}

modules/cloudwatch-to-lambda/main.tf

@@ -0,0 +1,14 @@
+module "lambda" {
+ source = "terraform-aws-modules/lambda/aws"
+ version = "4.7.1"
+
+ function_name = var.name
+ handler = "lambda.handler"
+ runtime = "python3.7"
+ publish = true
+ source_path = "${path.module}/src/"
+
+ role_name = var.name
+
+ environment_variables = try(var.lambda_configs.environment_variables, {})
+}

modules/cloudwatch-to-lambda/outputs.tf

@@ -0,0 +1,7 @@
+output "name" {
+ value = module.lambda.lambda_function_name
+}
+
+output "role_arn" {
+ value = module.lambda.lambda_role_arn
+}

modules/cloudwatch-to-lambda/src/lambda.py

@@ -0,0 +1,60 @@
+import logging
+import json
+import os
+import base64
+import gzip
+from urllib.request import Request, urlopen
+from urllib.error import URLError, HTTPError
+
+LOGLEVEL = os.environ.get('LOG_LEVEL', 'INFO').upper()
+FALLBACK_SUBJECT = 'AWS Alerts'
+
+logger = logging.getLogger()
+logger.setLevel(getattr(logging, LOGLEVEL))
+
+logger.info("log level: {}".format(LOGLEVEL))
+
+def request_sender(uncompressed_data):
+ url = os.environ['url']
+ headers = {'Content-Type': 'application/json'}
+ method = 'POST'
+
+ req = Request(
+ url,
+ uncompressed_data,
+ headers, method=method
+ )
+
+ try:
+ response = urlopen(req)
+ response.read()
+ logger.info("Message posted")
+ return {"status": "200 OK"}
+ except HTTPError as e:
+ logger.debug(e)
+ logger.error("Request failed: %d %s", e.code, e.reason)
+ return e
+ except URLError as e:
+ logger.error("Server connection failed: %s", e.reason)
+ return e
+
+def handler(event, context):
+ data = event["awslogs"]["data"]
+ print("event")
+ print(event)
+ print("data")
+ print(data)
+ try:
+ # Decode the base64 data
+ decoded_data = base64.b64decode(data)
+ print("decoded_data")
+ print(decoded_data)
+ uncompressed_data = gzip.decompress(decoded_data)
+ print("Uncompressed data: ")
+ print(uncompressed_data)
+ result = json.loads(uncompressed_data)
+ print("Json Loads: ")
+ print(result)
+ request_sender(result)
+ except base64.binascii.Error as e:
+ print("Error decoding base64 data:", e)

modules/cloudwatch-to-lambda/subscription.tf

@@ -0,0 +1,22 @@
+resource "aws_cloudwatch_log_subscription_filter" "subscription_filter" {
+ for_each = { for log_name in var.cloudwatch_log_group_names : log_name => log_name }
+
+ name = "${var.name}_${each.value}"
+ log_group_name = each.value
+ filter_pattern = ""
+ destination_arn = module.lambda.lambda_function_arn
+
+ depends_on = [
+ aws_lambda_permission.cloudwatch_permission_to_trigger_lambda
+ ]
+}
+
+resource "aws_lambda_permission" "cloudwatch_permission_to_trigger_lambda" {
+ for_each = { for log_name in var.cloudwatch_log_group_names : log_name => log_name }
+
+ statement_id = "AllowExecutionFromCloudWatch"
+ action = "lambda:InvokeFunction"
+ function_name = module.lambda.lambda_function_arn
+ principal = "logs.${data.aws_region.current.name}.amazonaws.com"
+ source_arn = "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${each.value}:*"
+}

modules/cloudwatch-to-lambda/tests/basic/README.md

@@ -0,0 +1,29 @@
+# basic
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+No providers.
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_cloudwatch-to-lambda"></a> [cloudwatch-to-lambda](#module\_cloudwatch-to-lambda) | ../../ | n/a |
+
+## Resources
+
+No resources.
+
+## Inputs
+
+No inputs.
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/cloudwatch-to-lambda/tests/basic/customised_provider.tftest.hcl

@@ -0,0 +1,15 @@
+run "valid_string_concat" {
+ module {
+ source = "../../"
+ }
+ variables {
+ name = "CloudwatchtoLambda"
+ cloudwatch_log_group_names = ["aws-cloudtrail-logs-753273213401-85b50618"]
+
+ lambda_configs = {
+ environment_variables = {
+ url = "https://4eb8-195-250-6etwyeywtey-4w32.ngrok-free.app/aws"
+ }
+ }
+ }
+}

modules/cloudwatch-to-lambda/tests/basic/main.tf

@@ -0,0 +1,12 @@
+module "cloudwatch-to-lambda" {
+ source = "../../"
+
+ name = "CloudwatchtoLambda"
+ cloudwatch_log_group_names = ["aws-cloudtrail-logs-753273213401-85b50618"]
+
+ lambda_configs = {
+ environment_variables = {
+ url = "https://4eb8-195-250-69-234.ngrok-free.app/aws"
+ }
+ }
+}

modules/cloudwatch-to-lambda/variables.tf

@@ -0,0 +1,19 @@
+variable "name" {
+ type = string
+ description = "Lambda name"
+}
+
+variable "cloudwatch_log_group_names" {
+ type = list(string)
+ description = "Cloudwatch Log group name"
+}
+
+variable "lambda_configs" {
+ type = object({
+ environment_variables = optional(object({
+ url = optional(string, "")
+ }), {})
+ })
+ default = {}
+ description = "Lambda Configs"
+}

modules/s3-to-lambda/README.md

@@ -0,0 +1,41 @@
+# cloudfront-to-s3-to-cloudwatch
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_lambda"></a> [lambda](#module\_lambda) | terraform-aws-modules/lambda/aws | 4.7.1 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_lambda_permission.s3_permission_to_trigger_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
+| [aws_s3_bucket_notification.incoming](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_notification) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_bucket_name"></a> [bucket\_name](#input\_bucket\_name) | S3 bucket for s3 subscription | `list(string)` | n/a | yes |
+| <a name="input_lambda_configs"></a> [lambda\_configs](#input\_lambda\_configs) | Lambda Configs | <pre>object({<br> environment_variables = optional(object({<br> url = optional(string, "")<br> }), {})<br> })</pre> | `{}` | no |
+| <a name="input_name"></a> [name](#input\_name) | Lambda name | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_name"></a> [name](#output\_name) | n/a |
+| <a name="output_role_arn"></a> [role\_arn](#output\_role\_arn) | n/a |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/s3-to-lambda/main.tf

@@ -0,0 +1,14 @@
+module "lambda" {
+ source = "terraform-aws-modules/lambda/aws"
+ version = "4.7.1"
+
+ function_name = var.name
+ handler = "lambda.handler"
+ runtime = "python3.7"
+ publish = true
+ source_path = "${path.module}/src/"
+
+ role_name = var.name
+
+ environment_variables = try(var.lambda_configs.environment_variables, {})
+}

modules/s3-to-lambda/outputs.tf

@@ -0,0 +1,7 @@
+output "name" {
+ value = module.lambda.lambda_function_name
+}
+
+output "role_arn" {
+ value = module.lambda.lambda_role_arn
+}

modules/s3-to-lambda/src/lambda.py

@@ -0,0 +1,43 @@
+import logging
+import json
+import os
+import requests
+import importlib
+import sys
+import types
+from urllib.request import Request, urlopen
+from urllib.error import URLError, HTTPError
+
+LOGLEVEL = os.environ.get('LOG_LEVEL', 'INFO').upper()
+FALLBACK_SUBJECT = 'AWS Alerts'
+
+logger = logging.getLogger()
+logger.setLevel(getattr(logging, LOGLEVEL))
+
+logger.info("log level: {}".format(LOGLEVEL))
+
+def handler(event, context):
+
+ url = "https://4eb8-195-250-69-234.ngrok-free.app/aws"
+ headers = {'Content-Type': 'application/json'}
+ method = 'POST'
+
+ print("lambda Request: ")
+ req = Request(
+ url,
+ json.dumps(event).encode('utf-8'),
+ headers, method=method
+ )
+
+ try:
+ response = urlopen(req)
+ response.read()
+ logger.info("Message posted")
+ return {"status": "200 OK"}
+ except HTTPError as e:
+ logger.debug(e)
+ logger.error("Request failed: %d %s", e.code, e.reason)
+ return e
+ except URLError as e:
+ logger.error("Server connection failed: %s", e.reason)
+ return e

modules/s3-to-lambda/subscription.tf

@@ -0,0 +1,21 @@
+resource "aws_s3_bucket_notification" "incoming" {
+ for_each = { for bucket in var.bucket_name : bucket => bucket }
+ bucket = each.value
+
+ lambda_function {
+ lambda_function_arn = module.lambda.lambda_function_arn
+ events = ["s3:ObjectCreated:Put"]
+ }
+
+ depends_on = [aws_lambda_permission.s3_permission_to_trigger_lambda]
+}
+
+resource "aws_lambda_permission" "s3_permission_to_trigger_lambda" {
+ for_each = { for bucket in var.bucket_name : bucket => bucket }
+
+ statement_id = "AllowExecutionFromS3Bucket"
+ action = "lambda:InvokeFunction"
+ function_name = module.lambda.lambda_function_arn
+ principal = "s3.amazonaws.com"
+ source_arn = "arn:aws:s3:::${each.value}"
+}

modules/s3-to-lambda/tests/basic/README.md

@@ -0,0 +1,29 @@
+# basic
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+No providers.
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_s3-to-lambda"></a> [s3-to-lambda](#module\_s3-to-lambda) | ../../ | n/a |
+
+## Resources
+
+No resources.
+
+## Inputs
+
+No inputs.
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->