From 2e2a108a67e213aa13c587a22be2eb42a9caf0ed Mon Sep 17 00:00:00 2001 From: NogaNHS Date: Thu, 25 Jul 2024 12:05:19 +0100 Subject: [PATCH] [PRMP-639] - new resources --- .../terraform/bulk-ods-update.tf | 33 ------------- .../{ods-dynamodb.tf => dynamodb-gp-ods.tf} | 46 ++++++++----------- ...{icb-ods-dynamo.tf => dynamodb-icb-ods.tf} | 31 +++++++++---- .../terraform/event-enrichment-lambda.tf | 24 +++++----- .../terraform/iam-event-enrichment.tf | 4 +- .../terraform/lambda-bulk-ods-update.tf | 43 +++++++++++++++++ .../gp-registrations-mi/terraform/provider.tf | 2 +- .../terraform/s3-ods-csv-updates.tf | 46 ++++++++++++------- stacks/gp-registrations-mi/terraform/s3.tf | 6 +-- .../terraform/variables.tf | 4 ++ 10 files changed, 135 insertions(+), 104 deletions(-) delete mode 100644 stacks/gp-registrations-mi/terraform/bulk-ods-update.tf rename stacks/gp-registrations-mi/terraform/{ods-dynamodb.tf => dynamodb-gp-ods.tf} (62%) rename stacks/gp-registrations-mi/terraform/{icb-ods-dynamo.tf => dynamodb-icb-ods.tf} (58%) create mode 100644 stacks/gp-registrations-mi/terraform/lambda-bulk-ods-update.tf diff --git a/stacks/gp-registrations-mi/terraform/bulk-ods-update.tf b/stacks/gp-registrations-mi/terraform/bulk-ods-update.tf deleted file mode 100644 index 3d88cbe..0000000 --- a/stacks/gp-registrations-mi/terraform/bulk-ods-update.tf +++ /dev/null @@ -1,33 +0,0 @@ -variable "ods_bulk_update_lambda_name" { - default = "ods_bulk_update_lambda" -} - -resource "aws_lambda_function" "event_enrichment_lambda" { - filename = "${path.cwd}/${var.ods_bulk_update_lambda_name}" - function_name = "${var.environment}-${var.ods_bulk_update_lambda_name}" - role = aws_iam_role.bulk_ods_lambda_role.arn - handler = "ods_bulk_update.lambda_handler" - source_code_hash = filebase64sha256("${path.cwd}/${var.bulk_ods_update_lambda_zip}") - runtime = "python3.12" - timeout = 300 - tags = merge( - local.common_tags, - { - Name = "${var.environment}-gp-mi-ods_bulk" - ApplicationRole = "AwsLambdaFunction" - } - ) -} - - -resource "aws_cloudwatch_log_group" "bulk_ods_update_lambda" { - name = "/aws/lambda/${var.environment}-${var.ods_bulk_update_lambda_name}" - tags = merge( - local.common_tags, - { - Name = "${var.environment}-${var.ods_bulk_update_lambda_name}" - ApplicationRole = "AwsCloudwatchLogGroup" - } - ) - retention_in_days = 60 -} diff --git a/stacks/gp-registrations-mi/terraform/ods-dynamodb.tf b/stacks/gp-registrations-mi/terraform/dynamodb-gp-ods.tf similarity index 62% rename from stacks/gp-registrations-mi/terraform/ods-dynamodb.tf rename to stacks/gp-registrations-mi/terraform/dynamodb-gp-ods.tf index 16735ca..3805e40 100644 --- a/stacks/gp-registrations-mi/terraform/ods-dynamodb.tf +++ b/stacks/gp-registrations-mi/terraform/dynamodb-gp-ods.tf @@ -1,44 +1,38 @@ -resource "aws_dynamodb_table" "mi-api-ods_dynamodb_table" { +resource "aws_dynamodb_table" "mi_api_gp_ods" { name = "${var.environment}_mi_enrichment_practice_ods" billing_mode = "PAY_PER_REQUEST" deletion_protection_enabled = false - hash_key = "PracticeOdsCode" + hash_key = "PracticeOdsCode" attribute { name = "PracticeOdsCode" type = "S" } - attribute { - name = "PracticeName" - type = "S" - } - - attribute { - name = "IcbOdsCode" - type = "S" - } - - attribute { - name = "SupplierName" - type = "S" - } - - attribute { - name = "SDSLastUpdated" - type = "N" - } - tags = { Name = "mi_enrichment_practice_ods" Environment = var.environment } + import_table { + input_format = "CSV" + input_compression_type = "NONE" + s3_bucket_source { + bucket = aws_s3_bucket.ods_csv_files.id + key_prefix = aws_s3_object.initial_gp_ods.key + } + input_format_options { + csv { + delimiter = "," + header_list = ["PracticeOdsCode", "PracticeName", "IcbOdsCode"] + } + } + } } resource "aws_iam_policy" "dynamodb_policy_ods_enrichment_lambda" { - name = "${var.environment}_${aws_dynamodb_table.mi-api-ods_dynamodb_table.name}_policy" + name = "${var.environment}_${aws_dynamodb_table.mi_api_gp_ods.name}_policy" path = "/" policy = jsonencode({ @@ -52,10 +46,9 @@ resource "aws_iam_policy" "dynamodb_policy_ods_enrichment_lambda" { "dynamodb:PutItem", ], "Resource" : [ - aws_dynamodb_table.mi-api-ods_dynamodb_table.arn + aws_dynamodb_table.mi_api_gp_ods.arn ] } - ] }) } @@ -73,10 +66,9 @@ resource "aws_iam_policy" "dynamodb_policy_bulk_ods_data_lambda" { "dynamodb:UpdateItem", ], "Resource" : [ - aws_dynamodb_table.mi-api-ods_dynamodb_table.arn + aws_dynamodb_table.mi_api_gp_ods.arn ] } - ] }) } \ No newline at end of file diff --git a/stacks/gp-registrations-mi/terraform/icb-ods-dynamo.tf b/stacks/gp-registrations-mi/terraform/dynamodb-icb-ods.tf similarity index 58% rename from stacks/gp-registrations-mi/terraform/icb-ods-dynamo.tf rename to stacks/gp-registrations-mi/terraform/dynamodb-icb-ods.tf index 45f33a8..4873f09 100644 --- a/stacks/gp-registrations-mi/terraform/icb-ods-dynamo.tf +++ b/stacks/gp-registrations-mi/terraform/dynamodb-icb-ods.tf @@ -1,17 +1,29 @@ -resource "aws_dynamodb_table" "mi-api-icb-ods_dynamodb_table" { +resource "aws_dynamodb_table" "mi_api_icb_ods" { name = "${var.environment}_mi_enrichment_icb_ods" billing_mode = "PAY_PER_REQUEST" deletion_protection_enabled = false - hash_key = "IcbOdsCode" + hash_key = "IcbOdsCode" attribute { name = "IcbOdsCode" type = "S" } - attribute { - name = "IcbName" - type = "S" + + import_table { + input_format = "CSV" + input_compression_type = "NONE" + s3_bucket_source { + bucket = aws_s3_bucket.ods_csv_files.id + key_prefix = aws_s3_object.initial_icb_ods.key + } + + input_format_options { + csv { + delimiter = "," + header_list = ["IcbOdsCode", "IcbName"] + } + } } tags = { @@ -21,7 +33,7 @@ resource "aws_dynamodb_table" "mi-api-icb-ods_dynamodb_table" { } resource "aws_iam_policy" "dynamodb_policy_icb_ods_enrichment_lambda" { - name = "${var.environment}_${aws_dynamodb_table.mi-api-icb-ods_dynamodb_table.name}_policy" + name = "${var.environment}_${aws_dynamodb_table.mi_api_icb_ods.name}_policy" path = "/" policy = jsonencode({ @@ -33,10 +45,9 @@ resource "aws_iam_policy" "dynamodb_policy_icb_ods_enrichment_lambda" { "dynamodb:GetItem", "dynamodb:UpdateItem", "dynamodb:PutItem", - ], "Resource" : [ - aws_dynamodb_table.mi-api-ods_dynamodb_table.arn + aws_dynamodb_table.mi_api_icb_ods.arn ] } ] @@ -44,7 +55,7 @@ resource "aws_iam_policy" "dynamodb_policy_icb_ods_enrichment_lambda" { } resource "aws_iam_policy" "dynamodb_policy_bulk_icb_ods_data_lambda" { - name = "${var.environment}_mi_bulk_${aws_dynamodb_table.mi-api-icb-ods_dynamodb_table.name}_policy" + name = "${var.environment}_mi_bulk_${aws_dynamodb_table.mi_api_icb_ods.name}_policy" path = "/" policy = jsonencode({ @@ -56,7 +67,7 @@ resource "aws_iam_policy" "dynamodb_policy_bulk_icb_ods_data_lambda" { "dynamodb:UpdateItem", ], "Resource" : [ - aws_dynamodb_table.mi-api-icb-ods_dynamodb_table.arn + aws_dynamodb_table.mi_api_icb_ods.arn ] } ] diff --git a/stacks/gp-registrations-mi/terraform/event-enrichment-lambda.tf b/stacks/gp-registrations-mi/terraform/event-enrichment-lambda.tf index 5b8c853..2adf17c 100644 --- a/stacks/gp-registrations-mi/terraform/event-enrichment-lambda.tf +++ b/stacks/gp-registrations-mi/terraform/event-enrichment-lambda.tf @@ -3,17 +3,17 @@ variable "event_enrichment_lambda_name" { } resource "aws_lambda_function" "event_enrichment_lambda" { - filename = "${path.cwd}/${var.event_enrichment_lambda_zip}" - function_name = "${var.environment}-${var.event_enrichment_lambda_name}" - role = aws_iam_role.event_enrichment_lambda_role.arn - handler = "event_enrichment_main.lambda_handler" + filename = "${path.cwd}/${var.event_enrichment_lambda_zip}" + function_name = "${var.environment}-${var.event_enrichment_lambda_name}" + role = aws_iam_role.event_enrichment_lambda_role.arn + handler = "event_enrichment_main.lambda_handler" source_code_hash = filebase64sha256("${path.cwd}/${var.event_enrichment_lambda_zip}") - runtime = "python3.12" - timeout = 300 + runtime = "python3.12" + timeout = 300 tags = merge( local.common_tags, { - Name = "${var.environment}-gp-registrations-mi" + Name = "${var.environment}-gp-registrations-mi" ApplicationRole = "AwsLambdaFunction" } ) @@ -21,9 +21,9 @@ resource "aws_lambda_function" "event_enrichment_lambda" { environment { variables = { SPLUNK_CLOUD_EVENT_UPLOADER_SQS_QUEUE_URL = aws_sqs_queue.incoming_mi_events_for_splunk_cloud_event_uploader.url, - ENRICHED_EVENTS_SNS_TOPIC_ARN = aws_sns_topic.enriched_events_topic.arn, - SDS_FHIR_API_KEY_PARAM_NAME = var.sds_fhir_api_key_param_name, - SDS_FHIR_API_URL_PARAM_NAME = var.sds_fhir_api_url_param_name, + ENRICHED_EVENTS_SNS_TOPIC_ARN = aws_sns_topic.enriched_events_topic.arn, + SDS_FHIR_API_KEY_PARAM_NAME = var.sds_fhir_api_key_param_name, + SDS_FHIR_API_URL_PARAM_NAME = var.sds_fhir_api_url_param_name, } } } @@ -35,7 +35,7 @@ resource "aws_lambda_event_source_mapping" "sqs_queue_event_enrichment_lambda_tr filter { pattern = jsonencode({ body = { - eventType : [ { "anything-but": [ "DEGRADES" ] } ] + eventType : [{ "anything-but" : ["DEGRADES"] }] } }) } @@ -47,7 +47,7 @@ resource "aws_cloudwatch_log_group" "event_enrichment_lambda" { tags = merge( local.common_tags, { - Name = "${var.environment}-${var.event_enrichment_lambda_name}" + Name = "${var.environment}-${var.event_enrichment_lambda_name}" ApplicationRole = "AwsCloudwatchLogGroup" } ) diff --git a/stacks/gp-registrations-mi/terraform/iam-event-enrichment.tf b/stacks/gp-registrations-mi/terraform/iam-event-enrichment.tf index 4a98dad..361d85c 100644 --- a/stacks/gp-registrations-mi/terraform/iam-event-enrichment.tf +++ b/stacks/gp-registrations-mi/terraform/iam-event-enrichment.tf @@ -1,5 +1,5 @@ #Lambda -resource "aws_iam_role" "event_enrichment_lambda_role" { +resource "aws_iam_role" "event_enrichment_lambda" { name = "${var.environment}-event-enrichment-lambda-role" assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json managed_policy_arns = [ @@ -13,7 +13,7 @@ resource "aws_iam_role" "event_enrichment_lambda_role" { ] } -resource "aws_iam_role" "bulk_ods_lambda_role" { +resource "aws_iam_role" "bulk_ods_lambda" { name = "${var.environment}-bulk-ods-lambda-role" assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json managed_policy_arns = [ diff --git a/stacks/gp-registrations-mi/terraform/lambda-bulk-ods-update.tf b/stacks/gp-registrations-mi/terraform/lambda-bulk-ods-update.tf new file mode 100644 index 0000000..c551bac --- /dev/null +++ b/stacks/gp-registrations-mi/terraform/lambda-bulk-ods-update.tf @@ -0,0 +1,43 @@ +resource "aws_lambda_function" "ods_bulk_update" { + filename = "${path.cwd}/${var.ods_bulk_update_lambda_name}" + function_name = "${var.environment}-${var.ods_bulk_update_lambda_name}" + role = aws_iam_role.bulk_ods_lambda_role.arn + handler = "ods_bulk_update.lambda_handler" + source_code_hash = filebase64sha256("${path.cwd}/${var.bulk_ods_update_lambda_zip}") + runtime = "python3.12" + timeout = 300 + environment { + variables = { + TRUD_API_KEY_PARAM_NAME = data.aws_ssm_parameter.trud_api_key, + TRUD_FHIR_API_URL_PARAM_NAME = data.aws_ssm_parameter.trud_api_endpoint, + } + } + tags = merge( + local.common_tags, + { + Name = "${var.environment}-gp-mi-ods_bulk" + ApplicationRole = "AwsLambdaFunction" + } + ) +} + + +resource "aws_cloudwatch_log_group" "bulk_ods_update_lambda" { + name = "/aws/lambda/${var.environment}-${var.ods_bulk_update_lambda_name}" + tags = merge( + local.common_tags, + { + Name = "${var.environment}-${var.ods_bulk_update_lambda_name}" + ApplicationRole = "AwsCloudwatchLogGroup" + } + ) + retention_in_days = 60 +} + +data "aws_ssm_parameter" "trud_api_key" { + name = "TRUD_api_secret_key" +} + +data "aws_ssm_parameter" "trud_api_endpoint" { + name = "TRUD_api_download_endoint" +} \ No newline at end of file diff --git a/stacks/gp-registrations-mi/terraform/provider.tf b/stacks/gp-registrations-mi/terraform/provider.tf index d481c01..5ea34bd 100644 --- a/stacks/gp-registrations-mi/terraform/provider.tf +++ b/stacks/gp-registrations-mi/terraform/provider.tf @@ -6,7 +6,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 5.59.0" + version = ">= 5.0" } } } \ No newline at end of file diff --git a/stacks/gp-registrations-mi/terraform/s3-ods-csv-updates.tf b/stacks/gp-registrations-mi/terraform/s3-ods-csv-updates.tf index 257fbaa..dbb6b05 100644 --- a/stacks/gp-registrations-mi/terraform/s3-ods-csv-updates.tf +++ b/stacks/gp-registrations-mi/terraform/s3-ods-csv-updates.tf @@ -1,4 +1,4 @@ -resource "aws_s3_bucket" "ods-csv-files-bucket" { +resource "aws_s3_bucket" "ods_csv_files" { bucket = "${terraform.workspace}-ods-csv-files" force_destroy = true @@ -8,11 +8,11 @@ resource "aws_s3_bucket" "ods-csv-files-bucket" { } } -resource "aws_s3_bucket_lifecycle_configuration" "mi_events_lifecycle" { - bucket = aws_s3_bucket.mi_events_output.id +resource "aws_s3_bucket_lifecycle_configuration" "csv_ods" { + bucket = aws_s3_bucket.ods_csv_files.id rule { - id = "expire-ods-csv-after-3-months" + id = "expire-ods-csv-after-3-months" status = "Enabled" expiration { @@ -21,8 +21,8 @@ resource "aws_s3_bucket_lifecycle_configuration" "mi_events_lifecycle" { } } -resource "aws_s3_bucket_public_access_block" "ods-csv-files-bucket_output" { - bucket = aws_s3_bucket.ods-csv-files-bucket.id +resource "aws_s3_bucket_public_access_block" "ods_csv_files" { + bucket = aws_s3_bucket.ods_csv_files.id block_public_acls = true block_public_policy = true @@ -30,28 +30,28 @@ resource "aws_s3_bucket_public_access_block" "ods-csv-files-bucket_output" { restrict_public_buckets = true } -resource "aws_s3_bucket_versioning" "mi_events_output" { - bucket = aws_s3_bucket.ods-csv-files-bucket.id +resource "aws_s3_bucket_versioning" "ods_csv_files" { + bucket = aws_s3_bucket.ods_csv_files.id versioning_configuration { status = "Enabled" } } -resource "aws_s3_bucket_ownership_controls" "s3_bucket_acl_ownership" { - bucket = aws_s3_bucket.ods-csv-files-bucket.id +resource "aws_s3_bucket_ownership_controls" "ods_csv_files" { + bucket = aws_s3_bucket.ods_csv_files.id rule { object_ownership = "ObjectWriter" } } -resource "aws_s3_bucket_acl" "bucket_acl" { - bucket = aws_s3_bucket.ods-csv-files-bucket.id - acl = "private" +resource "aws_s3_bucket_acl" "ods_csv_files" { + bucket = aws_s3_bucket.ods_csv_files.id + acl = "private" } -resource "aws_iam_policy" "s3_ods_csv_document_data_policy" { - name = "${terraform.workspace}_${aws_s3_bucket.ods-csv-files-bucket.bucket}_get_document_data_policy" +resource "aws_iam_policy" "ods_csv_files_data_policy" { + name = "${terraform.workspace}_${aws_s3_bucket.ods_csv_files.bucket}_get_document_data_policy" policy = jsonencode({ "Version" : "2012-10-17", @@ -62,8 +62,22 @@ resource "aws_iam_policy" "s3_ods_csv_document_data_policy" { "s3:GetObject", "s3:PutObject", ], - "Resource" : ["${aws_s3_bucket.ods-csv-files-bucket}/*"] + "Resource" : ["${aws_s3_bucket.ods_csv_files.arn}/*"] } ] }) } + +resource "aws_s3_object" "initial_gp_ods_csv" { + bucket = aws_s3_bucket.ods_csv_files.id + key = "init/initial-gps-ods-csv" + source = "../ods-csv/initial-gps-ods-csv.csv" + etag = filemd5("") +} + +resource "aws_s3_object" "initial_icb_ods_csv" { + bucket = aws_s3_bucket.ods_csv_files.id + key = "init/initial-icb-ods-csv" + source = "../ods-csv/initial-gps-ods-csv.csv" + etag = filemd5("") +} \ No newline at end of file diff --git a/stacks/gp-registrations-mi/terraform/s3.tf b/stacks/gp-registrations-mi/terraform/s3.tf index a4a4aae..f2d2395 100644 --- a/stacks/gp-registrations-mi/terraform/s3.tf +++ b/stacks/gp-registrations-mi/terraform/s3.tf @@ -8,17 +8,17 @@ resource "aws_s3_bucket" "mi_events_output" { tags = merge( local.common_tags, { - Name = "${var.environment}-prm-gp-registrations-mi-s3-mi-events" + Name = "${var.environment}-prm-gp-registrations-mi-s3-mi-events" ApplicationRole = "AwsS3Bucket" } ) } -resource "aws_s3_bucket_lifecycle_configuration" "mi_events_lifecycle" { +resource "aws_s3_bucket_lifecycle_configuration" "mi_events_output" { bucket = aws_s3_bucket.mi_events_output.id rule { - id = "expire-mi-objects-after-2-years" + id = "expire-mi-objects-after-2-years" status = "Enabled" expiration { diff --git a/stacks/gp-registrations-mi/terraform/variables.tf b/stacks/gp-registrations-mi/terraform/variables.tf index 28f4efc..89cca54 100644 --- a/stacks/gp-registrations-mi/terraform/variables.tf +++ b/stacks/gp-registrations-mi/terraform/variables.tf @@ -133,3 +133,7 @@ variable "sds_fhir_api_url_param_name" { type = string description = "SSM param containing SDS FHIR API url to retrieve supplier details" } + +variable "ods_bulk_update_lambda_name" { + default = "ods_bulk_update_lambda" +} \ No newline at end of file