Update deployment templates for Aurora Serverless v2

CHANGELOG.md

@@ -82,6 +82,8 @@ the CloudWatch logs for your async operations (e.g. `PREFIX-AsyncOperationEcsLog
   - Added user guide on querying dead-letter-archive messages using AWS Athena.
 
 ### Changed
+- **CUMULUS-3669**
+  - Updates deployment template to create and utilize an Aurora Serverless V2 PostgreSQL cluster.
 - **CUMULUS-3570**
   - Updated Kinesis docs to support latest AWS UI and recommend server-side encryption.
 - **CUMULUS-3519**

example/rds-cluster-tf/main.tf

@@ -23,11 +23,10 @@ module "rds_cluster" {
   engine_version             = var.engine_version
   deletion_protection        = true
   cluster_identifier         = var.cluster_identifier
+  cluster_instance_count     = var.cluster_instance_count
   tags                       = var.tags
   snapshot_identifier        = var.snapshot_identifier
   lambda_timeouts            = var.lambda_timeouts
   lambda_memory_sizes        = var.lambda_memory_sizes
-  enable_upgrade             = var.enable_upgrade
-  parameter_group_family     = var.parameter_group_family
   parameter_group_family_v13 = var.parameter_group_family_v13
 }

example/rds-cluster-tf/outputs.tf

@@ -6,6 +6,10 @@ output "rds_endpoint" {
   value = module.rds_cluster.rds_endpoint
 }
 
+output "rds_reader_endpoint" {
+  value = module.rds_cluster.rds_reader_endpoint
+}
+
 output "admin_db_login_secret_arn" {
   value = module.rds_cluster.admin_db_login_secret_arn
 }

example/rds-cluster-tf/terraform.tfvars.example

@@ -1,9 +1,10 @@
-prefix              = "prefix"
-db_admin_username   = "changethisuser"
-db_admin_password   = "changethispassword"
-region              = "us-east-1"
-vpc_id              = "vpc_id"
-subnets             = ["subnet-some-subnet-1", "subnet-some-subnet-in-another-az-2"]
-deletion_protection = false
-cluster_identifier  = "some_cluster"
-tags                = { "Deployment" = "some_deployment_identifier" }
+prefix                 = "prefix"
+db_admin_username      = "changethisuser"
+db_admin_password      = "changethispassword"
+region                 = "us-east-1"
+vpc_id                 = "vpc_id"
+subnets                = ["subnet-some-subnet-1", "subnet-some-subnet-in-another-az-2"]
+deletion_protection    = false
+cluster_identifier     = "some_cluster"
+cluster_instance_count = 1
+tags                   = { "Deployment" = "some_deployment_identifier" }

example/rds-cluster-tf/variables.tf

@@ -48,6 +48,16 @@ variable "cluster_identifier" {
   default     = "cumulus-rds-serverless-default-cluster"
 }
 
+variable "cluster_instance_count" {
+  description = "Number of instances to create inside of the cluster"
+  type = number
+  default = 1
+  validation {
+    condition = var.cluster_instance_count >= 1 && var.cluster_instance_count <= 16
+    error_message = "Variable cluster_instance_count should be between 1 and 16."
+  }
+}
+
 variable "snapshot_identifier" {
   description = "Optional database snapshot for restoration"
   type = string
@@ -86,12 +96,6 @@ variable "lambda_memory_sizes" {
   }
 }
 
-variable "enable_upgrade" {
-  description = "Flag to enable use of updated parameter group"
-  type = bool
-  default = false
-}
-
 variable "lambda_timeouts" {
   description = "Configurable map of timeouts for lambdas"
   type = map(number)
@@ -100,12 +104,6 @@ variable "lambda_timeouts" {
   }
 }
 
-variable "parameter_group_family" {
-  description = "Database family to use for creating database parameter group"
-  type = string
-  default = "aurora-postgresql11"
-}
-
 variable "parameter_group_family_v13" {
   description = "Database family to use for creating database parameter group under postgres 13 upgrade conditions"
   type = string

tf-modules/cumulus-rds-tf/main.tf

@@ -38,6 +38,7 @@ resource "aws_secretsmanager_secret_version" "rds_login" {
     database            = "postgres"
     engine              = "postgres"
     host                = aws_rds_cluster.cumulus.endpoint
+    hostReader          = aws_rds_cluster.cumulus.reader_endpoint
     port                = 5432
     dbClusterIdentifier = aws_rds_cluster.cumulus.id
   })
@@ -52,21 +53,6 @@ resource "aws_security_group_rule" "rds_security_group_allow_postgres" {
   self              = true
 }
 
-resource "aws_rds_cluster_parameter_group" "rds_cluster_group" {
-  count = var.enable_upgrade ? 0 : 1
-  name   = "${var.prefix}-cluster-parameter-group"
-  family = var.parameter_group_family
-
-  dynamic "parameter" {
-    for_each = var.db_parameters
-    content {
-      apply_method = parameter.value["apply_method"]
-      name = parameter.value["name"]
-      value = parameter.value["value"]
-    }
-  }
-}
-
 resource "aws_rds_cluster_parameter_group" "rds_cluster_group_v13" {
   name   = "${var.prefix}-cluster-parameter-group-v13"
   family = var.parameter_group_family_v13
@@ -82,9 +68,9 @@ resource "aws_rds_cluster_parameter_group" "rds_cluster_group_v13" {
 }
 
 resource "aws_rds_cluster" "cumulus" {
-  depends_on              = [aws_db_subnet_group.default, aws_rds_cluster_parameter_group.rds_cluster_group]
+  depends_on              = [aws_db_subnet_group.default, aws_rds_cluster_parameter_group.rds_cluster_group_v13]
   cluster_identifier      = var.cluster_identifier
-  engine_mode             = "serverless"
+  engine_mode             = "provisioned"
   engine                  = "aurora-postgresql"
   engine_version          = var.engine_version
   database_name           = "postgres"
@@ -94,21 +80,35 @@ resource "aws_rds_cluster" "cumulus" {
   preferred_backup_window = var.backup_window
   db_subnet_group_name    = aws_db_subnet_group.default.id
   apply_immediately       = var.apply_immediately
+  storage_encrypted       = true
 
-  scaling_configuration {
+  serverlessv2_scaling_configuration {
     max_capacity = var.max_capacity
     min_capacity = var.min_capacity
-    timeout_action = var.rds_scaling_timeout_action
   }
   vpc_security_group_ids          = [aws_security_group.rds_cluster_access.id]
   deletion_protection             = var.deletion_protection
   enable_http_endpoint            = true
   tags                            = var.tags
   final_snapshot_identifier       = "${var.cluster_identifier}-final-snapshot"
   snapshot_identifier             = var.snapshot_identifier
-  db_cluster_parameter_group_name = var.enable_upgrade ? aws_rds_cluster_parameter_group.rds_cluster_group_v13.id : aws_rds_cluster_parameter_group.rds_cluster_group[0].id
+  db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.rds_cluster_group_v13.id
 
   lifecycle {
     ignore_changes = [engine_version]
+    prevent_destroy = true
   }
 }
+
+resource "aws_rds_cluster_instance" "cumulus" {
+  cluster_identifier = aws_rds_cluster.cumulus.id
+  identifier = "${aws_rds_cluster.cumulus.id}-instance-${count.index+1}"
+  count              = var.cluster_instance_count
+  instance_class     = "db.serverless"
+  engine             = aws_rds_cluster.cumulus.engine
+  engine_version     = aws_rds_cluster.cumulus.engine_version
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}

tf-modules/cumulus-rds-tf/outputs.tf

@@ -6,6 +6,10 @@ output "rds_endpoint" {
   value = aws_rds_cluster.cumulus.endpoint
 }
 
+output "rds_reader_endpoint" {
+  value = aws_rds_cluster.cumulus.reader_endpoint
+}
+
 output "admin_db_login_secret_arn" {
   value = aws_secretsmanager_secret_version.rds_login.arn
 }

tf-modules/cumulus-rds-tf/variables.tf

@@ -91,24 +91,12 @@ variable "engine_version" {
   default     = "13.12"
 }
 
-variable "parameter_group_family" {
-  description = "Database family to use for creating database parameter group"
-  type = string
-  default = "aurora-postgresql11"
-}
-
 variable "parameter_group_family_v13" {
   description = "Database family to use for creating database parameter group under postgres 13 upgrade conditions"
   type = string
   default = "aurora-postgresql13"
 }
 
-variable "enable_upgrade" {
-  description = "Flag to enable use of updated parameter group for postgres v13"
-  type = bool
-  default = true
-}
-
 variable "max_capacity" {
   type = number
   default = 4
@@ -119,6 +107,16 @@ variable "min_capacity" {
   default = 2
 }
 
+variable "cluster_instance_count" {
+  description = "Number of instances to create inside of the cluster"
+  type = number
+  default = 1
+  validation {
+    condition = var.cluster_instance_count >= 1 && var.cluster_instance_count <= 16
+    error_message = "Variable cluster_instance_count should be between 1 and 16."
+  }
+}
+
 ### Required for user/database provisioning
 variable "prefix" {
   type = string