feat: Add support for additional s3_settings attributes on DMS endp…

…oint for upstream hashicorp/terraform-provider-aws#20913

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.62.3
+    rev: v1.64.0
     hooks:
       - id: terraform_fmt
       - id: terraform_validate
@@ -23,7 +23,7 @@ repos:
           - '--args=--only=terraform_standard_module_structure'
           - '--args=--only=terraform_workspace_remote'
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.1.0
     hooks:
       - id: check-merge-conflict
       - id: end-of-file-fixer

README.md

@@ -318,13 +318,13 @@ Security scanning results provided by Bridgecrew. Bridgecrew is the leading full
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.1 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.64 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.2 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.64 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.2 |
 
 ## Modules
 

examples/complete/README.md

@@ -30,14 +30,14 @@ Note that this example may create resources which will incur monetary charges on
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.1 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.64 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.2 |
 | <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.64 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.2 |
 | <a name="provider_random"></a> [random](#provider\_random) | >= 3.0 |
 
 ## Modules
@@ -49,7 +49,6 @@ Note that this example may create resources which will incur monetary charges on
 | <a name="module_dms_disabled"></a> [dms\_disabled](#module\_dms\_disabled) | ../.. | n/a |
 | <a name="module_msk_cluster"></a> [msk\_cluster](#module\_msk\_cluster) | clowdhaus/msk-kafka-cluster/aws | ~> 1.0 |
 | <a name="module_rds_aurora"></a> [rds\_aurora](#module\_rds\_aurora) | terraform-aws-modules/rds-aurora/aws | ~> 6.0 |
-| <a name="module_s3_bucket"></a> [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 2.0 |
 | <a name="module_security_group"></a> [security\_group](#module\_security\_group) | terraform-aws-modules/security-group/aws | ~> 4.0 |
 | <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 |
 | <a name="module_vpc_endpoint_security_group"></a> [vpc\_endpoint\_security\_group](#module\_vpc\_endpoint\_security\_group) | terraform-aws-modules/security-group/aws | ~> 4.0 |
@@ -62,7 +61,8 @@ Note that this example may create resources which will incur monetary charges on
 | [aws_iam_role.s3_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_kms_key.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
 | [aws_rds_cluster_parameter_group.postgresql](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rds_cluster_parameter_group) | resource |
-| [aws_s3_bucket_object.hr_data](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_object) | resource |
+| [aws_s3_bucket.example](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
+| [aws_s3_object.hr_data](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
 | [aws_secretsmanager_secret.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource |
 | [aws_secretsmanager_secret_policy.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_policy) | resource |
 | [aws_secretsmanager_secret_version.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource |

examples/complete/main.tf

@@ -219,32 +219,39 @@ resource "aws_sns_topic" "example" {
   tags = local.tags
 }
 
-module "s3_bucket" {
-  source  = "terraform-aws-modules/s3-bucket/aws"
-  version = "~> 2.0"
+# TODO - disabling until v4.x of provider is supported
+# module "s3_bucket" {
+#   source  = "terraform-aws-modules/s3-bucket/aws"
+#   version = "~> 2.0"
 
-  bucket = local.bucket_name
+#   bucket = local.bucket_name
 
-  attach_deny_insecure_transport_policy = true
+#   attach_deny_insecure_transport_policy = true
 
-  block_public_acls       = true
-  block_public_policy     = true
-  ignore_public_acls      = true
-  restrict_public_buckets = true
+#   block_public_acls       = true
+#   block_public_policy     = true
+#   ignore_public_acls      = true
+#   restrict_public_buckets = true
 
-  server_side_encryption_configuration = {
-    rule = {
-      apply_server_side_encryption_by_default = {
-        sse_algorithm = "AES256"
-      }
-    }
-  }
+#   server_side_encryption_configuration = {
+#     rule = {
+#       apply_server_side_encryption_by_default = {
+#         sse_algorithm = "AES256"
+#       }
+#     }
+#   }
+
+#   tags = local.tags
+# }
+
+resource "aws_s3_bucket" "example" {
+  bucket = local.bucket_name
 
   tags = local.tags
 }
 
-resource "aws_s3_bucket_object" "hr_data" {
-  bucket                 = module.s3_bucket.s3_bucket_id
+resource "aws_s3_object" "hr_data" {
+  bucket                 = aws_s3_bucket.example.id
   key                    = "sourcedata/hr/employee/LOAD0001.csv"
   source                 = "data/hr.csv"
   etag                   = filemd5("data/hr.csv")
@@ -281,13 +288,13 @@ resource "aws_iam_role" "s3_role" {
           Sid      = "DMSRead"
           Action   = ["s3:GetObject"]
           Effect   = "Allow"
-          Resource = "${module.s3_bucket.s3_bucket_arn}/*"
+          Resource = "${aws_s3_bucket.example.arn}/*"
         },
         {
           Sid      = "DMSList"
           Action   = ["s3:ListBucket"]
           Effect   = "Allow"
-          Resource = module.s3_bucket.s3_bucket_arn
+          Resource = aws_s3_bucket.example.arn
         },
       ]
     })

examples/complete/versions.tf

@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.64"
+      version = ">= 4.2"
     }
     random = {
       source  = "hashicorp/random"

main.tf

@@ -28,7 +28,7 @@ data "aws_iam_policy_document" "dms_assume_role" {
 data "aws_iam_policy_document" "dms_assume_role_redshift" {
   count = var.create && var.create_iam_roles ? 1 : 0
 
-  source_json = data.aws_iam_policy_document.dms_assume_role[0].json
+  source_policy_documents = [data.aws_iam_policy_document.dms_assume_role[0].json]
 
   statement {
     actions = ["sts:AssumeRole"]
@@ -176,7 +176,7 @@ resource "aws_dms_endpoint" "this" {
 
   # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.Elasticsearch.html
   dynamic "elasticsearch_settings" {
-    for_each = can(each.value["elasticsearch_settings"]) ? [each.value.elasticsearch_settings] : []
+    for_each = try([each.value.elasticsearch_settings], [])
     content {
       endpoint_uri               = elasticsearch_settings.value.endpoint_uri
       error_retry_duration       = lookup(elasticsearch_settings.value, "error_retry_duration", null)
@@ -187,7 +187,7 @@ resource "aws_dms_endpoint" "this" {
 
   # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.Kafka.html
   dynamic "kafka_settings" {
-    for_each = can(each.value["kafka_settings"]) ? [each.value.kafka_settings] : []
+    for_each = try([each.value.kafka_settings], [])
     content {
       broker                         = kafka_settings.value.broker
       include_control_details        = lookup(kafka_settings.value, "include_control_details", null)
@@ -212,7 +212,7 @@ resource "aws_dms_endpoint" "this" {
 
   # https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.Kinesis.html
   dynamic "kinesis_settings" {
-    for_each = can(each.value["kinesis_settings"]) ? [each.value.kinesis_settings] : []
+    for_each = try([each.value.kinesis_settings], [])
     content {
       include_control_details        = lookup(kinesis_settings.value, "include_control_details", null)
       include_null_and_empty         = lookup(kinesis_settings.value, "include_null_and_empty", null)
@@ -244,19 +244,42 @@ resource "aws_dms_endpoint" "this" {
   dynamic "s3_settings" {
     for_each = can(each.value["s3_settings"]) ? [each.value.s3_settings] : []
     content {
+      add_column_name                   = lookup(s3_settings.value, "add_column_name", null)
       bucket_folder                     = lookup(s3_settings.value, "bucket_folder", null)
       bucket_name                       = lookup(s3_settings.value, "bucket_name", null)
+      canned_acl_for_objects            = lookup(s3_settings.value, "canned_acl_for_objects", null)
+      cdc_inserts_and_updates           = lookup(s3_settings.value, "cdc_inserts_and_updates", null)
+      cdc_inserts_only                  = lookup(s3_settings.value, "cdc_inserts_only", null)
+      cdc_max_batch_interval            = lookup(s3_settings.value, "cdc_max_batch_interval", null)
+      cdc_min_file_size                 = lookup(s3_settings.value, "cdc_min_file_size", null)
+      cdc_path                          = lookup(s3_settings.value, "cdc_path", null)
       compression_type                  = lookup(s3_settings.value, "compression_type", null)
       csv_delimiter                     = lookup(s3_settings.value, "csv_delimiter", null)
+      csv_no_sup_value                  = lookup(s3_settings.value, "csv_no_sup_value", null)
+      csv_null_value                    = lookup(s3_settings.value, "csv_null_value", null)
       csv_row_delimiter                 = lookup(s3_settings.value, "csv_row_delimiter", null)
       data_format                       = lookup(s3_settings.value, "data_format", null)
+      data_page_size                    = lookup(s3_settings.value, "data_page_size", null)
+      date_partition_delimiter          = lookup(s3_settings.value, "date_partition_delimiter", null)
       date_partition_enabled            = lookup(s3_settings.value, "date_partition_enabled", null)
+      date_partition_sequence           = lookup(s3_settings.value, "date_partition_sequence", null)
+      dict_page_size_limit              = lookup(s3_settings.value, "dict_page_size_limit", null)
+      enable_statistics                 = lookup(s3_settings.value, "enable_statistics", null)
+      encoding_type                     = lookup(s3_settings.value, "encoding_type", null)
       encryption_mode                   = lookup(s3_settings.value, "encryption_mode", null)
       external_table_definition         = lookup(s3_settings.value, "external_table_definition", null)
+      ignore_headers_row                = lookup(s3_settings.value, "ignore_headers_row", null)
+      include_op_for_full_load          = lookup(s3_settings.value, "include_op_for_full_load", null)
+      max_file_size                     = lookup(s3_settings.value, "max_file_size", null)
       parquet_timestamp_in_millisecond  = lookup(s3_settings.value, "parquet_timestamp_in_millisecond", null)
       parquet_version                   = lookup(s3_settings.value, "parquet_version", null)
+      preserve_transactions             = lookup(s3_settings.value, "preserve_transactions", null)
+      rfc_4180                          = lookup(s3_settings.value, "rfc_4180", null)
+      row_group_length                  = lookup(s3_settings.value, "row_group_length", null)
       server_side_encryption_kms_key_id = lookup(s3_settings.value, "server_side_encryption_kms_key_id", null)
       service_access_role_arn           = lookup(s3_settings.value, "service_access_role_arn", null)
+      timestamp_column_name             = lookup(s3_settings.value, "timestamp_column_name", null)
+      use_csv_no_sup_value              = lookup(s3_settings.value, "use_csv_no_sup_value", null)
     }
   }
 

versions.tf

@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.64"
+      version = ">= 4.2"
     }
   }
 }