From ba6a68dc328a214c453f4a999612f3aafced275c Mon Sep 17 00:00:00 2001 From: Ryan Miville Date: Fri, 15 Nov 2024 10:42:21 -0500 Subject: [PATCH 1/5] add retention_configuration and orphan_file_deletion_configuration --- .../service/glue/catalog_table_optimizer.go | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/service/glue/catalog_table_optimizer.go b/internal/service/glue/catalog_table_optimizer.go index 2d1c107edc9..b941da1cf9b 100644 --- a/internal/service/glue/catalog_table_optimizer.go +++ b/internal/service/glue/catalog_table_optimizer.go @@ -315,8 +315,29 @@ type resourceCatalogTableOptimizerData struct { } type configurationData struct { - Enabled types.Bool `tfsdk:"enabled"` - RoleARN fwtypes.ARN `tfsdk:"role_arn"` + Enabled types.Bool `tfsdk:"enabled"` + RoleARN fwtypes.ARN `tfsdk:"role_arn"` + RetentionConfiguration fwtypes.ListNestedObjectValueOf[retentionConfigurationData] `tfsdk:"retention_configuration"` + OrphanFileDeletionConfiguration fwtypes.ListNestedObjectValueOf[orphanFileDeletionConfigurationData] `tfsdk:"orphan_file_deletion_configuration"` +} + +type retentionConfigurationData struct { + IcebergConfiguration fwtypes.ListNestedObjectValueOf[icebergRetentionConfigurationData] `tfsdk:"iceberg_configuration"` +} + +type icebergRetentionConfigurationData struct { + SnapshotRetentionPeriodInDays types.Int32 `tfsdk:"snapshot_retention_period_in_days"` + NumberOfSnapshotsToRetain types.Int32 `tfsdk:"number_of_snapshots_to_retain"` + CleanExpiredFiles types.Bool `tfsdk:"clean_expired_files"` +} + +type orphanFileDeletionConfigurationData struct { + IcebergConfiguration fwtypes.ListNestedObjectValueOf[icebergOrphanFileDeletionConfigurationData] `tfsdk:"iceberg_configuration"` +} + +type icebergOrphanFileDeletionConfigurationData struct { + OrphanFileRetentionPeriodInDays types.Int32 `tfsdk:"orphan_file_retention_period_in_days"` + Location types.String `tfsdk:"location"` } func findCatalogTableOptimizer(ctx context.Context, conn *glue.Client, catalogID, dbName, tableName, optimizerType string) (*glue.GetTableOptimizerOutput, error) { From ca51780b283dee6eec0e6b83689ee992f927472b Mon Sep 17 00:00:00 2001 From: Ryan Miville Date: Fri, 15 Nov 2024 17:14:23 -0500 Subject: [PATCH 2/5] update schema and add acc tests --- .../service/glue/catalog_table_optimizer.go | 57 ++++++++ .../glue/catalog_table_optimizer_test.go | 128 ++++++++++++++++++ 2 files changed, 185 insertions(+) diff --git a/internal/service/glue/catalog_table_optimizer.go b/internal/service/glue/catalog_table_optimizer.go index b941da1cf9b..41400c7361c 100644 --- a/internal/service/glue/catalog_table_optimizer.go +++ b/internal/service/glue/catalog_table_optimizer.go @@ -97,6 +97,63 @@ func (r *resourceCatalogTableOptimizer) Schema(ctx context.Context, _ resource.S Required: true, }, }, + Blocks: map[string]schema.Block{ + "retention_configuration": schema.ListNestedBlock{ + CustomType: fwtypes.NewListNestedObjectTypeOf[retentionConfigurationData](ctx), + Validators: []validator.List{ + listvalidator.SizeAtMost(1), + }, + NestedObject: schema.NestedBlockObject{ + Blocks: map[string]schema.Block{ + "iceberg_configuration": schema.ListNestedBlock{ + CustomType: fwtypes.NewListNestedObjectTypeOf[icebergRetentionConfigurationData](ctx), + Validators: []validator.List{ + listvalidator.SizeAtMost(1), + }, + NestedObject: schema.NestedBlockObject{ + Attributes: map[string]schema.Attribute{ + "snapshot_retention_period_in_days": schema.Int32Attribute{ + Optional: true, + }, + "number_of_snapshots_to_retain": schema.Int32Attribute{ + Optional: true, + }, + "clean_expired_files": schema.BoolAttribute{ + Optional: true, + }, + }, + }, + }, + }, + }, + }, + "orphan_file_deletion_configuration": schema.ListNestedBlock{ + CustomType: fwtypes.NewListNestedObjectTypeOf[orphanFileDeletionConfigurationData](ctx), + Validators: []validator.List{ + listvalidator.SizeAtMost(1), + }, + NestedObject: schema.NestedBlockObject{ + Blocks: map[string]schema.Block{ + "iceberg_configuration": schema.ListNestedBlock{ + CustomType: fwtypes.NewListNestedObjectTypeOf[icebergOrphanFileDeletionConfigurationData](ctx), + Validators: []validator.List{ + listvalidator.SizeAtMost(1), + }, + NestedObject: schema.NestedBlockObject{ + Attributes: map[string]schema.Attribute{ + "orphan_file_retention_period_in_days": schema.Int32Attribute{ + Optional: true, + }, + "location": schema.StringAttribute{ + Optional: true, + }, + }, + }, + }, + }, + }, + }, + }, }, }, }, diff --git a/internal/service/glue/catalog_table_optimizer_test.go b/internal/service/glue/catalog_table_optimizer_test.go index 45956ceb5dd..b3519dcab7a 100644 --- a/internal/service/glue/catalog_table_optimizer_test.go +++ b/internal/service/glue/catalog_table_optimizer_test.go @@ -123,6 +123,83 @@ func testAccCatalogTableOptimizer_disappears(t *testing.T) { }) } +func TestAccCatalogTableOptimizer_RetentionConfiguration(t *testing.T) { + ctx := acctest.Context(t) + var catalogTableOptimizer glue.GetTableOptimizerOutput + + resourceName := "aws_glue_catalog_table_optimizer.test" + + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.GlueServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckCatalogTableOptimizerDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccCatalogTableOptimizerConfig_retentionConfiguration(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckCatalogTableOptimizerExists(ctx, resourceName, &catalogTableOptimizer), + acctest.CheckResourceAttrAccountID(resourceName, names.AttrCatalogID), + resource.TestCheckResourceAttr(resourceName, names.AttrDatabaseName, rName), + resource.TestCheckResourceAttr(resourceName, names.AttrTableName, rName), + resource.TestCheckResourceAttr(resourceName, names.AttrType, "retention"), + resource.TestCheckResourceAttr(resourceName, "configuration.0.enabled", acctest.CtTrue), + resource.TestCheckResourceAttr(resourceName, "configuration.0.retention_configuration.0.iceberg_configuration.0.snapshot_retention_period_in_days", "7"), + resource.TestCheckResourceAttr(resourceName, "configuration.0.retention_configuration.0.iceberg_configuration.0.number_of_snapshots_to_retain", "3"), + resource.TestCheckResourceAttr(resourceName, "configuration.0.retention_configuration.0.iceberg_configuration.0.clean_expired_files", acctest.CtTrue), + ), + }, + { + ResourceName: resourceName, + ImportStateIdFunc: testAccCatalogTableOptimizerStateIDFunc(resourceName), + ImportStateVerifyIdentifierAttribute: names.AttrTableName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccCatalogTableOptimizer_DeleteOrphanFileConfiguration(t *testing.T) { + ctx := acctest.Context(t) + var catalogTableOptimizer glue.GetTableOptimizerOutput + + resourceName := "aws_glue_catalog_table_optimizer.test" + + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.GlueServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckCatalogTableOptimizerDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccCatalogTableOptimizerConfig_orphanFileDeletionConfiguration(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckCatalogTableOptimizerExists(ctx, resourceName, &catalogTableOptimizer), + acctest.CheckResourceAttrAccountID(resourceName, names.AttrCatalogID), + resource.TestCheckResourceAttr(resourceName, names.AttrDatabaseName, rName), + resource.TestCheckResourceAttr(resourceName, names.AttrTableName, rName), + resource.TestCheckResourceAttr(resourceName, names.AttrType, "orphan_file_deletion"), + resource.TestCheckResourceAttr(resourceName, "configuration.0.enabled", acctest.CtTrue), + resource.TestCheckResourceAttr(resourceName, "configuration.0.orphan_file_deletion_configuration.0.iceberg_configuration.0.orphan_file_retention_period_in_days", "7"), + resource.TestCheckResourceAttr(resourceName, "configuration.0.orphan_file_deletion_configuration.0.iceberg_configuration.0.location", fmt.Sprintf("s3://%s/files/", rName)), + ), + }, + { + ResourceName: resourceName, + ImportStateIdFunc: testAccCatalogTableOptimizerStateIDFunc(resourceName), + ImportStateVerifyIdentifierAttribute: names.AttrTableName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func testAccCatalogTableOptimizerStateIDFunc(resourceName string) resource.ImportStateIdFunc { return func(s *terraform.State) (string, error) { rs, ok := s.RootModule().Resources[resourceName] @@ -345,3 +422,54 @@ resource "aws_glue_catalog_table_optimizer" "test" { } `, enabled)) } + +func testAccCatalogTableOptimizerConfig_retentionConfiguration(rName string) string { + return acctest.ConfigCompose( + testAccCatalogTableOptimizerConfig_baseConfig(rName), ` +resource "aws_glue_catalog_table_optimizer" "test" { + catalog_id = data.aws_caller_identity.current.account_id + database_name = aws_glue_catalog_database.test.name + table_name = aws_glue_catalog_table.test.name + type = "retention" + + configuration { + role_arn = aws_iam_role.test.arn + enabled = true + + retention_configuration { + iceberg_configuration { + snapshot_retention_period_in_days = 7 + number_of_snapshots_to_retain = 3 + clean_expired_files = true + } + } + } +} +`, + ) +} + +func testAccCatalogTableOptimizerConfig_orphanFileDeletionConfiguration(rName string) string { + return acctest.ConfigCompose( + testAccCatalogTableOptimizerConfig_baseConfig(rName), ` +resource "aws_glue_catalog_table_optimizer" "test" { + catalog_id = data.aws_caller_identity.current.account_id + database_name = aws_glue_catalog_database.test.name + table_name = aws_glue_catalog_table.test.name + type = "orphan_file_deletion" + + configuration { + role_arn = aws_iam_role.test.arn + enabled = true + + orphan_file_deletion_configuration { + iceberg_configuration { + orphan_file_retention_period_in_days = 7 + location = "s3://${aws_s3_bucket.bucket.bucket}/files/" + } + } + } +} +`, + ) +} From 5e898819043ac6fd196eb5e7683a11acdc0e629f Mon Sep 17 00:00:00 2001 From: Ryan Miville Date: Tue, 19 Nov 2024 16:15:42 -0500 Subject: [PATCH 3/5] update documentation --- ...glue_catalog_table_optimizer.html.markdown | 88 +++++++++++++++++-- 1 file changed, 83 insertions(+), 5 deletions(-) diff --git a/website/docs/r/glue_catalog_table_optimizer.html.markdown b/website/docs/r/glue_catalog_table_optimizer.html.markdown index a3140c27aff..9aa00f82c52 100644 --- a/website/docs/r/glue_catalog_table_optimizer.html.markdown +++ b/website/docs/r/glue_catalog_table_optimizer.html.markdown @@ -12,7 +12,7 @@ Terraform resource for managing an AWS Glue Catalog Table Optimizer. ## Example Usage -### Basic Usage +### Compaction Optimizer ```terraform resource "aws_glue_catalog_table_optimizer" "example" { @@ -29,6 +29,57 @@ resource "aws_glue_catalog_table_optimizer" "example" { } ``` +### Snapshot Retention Optimizer + +```terraform +resource "aws_glue_catalog_table_optimizer" "example" { + catalog_id = "123456789012" + database_name = "example_database" + table_name = "example_table" + + configuration { + role_arn = "arn:aws:iam::123456789012:role/example-role" + enabled = true + + retention_configuration { + iceberg_configuration { + snapshot_retention_period_in_days = 7 + number_of_snapshots_to_retain = 3 + clean_expired_files = true + } + } + + } + + type = "retention" +} +``` + +### Orphan File Deletion Optimizer + +```terraform +resource "aws_glue_catalog_table_optimizer" "example" { + catalog_id = "123456789012" + database_name = "example_database" + table_name = "example_table" + + configuration { + role_arn = "arn:aws:iam::123456789012:role/example-role" + enabled = true + + orphan_file_deletion_configuration { + iceberg_configuration { + orphan_file_retention_period_in_days = 7 + location = "s3://example-bucket/example_table/" + } + } + + } + + type = "orphan_file_deletion" +} +``` + ## Argument Reference The following arguments are required: @@ -36,10 +87,37 @@ The following arguments are required: * `catalog_id` - (Required) The Catalog ID of the table. * `database_name` - (Required) The name of the database in the catalog in which the table resides. * `table_name` - (Required) The name of the table. -* `type` - (Required) The type of table optimizer. Currently, the only valid value is compaction. -* `configuration` - (Required) A configuration block that defines the table optimizer settings. The block contains: - * `role_arn` - (Required) The ARN of the IAM role to use for the table optimizer. - * `enabled` - (Required) Indicates whether the table optimizer is enabled. +* `type` - (Required) The type of table optimizer. Valid values are `compaction`, `retention`, and `orphan_file_deletion`. +* `configuration` - (Required) A configuration block that defines the table optimizer settings. Detailed below. + +### `configuration` + +The following arguments are required: + +* `role_arn` - (Required) The ARN of the IAM role to use for the table optimizer. +* `enabled` - (Required) Indicates whether the table optimizer is enabled. + +The following arguments are optional: + +* `retention_configuration` (Optional) - The configuration block for a snapshot retention optimizer. Detailed below. +* `orphan_file_deletion_configuration` (Optional) - The configuration block for an orphan file deletion optimizer. Detailed below. + +### `retention_configuration` + +The following arguments are optional: + +* `iceberg_configuration` (Optional) - The configuration for an Iceberg snapshot retention optimizer. + * `snapshot_retention_period_in_days` (Optional) - The number of days to retain the Iceberg snapshots. Defaults to `5`, or the corresponding Iceberg table configuration field if it exsists. + * `number_of_snapshots_to_retain` (Optional) - The number of Iceberg snapshots to retain within the retention period. Defaults to `1` or the corresponding Icebert table configuration field if it exists. + * `clean_expired_files` (Optional) - If set to `false`, snapshots are only deleted from table metadata, and the underlying data and metadata files are not deleted. Defaults to `false`. + +### `orphan_file_deletion_configuration` + +The following arguments are optional: + +* `iceberg_configuration` (Optional) - The configuration for an Iceberg orphan file deletion optimizer. + * `orphan_file_retention_period_in_days` (Optional) - The number of days that orphan files should be retained before file deletion. Defaults to `3`. + * `location` (Optional) - Specifies a directory in which to look for files. You may choose a sub-directory rather than the top-level table location. Defaults to the table's location. ## Attribute Reference From 7baa0a332c7497f70baf377a608efa92bff71156 Mon Sep 17 00:00:00 2001 From: Ryan Miville Date: Tue, 19 Nov 2024 16:50:58 -0500 Subject: [PATCH 4/5] fix test names --- internal/service/glue/catalog_table_optimizer_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/service/glue/catalog_table_optimizer_test.go b/internal/service/glue/catalog_table_optimizer_test.go index b3519dcab7a..ffb639530d3 100644 --- a/internal/service/glue/catalog_table_optimizer_test.go +++ b/internal/service/glue/catalog_table_optimizer_test.go @@ -123,7 +123,7 @@ func testAccCatalogTableOptimizer_disappears(t *testing.T) { }) } -func TestAccCatalogTableOptimizer_RetentionConfiguration(t *testing.T) { +func TestAccGlueCatalogTableOptimizer_RetentionConfiguration(t *testing.T) { ctx := acctest.Context(t) var catalogTableOptimizer glue.GetTableOptimizerOutput @@ -162,7 +162,7 @@ func TestAccCatalogTableOptimizer_RetentionConfiguration(t *testing.T) { }) } -func TestAccCatalogTableOptimizer_DeleteOrphanFileConfiguration(t *testing.T) { +func TestAccGlueCatalogTableOptimizer_DeleteOrphanFileConfiguration(t *testing.T) { ctx := acctest.Context(t) var catalogTableOptimizer glue.GetTableOptimizerOutput From 9e1022f5239d1e22a118ed267f86562bc00241f8 Mon Sep 17 00:00:00 2001 From: Ryan Miville Date: Tue, 19 Nov 2024 17:05:19 -0500 Subject: [PATCH 5/5] add changelog --- .archive/changelog/40199.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .archive/changelog/40199.txt diff --git a/.archive/changelog/40199.txt b/.archive/changelog/40199.txt new file mode 100644 index 00000000000..ba7fa111ea0 --- /dev/null +++ b/.archive/changelog/40199.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +resource/aws_glue_catalog_table_optimizer: Add configuration support for `retention` and `orphan_file_deletion` types. +```