From ffef36dba27e728c5097dbe08fb739b0c05b0cff Mon Sep 17 00:00:00 2001 From: Guillaume Legrain <30684712+SoulKyu@users.noreply.github.com> Date: Sat, 9 Nov 2024 02:14:33 +0100 Subject: [PATCH] Breaking change: Update MinIO ILM policy with improved validation, noncurrent version support, and better schema structure (#588) Co-authored-by: Victor Nogueira --- docs/resources/ilm_policy.md | 102 +++-- .../resources/minio_ilm_policy/resource.tf | 70 +++- minio/resource_minio_ilm_policy.go | 354 ++++++++++++++---- minio/resource_minio_ilm_policy_test.go | 14 +- 4 files changed, 434 insertions(+), 106 deletions(-) diff --git a/docs/resources/ilm_policy.md b/docs/resources/ilm_policy.md index e94f615e..2bc7138d 100644 --- a/docs/resources/ilm_policy.md +++ b/docs/resources/ilm_policy.md @@ -17,6 +17,7 @@ resource "minio_s3_bucket" "bucket" { bucket = "bucket" } +# Simple expiration rule resource "minio_ilm_policy" "bucket-lifecycle-rules" { bucket = minio_s3_bucket.bucket.bucket @@ -25,48 +26,107 @@ resource "minio_ilm_policy" "bucket-lifecycle-rules" { expiration = "7d" } } + +# Complex lifecycle policy with multiple rules +resource "minio_ilm_policy" "comprehensive-rules" { + bucket = minio_s3_bucket.bucket.bucket + + # Rule with transition and expiration + rule { + id = "documents" + transition { + days = "30d" + storage_class = "STANDARD_IA" + } + expiration = "90d" + filter = "documents/" + tags = { + "department" = "finance" + } + } + + # Rule with noncurrent version management + rule { + id = "versioning" + noncurrent_expiration { + days = "60d" + newer_versions = 5 + } + noncurrent_transition { + days = "30d" + storage_class = "GLACIER" + newer_versions = 3 + } + } +} ``` - ## Schema ### Required -- `bucket` (String) -- `rule` (Block List, Min: 1) (see [below for nested schema](#nestedblock--rule)) +- `bucket` (String) The name of the bucket to which this lifecycle policy applies. Must be between 0 and 63 characters. +- `rule` (Block List, Min: 1) A list of lifecycle rules (see below for nested schema). ### Read-Only - `id` (String) The ID of this resource. - ### Nested Schema for `rule` -Required: +#### Required -- `id` (String) +- `id` (String) Unique identifier for the rule. -Optional: +#### Optional -- `expiration` (String) Value may be duration (5d), date (1970-01-01), or "DeleteMarker" to expire delete markers if `noncurrent_version_expiration_days` is used -- `filter` (String) -- `noncurrent_version_expiration_days` (Number) -- `noncurrent_version_transition_days` (Number) -- `tags` (Map of String) -- `transition` (Block List, Max: 1) (see [below for nested schema](#nestedblock--rule--transition)) +- `expiration` (String) When objects should expire. Value must be a duration (e.g., "7d"), date (e.g., "2024-12-31"), or "DeleteMarker". +- `filter` (String) Prefix path filter for the rule. +- `tags` (Map of String) Key-value pairs of tags to filter objects. +- `transition` (Block List, Max: 1) Configuration block for transitioning objects to a different storage class (see below). +- `noncurrent_transition` (Block List, Max: 1) Configuration for transitioning noncurrent object versions (see below). +- `noncurrent_expiration` (Block List, Max: 1) Configuration for expiring noncurrent object versions (see below). -Read-Only: +#### Read-Only -- `status` (String) +- `status` (String) Current status of the rule. - ### Nested Schema for `rule.transition` -Required: +#### Required + +- `storage_class` (String) The storage class to transition objects to. + +#### Optional + +- `days` (String) Number of days after which objects should transition, in format "Nd" (e.g., "30d"). +- `date` (String) Specific date for the transition in "YYYY-MM-DD" format. + +### Nested Schema for `rule.noncurrent_transition` + +#### Required + +- `days` (String) Number of days after which noncurrent versions should transition, in format "Nd". +- `storage_class` (String) The storage class to transition noncurrent versions to. + +#### Optional + +- `newer_versions` (Number) Number of newer versions to retain before transition. Must be non-negative. + +### Nested Schema for `rule.noncurrent_expiration` + +#### Required + +- `days` (String) Number of days after which noncurrent versions should be deleted, in format "Nd". + +#### Optional + +- `newer_versions` (Number) Number of newer versions to retain before expiration. Must be non-negative. -- `storage_class` (String) +## Import -Optional: +MinIO lifecycle policies can be imported using the bucket name: -- `date` (String) -- `days` (String) +```shell +terraform import minio_ilm_policy.example bucket-name +``` \ No newline at end of file diff --git a/examples/resources/minio_ilm_policy/resource.tf b/examples/resources/minio_ilm_policy/resource.tf index 52479672..8ac66c38 100644 --- a/examples/resources/minio_ilm_policy/resource.tf +++ b/examples/resources/minio_ilm_policy/resource.tf @@ -2,11 +2,73 @@ resource "minio_s3_bucket" "bucket" { bucket = "bucket" } -resource "minio_ilm_policy" "bucket-lifecycle-rules" { +resource "minio_ilm_policy" "test_policy" { bucket = minio_s3_bucket.bucket.bucket rule { - id = "expire-7d" - expiration = "7d" + id = "rule1" + # Delete objects after 90 days + expiration = "90d" + filter = "documents/" + tags = { + "environment" = "test" + "type" = "document" + } } -} + + rule { + id = "rule2" + # Move objects to GLACIER after 30 days + transition { + days = "30d" + storage_class = "GLACIER" + } + filter = "backups/" + } + + rule { + id = "rule3" + # Specific date for transition + transition { + date = "2024-12-31" + storage_class = "STANDARD_IA" + } + filter = "archives/" + } + + rule { + id = "rule4" + # Handle noncurrent versions + noncurrent_transition { + days = "45d" + storage_class = "GLACIER" + newer_versions = 3 + } + filter = "versioned-data/" + } + + rule { + id = "rule5" + # Delete old versions + noncurrent_expiration { + days = "365d" + newer_versions = 5 + } + filter = "old-versions/" + } + + rule { + id = "rule6" + # Combined policy: transition then expire + transition { + days = "60d" + storage_class = "STANDARD_IA" + } + expiration = "180d" + filter = "logs/" + tags = { + "retention" = "short-term" + "type" = "logs" + } + } +} \ No newline at end of file diff --git a/minio/resource_minio_ilm_policy.go b/minio/resource_minio_ilm_policy.go index c09049e2..0039aca7 100644 --- a/minio/resource_minio_ilm_policy.go +++ b/minio/resource_minio_ilm_policy.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "strings" "time" "github.com/hashicorp/go-cty/cty" @@ -45,7 +46,6 @@ func resourceMinioILMPolicy() *schema.Resource { Description: "Value may be duration (5d), date (1970-01-01), or \"DeleteMarker\" to expire delete markers if `noncurrent_version_expiration_days` is used", ValidateDiagFunc: validateILMExpiration, }, - "transition": { Type: schema.TypeList, MaxItems: 1, @@ -53,12 +53,14 @@ func resourceMinioILMPolicy() *schema.Resource { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "days": { - Type: schema.TypeString, - Optional: true, + Type: schema.TypeString, + Optional: true, + ValidateDiagFunc: validateILMDays, }, "date": { - Type: schema.TypeString, - Optional: true, + Type: schema.TypeString, + Optional: true, + ValidateDiagFunc: validateILMDate, }, "storage_class": { Type: schema.TypeString, @@ -67,15 +69,47 @@ func resourceMinioILMPolicy() *schema.Resource { }, }, }, - "noncurrent_version_expiration_days": { - Type: schema.TypeInt, - Optional: true, - ValidateDiagFunc: validateILMNoncurrentVersionExpiration, + "noncurrent_transition": { + Type: schema.TypeList, + MaxItems: 1, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "storage_class": { + Type: schema.TypeString, + Required: true, + }, + "days": { + Type: schema.TypeString, + Required: true, + ValidateDiagFunc: validateILMDays, + }, + "newer_versions": { + Type: schema.TypeInt, + Optional: true, + ValidateDiagFunc: validateILMVersions, + }, + }, + }, }, - "noncurrent_version_transition_days": { - Type: schema.TypeInt, - Optional: true, - ValidateDiagFunc: validateILMNoncurrentVersionTransition, + "noncurrent_expiration": { + Type: schema.TypeList, + MaxItems: 1, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "days": { + Type: schema.TypeString, + Required: true, + ValidateDiagFunc: validateILMDays, + }, + "newer_versions": { + Type: schema.TypeInt, + Optional: true, + ValidateDiagFunc: validateILMVersions, + }, + }, + }, }, "status": { Type: schema.TypeString, @@ -88,6 +122,9 @@ func resourceMinioILMPolicy() *schema.Resource { "tags": { Type: schema.TypeMap, Optional: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, }, }, }, @@ -107,74 +144,132 @@ func validateILMExpiration(v interface{}, p cty.Path) (errors diag.Diagnostics) return } -func validateILMNoncurrentVersionExpiration(v interface{}, p cty.Path) (errors diag.Diagnostics) { - value := v.(int) - - if value < 1 { - return diag.Errorf("noncurrent_version_expiration_days must be strictly positive") +func validateILMDays(v interface{}, p cty.Path) diag.Diagnostics { + value := v.(string) + var days int + if _, err := fmt.Sscanf(value, "%dd", &days); err != nil { + return diag.Errorf("days must be in format '(number)d', got: %s", value) } + if days < 1 { + return diag.Errorf("days must be greater than 0, got: %d", days) + } + return nil +} - return +func validateILMDate(v interface{}, p cty.Path) diag.Diagnostics { + value := v.(string) + if _, err := time.Parse("2006-01-02", value); err != nil { + return diag.Errorf("date must be in format 'YYYY-MM-DD', got: %s", value) + } + return nil } -func validateILMNoncurrentVersionTransition(v interface{}, p cty.Path) (errors diag.Diagnostics) { +func validateILMVersions(v interface{}, p cty.Path) diag.Diagnostics { value := v.(int) - - if value < 1 { - return diag.Errorf("noncurrent_version_transition_days must be strictly positive") + if value < 0 { + return diag.Errorf("newer_versions must be non-negative, got: %d", value) } - - return + return nil } func minioCreateILMPolicy(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { c := meta.(*S3MinioClient).S3Client + bucket := d.Get("bucket").(string) - config := lifecycle.NewConfiguration() + _, err := c.BucketExists(ctx, bucket) + if err != nil { + return diag.FromErr(fmt.Errorf("bucket validation failed: %v", err)) + } - bucket := d.Get("bucket").(string) + oldConfig, err := c.GetBucketLifecycle(ctx, bucket) + if err != nil && !isNotFoundError(err) { + return diag.FromErr(fmt.Errorf("failed to get existing lifecycle: %v", err)) + } + + config := lifecycle.NewConfiguration() rules := d.Get("rule").([]interface{}) - for _, ruleI := range rules { - rule := ruleI.(map[string]interface{}) - var filter lifecycle.Filter + for _, ruleI := range rules { + rule, ok := ruleI.(map[string]interface{}) + if !ok { + return diag.Errorf("invalid rule format") + } - noncurrentVersionExpirationDays := lifecycle.NoncurrentVersionExpiration{NoncurrentDays: lifecycle.ExpirationDays(rule["noncurrent_version_expiration_days"].(int))} - noncurrentVersionTransitionDays := lifecycle.NoncurrentVersionTransition{NoncurrentDays: lifecycle.ExpirationDays(rule["noncurrent_version_transition_days"].(int))} - tags := map[string]string{} - for k, v := range rule["tags"].(map[string]interface{}) { - tags[k] = v.(string) + lifecycleRule, err := createLifecycleRule(rule) + if err != nil { + return diag.FromErr(err) } - if len(tags) > 0 { - filter.And.Prefix = rule["filter"].(string) - for k, v := range tags { - filter.And.Tags = append(filter.And.Tags, lifecycle.Tag{Key: k, Value: v}) + config.Rules = append(config.Rules, lifecycleRule) + } + + if err := c.SetBucketLifecycle(ctx, bucket, config); err != nil { + if oldConfig != nil { + if rbErr := c.SetBucketLifecycle(ctx, bucket, oldConfig); rbErr != nil { + return diag.FromErr(fmt.Errorf("policy update failed and rollback failed: %v, rollback error: %v", err, rbErr)) } - } else { - filter.Prefix = rule["filter"].(string) } + return diag.FromErr(fmt.Errorf("failed to set lifecycle: %v", err)) + } - r := lifecycle.Rule{ - ID: rule["id"].(string), - Expiration: parseILMExpiration(rule["expiration"].(string)), - Transition: parseILMTransition(rule["transition"].([]interface{})), - NoncurrentVersionExpiration: noncurrentVersionExpirationDays, - NoncurrentVersionTransition: noncurrentVersionTransitionDays, - Status: "Enabled", - RuleFilter: filter, + d.SetId(bucket) + return minioReadILMPolicy(ctx, d, meta) +} + +func createLifecycleRule(ruleData map[string]interface{}) (lifecycle.Rule, error) { + id, ok := getStringValue(ruleData, "id") + if !ok { + return lifecycle.Rule{}, fmt.Errorf("rule id is required") + } + + if transition, exists := ruleData["transition"].([]interface{}); exists && len(transition) > 0 { + t := transition[0].(map[string]interface{}) + if _, ok := t["storage_class"].(string); !ok { + return lifecycle.Rule{}, fmt.Errorf("storage_class is required for transition") } + } - config.Rules = append(config.Rules, r) + if nt, exists := ruleData["noncurrent_transition"].([]interface{}); exists && len(nt) > 0 { + t := nt[0].(map[string]interface{}) + days, ok := getStringValue(t, "days") + if !ok { + return lifecycle.Rule{}, fmt.Errorf("days is required for noncurrent_transition") + } + if err := validateILMDays(days, nil); err != nil { + return lifecycle.Rule{}, fmt.Errorf("invalid days format: %v", err) + } } - if err := c.SetBucketLifecycle(ctx, bucket, config); err != nil { - return NewResourceError("creating bucket lifecycle failed", bucket, err) + var filter lifecycle.Filter + tags := convertToStringMap(ruleData["tags"]) + + if len(tags) > 0 { + prefix, _ := getStringValue(ruleData, "filter") + filter.And.Prefix = prefix + for k, v := range tags { + filter.And.Tags = append(filter.And.Tags, lifecycle.Tag{Key: k, Value: v}) + } + } else { + prefix, _ := getStringValue(ruleData, "filter") + filter.Prefix = prefix } - d.SetId(bucket) + expiration, _ := getStringValue(ruleData, "expiration") - return minioReadILMPolicy(ctx, d, meta) + noncurrentTransition, err := parseILMNoncurrentTransition(ruleData["noncurrent_transition"]) + if err != nil { + return lifecycle.Rule{}, err + } + + return lifecycle.Rule{ + ID: id, + Expiration: parseILMExpiration(expiration), + Transition: parseILMTransition(ruleData["transition"]), + NoncurrentVersionExpiration: parseILMNoncurrentExpiration(ruleData["noncurrent_expiration"]), + NoncurrentVersionTransition: noncurrentTransition, + Status: "Enabled", + RuleFilter: filter, + }, nil } func minioReadILMPolicy(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { @@ -204,10 +299,9 @@ func minioReadILMPolicy(ctx context.Context, d *schema.ResourceData, meta interf expiration = r.Expiration.Date.Format("2006-01-02") } - transitions := make([]map[string]string, 0) - + transitions := make([]map[string]interface{}, 0) if !r.Transition.IsNull() { - transition := map[string]string{} + transition := map[string]interface{}{} if !r.Transition.IsDaysNull() { transition["days"] = fmt.Sprintf("%dd", r.Transition.Days) } else if !r.Transition.IsDateNull() { @@ -215,17 +309,23 @@ func minioReadILMPolicy(ctx context.Context, d *schema.ResourceData, meta interf } transition["storage_class"] = r.Transition.StorageClass transitions = append(transitions, transition) - } - var noncurrentVersionExpirationDays int + noncurrentExpiration := make([]map[string]interface{}, 0) if r.NoncurrentVersionExpiration.NoncurrentDays != 0 { - noncurrentVersionExpirationDays = int(r.NoncurrentVersionExpiration.NoncurrentDays) + noncurrentExpiration = append(noncurrentExpiration, map[string]interface{}{ + "days": fmt.Sprintf("%dd", r.NoncurrentVersionExpiration.NoncurrentDays), + "newer_versions": r.NoncurrentVersionExpiration.NewerNoncurrentVersions, + }) } - var noncurrentVersionTransitionDays int + noncurrentTransition := make([]map[string]interface{}, 0) if r.NoncurrentVersionTransition.NoncurrentDays != 0 { - noncurrentVersionTransitionDays = int(r.NoncurrentVersionTransition.NoncurrentDays) + noncurrentTransition = append(noncurrentTransition, map[string]interface{}{ + "days": fmt.Sprintf("%dd", r.NoncurrentVersionTransition.NoncurrentDays), + "storage_class": r.NoncurrentVersionTransition.StorageClass, + "newer_versions": r.NoncurrentVersionTransition.NewerNoncurrentVersions, + }) } var prefix string @@ -240,14 +340,14 @@ func minioReadILMPolicy(ctx context.Context, d *schema.ResourceData, meta interf } rule := map[string]interface{}{ - "id": r.ID, - "expiration": expiration, - "transition": transitions, - "noncurrent_version_expiration_days": noncurrentVersionExpirationDays, - "noncurrent_version_transition_days": noncurrentVersionTransitionDays, - "status": r.Status, - "filter": prefix, - "tags": tags, + "id": r.ID, + "expiration": expiration, + "transition": transitions, + "noncurrent_expiration": noncurrentExpiration, + "noncurrent_transition": noncurrentTransition, + "status": r.Status, + "filter": prefix, + "tags": tags, } rules = append(rules, rule) @@ -302,14 +402,118 @@ func parseILMTransition(transition interface{}) lifecycle.Transition { if len(transitions) == 0 { return lifecycle.Transition{} } + t := transitions[0].(map[string]interface{}) - var days int - if _, err := fmt.Sscanf(t["days"].(string), "%dd", &days); err == nil { - return lifecycle.Transition{Days: lifecycle.ExpirationDays(days), StorageClass: t["storage_class"].(string)} + if t == nil { + return lifecycle.Transition{} } - if date, err := time.Parse("2006-01-02", t["date"].(string)); err == nil { - return lifecycle.Transition{Date: lifecycle.ExpirationDate{Time: date}, StorageClass: t["storage_class"].(string)} + + days, ok := t["days"].(string) + if ok && days != "" { + var daysInt int + if _, err := fmt.Sscanf(days, "%dd", &daysInt); err == nil { + return lifecycle.Transition{ + Days: lifecycle.ExpirationDays(daysInt), + StorageClass: t["storage_class"].(string), + } + } + } + + date, ok := t["date"].(string) + if ok && date != "" { + if parsedDate, err := time.Parse("2006-01-02", date); err == nil { + return lifecycle.Transition{ + Date: lifecycle.ExpirationDate{Time: parsedDate}, + StorageClass: t["storage_class"].(string), + } + } } return lifecycle.Transition{} } + +func parseILMNoncurrentTransition(noncurrentTransition interface{}) (lifecycle.NoncurrentVersionTransition, error) { + if noncurrentTransition == nil { + return lifecycle.NoncurrentVersionTransition{}, nil + } + + transitions, ok := noncurrentTransition.([]interface{}) + if !ok || len(transitions) == 0 { + return lifecycle.NoncurrentVersionTransition{}, nil + } + + t, ok := transitions[0].(map[string]interface{}) + if !ok || t == nil { + return lifecycle.NoncurrentVersionTransition{}, fmt.Errorf("invalid noncurrent_transition format") + } + + days, ok := getStringValue(t, "days") + if !ok { + return lifecycle.NoncurrentVersionTransition{}, fmt.Errorf("days is required") + } + + var daysInt int + if _, err := fmt.Sscanf(days, "%dd", &daysInt); err != nil { + return lifecycle.NoncurrentVersionTransition{}, fmt.Errorf("invalid days format: %s", days) + } + + return lifecycle.NoncurrentVersionTransition{ + NoncurrentDays: lifecycle.ExpirationDays(daysInt), + StorageClass: t["storage_class"].(string), + NewerNoncurrentVersions: t["newer_versions"].(int), + }, nil +} + +func parseILMNoncurrentExpiration(noncurrentExpiration interface{}) lifecycle.NoncurrentVersionExpiration { + noncurrentExpirations := noncurrentExpiration.([]interface{}) + if len(noncurrentExpirations) == 0 { + return lifecycle.NoncurrentVersionExpiration{} + } + + t := noncurrentExpirations[0].(map[string]interface{}) + if t == nil { + return lifecycle.NoncurrentVersionExpiration{} + } + + days, ok := t["days"].(string) + if !ok || days == "" { + return lifecycle.NoncurrentVersionExpiration{} + } + + var daysInt int + if _, err := fmt.Sscanf(days, "%dd", &daysInt); err == nil { + newerVersions, _ := t["newer_versions"].(int) // Optional field + return lifecycle.NoncurrentVersionExpiration{ + NoncurrentDays: lifecycle.ExpirationDays(daysInt), + NewerNoncurrentVersions: newerVersions, + } + } + + return lifecycle.NoncurrentVersionExpiration{} +} + +func getStringValue(m map[string]interface{}, key string) (string, bool) { + if v, ok := m[key]; ok { + if s, ok := v.(string); ok { + return s, true + } + } + return "", false +} + +// Use this helper for tags conversion +func convertToStringMap(v interface{}) map[string]string { + result := make(map[string]string) + if m, ok := v.(map[string]interface{}); ok { + for k, v := range m { + if s, ok := v.(string); ok { + result[k] = s + } + } + } + return result +} + +func isNotFoundError(err error) bool { + return strings.Contains(err.Error(), "The lifecycle configuration does not exist") +} diff --git a/minio/resource_minio_ilm_policy_test.go b/minio/resource_minio_ilm_policy_test.go index db50671e..b7882de0 100644 --- a/minio/resource_minio_ilm_policy_test.go +++ b/minio/resource_minio_ilm_policy_test.go @@ -109,7 +109,7 @@ func TestAccILMPolicy_expireNoncurrentVersion(t *testing.T) { resource.TestCheckResourceAttr( resourceName, "rule.0.expiration", ""), resource.TestCheckResourceAttr( - resourceName, "rule.0.noncurrent_version_expiration_days", "5"), + resourceName, "rule.0.noncurrent_expiration.0.days", "5d"), ), }, }, @@ -301,7 +301,9 @@ resource "minio_ilm_policy" "rule4" { bucket = "${minio_s3_bucket.bucket4.id}" rule { id = "expireNoncurrentVersion" - noncurrent_version_expiration_days = 5 + noncurrent_expiration { + days = "5d" + } } } `, randInt) @@ -329,8 +331,8 @@ resource "minio_ilm_policy" "rule_transition" { rule { id = "asdf" transition { - days = "1d" - storage_class = "${minio_ilm_tier.remote_tier.name}" + days = "1d" + storage_class = "${minio_ilm_tier.remote_tier.name}" } } } @@ -344,8 +346,8 @@ resource "minio_ilm_policy" "rule_transition" { rule { id = "asdf" transition { - date = "2024-06-06" - storage_class = "${minio_ilm_tier.remote_tier.name}" + date = "2024-06-06" + storage_class = "${minio_ilm_tier.remote_tier.name}" } } }