From 92f9dd78613924d1a4dad38949a3b89bf30daeb2 Mon Sep 17 00:00:00 2001 From: pjsier Date: Sun, 4 Dec 2022 19:56:32 -0600 Subject: [PATCH] Add minio_s3_bucket_notification resource --- docker-compose.yml | 2 + docs/resources/s3_bucket_notification.md | 65 +++++ examples/bucket/bucket.tf | 17 ++ minio/check_config.go | 12 + minio/payload.go | 8 + minio/provider.go | 1 + .../resource_minio_s3_bucket_notification.go | 222 ++++++++++++++++++ ...ource_minio_s3_bucket_notification_test.go | 130 ++++++++++ 8 files changed, 457 insertions(+) create mode 100644 docs/resources/s3_bucket_notification.md create mode 100644 minio/resource_minio_s3_bucket_notification.go create mode 100644 minio/resource_minio_s3_bucket_notification_test.go diff --git a/docker-compose.yml b/docker-compose.yml index 09d6418a..0e43d536 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,8 @@ services: MINIO_ROOT_USER: minio MINIO_ROOT_PASSWORD: minio123 MINIO_CI_CD: "1" + MINIO_NOTIFY_WEBHOOK_ENABLE_primary: "on" + MINIO_NOTIFY_WEBHOOK_ENDPOINT_primary: https://webhook.example.com command: server --console-address :9001 /data{0...3} adminio-ui: image: docker.io/rzrbld/adminio-ui:v1.93 diff --git a/docs/resources/s3_bucket_notification.md b/docs/resources/s3_bucket_notification.md new file mode 100644 index 00000000..8ec22814 --- /dev/null +++ b/docs/resources/s3_bucket_notification.md @@ -0,0 +1,65 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "minio_s3_bucket_notification Resource - terraform-provider-minio" +subcategory: "" +description: |- + +--- + +# minio_s3_bucket_notification (Resource) + +## Example Usage + +```terraform +resource "minio_s3_bucket" "bucket" { + bucket = "example-bucket" +} + +resource "minio_s3_bucket_notification" "bucket" { + bucket = minio_s3_bucket.state_terraform_s3.bucket + + queue { + id = "notification-queue" + queue_arn = "arn:minio:sqs::primary:webhook" + + events = [ + "s3:ObjectCreated:*", + "s3:ObjectRemoved:Delete", + ] + + filter_prefix = "example/" + filter_suffix = ".png" + } +} +``` + + +## Schema + +### Required + +- `bucket` (String) + +### Optional + +- `queue` (Block List) (see [below for nested schema](#nested-schema-for-queue)) + +### Read-Only + +- `id` (String) The ID of this resource. + +### Nested Schema for `queue` + +Required: + +- `events` (Set of String) +- `queue_arn` (String) + +Optional: + +- `filter_prefix` (String) +- `filter_suffix` (String) + +Read-Only: + +- `id` (String) The ID of this resource. diff --git a/examples/bucket/bucket.tf b/examples/bucket/bucket.tf index 6b4a8577..cd40e301 100644 --- a/examples/bucket/bucket.tf +++ b/examples/bucket/bucket.tf @@ -43,3 +43,20 @@ resource "minio_s3_bucket_versioning" "bucket" { status = "Enabled" } } + +resource "minio_s3_bucket_notification" "bucket" { + bucket = minio_s3_bucket.state_terraform_s3.bucket + + queue { + id = "notification-queue" + queue_arn = "arn:minio:sqs::primary:webhook" + + events = [ + "s3:ObjectCreated:*", + "s3:ObjectRemoved:Delete", + ] + + filter_prefix = "example/" + filter_suffix = ".png" + } +} diff --git a/minio/check_config.go b/minio/check_config.go index a93d74b2..0eeb5bc9 100644 --- a/minio/check_config.go +++ b/minio/check_config.go @@ -44,6 +44,18 @@ func BucketVersioningConfig(d *schema.ResourceData, meta interface{}) *S3MinioBu } } +// BucketNotificationConfig creates config for managing minio bucket notifications +func BucketNotificationConfig(d *schema.ResourceData, meta interface{}) *S3MinioBucketNotification { + m := meta.(*S3MinioClient) + config := getNotificationConfiguration(d) + + return &S3MinioBucketNotification{ + MinioClient: m.S3Client, + MinioBucket: d.Get("bucket").(string), + Configuration: &config, + } +} + // NewConfig creates a new config for minio func NewConfig(d *schema.ResourceData) *S3MinioConfig { user := d.Get("minio_user").(string) diff --git a/minio/payload.go b/minio/payload.go index fe82b00f..36d8cde3 100644 --- a/minio/payload.go +++ b/minio/payload.go @@ -3,6 +3,7 @@ package minio import ( "github.com/minio/madmin-go" minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/notification" "github.com/minio/minio-go/v7/pkg/policy" "github.com/minio/minio-go/v7/pkg/set" ) @@ -63,6 +64,13 @@ type S3MinioBucketVersioning struct { VersioningConfiguration *S3MinioBucketVersioningConfiguration } +// S3MinioBucketNotification +type S3MinioBucketNotification struct { + MinioClient *minio.Client + MinioBucket string + Configuration *notification.Configuration +} + // S3MinioServiceAccountConfig defines service account config type S3MinioServiceAccountConfig struct { MinioAdmin *madmin.AdminClient diff --git a/minio/provider.go b/minio/provider.go index f4b46939..0319498b 100644 --- a/minio/provider.go +++ b/minio/provider.go @@ -122,6 +122,7 @@ func Provider() *schema.Provider { "minio_s3_bucket": resourceMinioBucket(), "minio_s3_bucket_policy": resourceMinioBucketPolicy(), "minio_s3_bucket_versioning": resourceMinioBucketVersioning(), + "minio_s3_bucket_notification": resourceMinioBucketNotification(), "minio_s3_object": resourceMinioObject(), "minio_iam_group": resourceMinioIAMGroup(), "minio_iam_group_membership": resourceMinioIAMGroupMembership(), diff --git a/minio/resource_minio_s3_bucket_notification.go b/minio/resource_minio_s3_bucket_notification.go new file mode 100644 index 00000000..444fb00b --- /dev/null +++ b/minio/resource_minio_s3_bucket_notification.go @@ -0,0 +1,222 @@ +package minio + +import ( + "context" + "fmt" + "log" + + "github.com/hashicorp/go-cty/cty" + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/minio/minio-go/v7/pkg/notification" +) + +func resourceMinioBucketNotification() *schema.Resource { + return &schema.Resource{ + CreateContext: minioPutBucketNotification, + ReadContext: minioReadBucketNotification, + UpdateContext: minioPutBucketNotification, + DeleteContext: minioDeleteBucketNotification, + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + Schema: map[string]*schema.Schema{ + "bucket": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "queue": { + Type: schema.TypeList, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "id": { + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + "filter_prefix": { + Type: schema.TypeString, + Optional: true, + }, + "filter_suffix": { + Type: schema.TypeString, + Optional: true, + }, + "queue_arn": { + Type: schema.TypeString, + Required: true, + ValidateDiagFunc: validateMinioArn, + }, + "events": { + Type: schema.TypeSet, + Required: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Set: schema.HashString, + }, + }, + }, + }, + }, + } +} + +func minioPutBucketNotification(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + bucketNotificationConfig := BucketNotificationConfig(d, meta) + + log.Printf("[DEBUG] S3 bucket: %s, put notification configuration: %v", bucketNotificationConfig.MinioBucket, bucketNotificationConfig.Configuration) + + err := bucketNotificationConfig.MinioClient.SetBucketNotification( + ctx, + bucketNotificationConfig.MinioBucket, + *bucketNotificationConfig.Configuration, + ) + + if err != nil { + return NewResourceError("error putting bucket notification configuration: %v", d.Id(), err) + } + + d.SetId(bucketNotificationConfig.MinioBucket) + + return nil +} + +func minioReadBucketNotification(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + bucketNotificationConfig := BucketNotificationConfig(d, meta) + + log.Printf("[DEBUG] S3 bucket notification configuration, read for bucket: %s", d.Id()) + + notificationConfig, err := bucketNotificationConfig.MinioClient.GetBucketNotification(ctx, d.Id()) + if err != nil { + return NewResourceError("failed to load bucket notification configuration", d.Id(), err) + } + + _ = d.Set("bucket", d.Id()) + + if err := d.Set("queue", flattenQueueNotificationConfiguration(notificationConfig.QueueConfigs)); err != nil { + return NewResourceError("failed to load bucket queue notifications", d.Id(), err) + } + + return nil +} + +func minioDeleteBucketNotification(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + bucketNotificationConfig := BucketNotificationConfig(d, meta) + + log.Printf("[DEBUG] S3 bucket: %s, removing notification configuration", bucketNotificationConfig.MinioBucket) + + err := bucketNotificationConfig.MinioClient.SetBucketNotification( + ctx, + bucketNotificationConfig.MinioBucket, + notification.Configuration{}, + ) + + if err != nil { + return NewResourceError("error removing bucket notifications: %s", bucketNotificationConfig.MinioBucket, err) + } + + return nil +} + +func flattenNotificationConfigurationFilter(filter *notification.Filter) map[string]interface{} { + filterRules := map[string]interface{}{} + if filter.S3Key.FilterRules == nil { + return filterRules + } + + for _, f := range filter.S3Key.FilterRules { + if f.Name == "prefix" { + filterRules["filter_prefix"] = f.Value + } + if f.Name == "suffix" { + filterRules["filter_suffix"] = f.Value + } + } + return filterRules +} + +func flattenQueueNotificationConfiguration(configs []notification.QueueConfig) []map[string]interface{} { + queueNotifications := make([]map[string]interface{}, 0, len(configs)) + for _, notification := range configs { + var conf map[string]interface{} + if filter := notification.Filter; filter != nil { + conf = flattenNotificationConfigurationFilter(filter) + } else { + conf = map[string]interface{}{} + } + + conf["id"] = notification.ID + conf["events"] = notification.Events + // The Config.Arn value is not set to the queue ARN even though it's + // expected in the submission, so we're getting the correct value + // from the Queue attribute on the response object + conf["queue_arn"] = notification.Queue + queueNotifications = append(queueNotifications, conf) + } + + return queueNotifications +} + +func getNotificationConfiguration(d *schema.ResourceData) notification.Configuration { + var config notification.Configuration + queueConfigs := getNotificationQueueConfigs(d) + + for _, c := range queueConfigs { + config.AddQueue(c) + } + + return config +} + +func getNotificationQueueConfigs(d *schema.ResourceData) []notification.Config { + queueFunctionNotifications := d.Get("queue").([]interface{}) + configs := make([]notification.Config, 0, len(queueFunctionNotifications)) + + for i, c := range queueFunctionNotifications { + config := notification.Config{Filter: ¬ification.Filter{}} + c := c.(map[string]interface{}) + + if queueArnStr, ok := c["queue_arn"].(string); ok { + queueArn, err := notification.NewArnFromString(queueArnStr) + if err != nil { + continue + } + config.Arn = queueArn + } + + if val, ok := c["id"].(string); ok && val != "" { + config.ID = val + } else { + config.ID = resource.PrefixedUniqueId("tf-s3-queue-") + } + + events := d.Get(fmt.Sprintf("queue.%d.events", i)).(*schema.Set).List() + for _, e := range events { + config.AddEvents(notification.EventType(e.(string))) + } + + if val, ok := c["filter_prefix"].(string); ok && val != "" { + config.AddFilterPrefix(val) + } + if val, ok := c["filter_suffix"].(string); ok && val != "" { + config.AddFilterSuffix(val) + } + + configs = append(configs, config) + } + + return configs +} + +func validateMinioArn(v interface{}, p cty.Path) (errors diag.Diagnostics) { + value := v.(string) + _, err := notification.NewArnFromString(value) + + if err != nil { + return diag.Errorf("value: %s is not a valid ARN", value) + } + + return nil +} diff --git a/minio/resource_minio_s3_bucket_notification_test.go b/minio/resource_minio_s3_bucket_notification_test.go new file mode 100644 index 00000000..735ce1d7 --- /dev/null +++ b/minio/resource_minio_s3_bucket_notification_test.go @@ -0,0 +1,130 @@ +package minio + +import ( + "context" + "fmt" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + "github.com/minio/minio-go/v7/pkg/notification" +) + +func TestS3BucketNotification_queue(t *testing.T) { + name := acctest.RandomWithPrefix("tf-notification-test") + + config := notification.Configuration{} + arn, _ := notification.NewArnFromString("arn:minio:sqs::primary:webhook") + qc := notification.NewConfig(arn) + qc.ID = "notification-queue" + qc.AddEvents(notification.ObjectCreatedAll, notification.ObjectRemovedDelete) + qc.AddFilterPrefix("tf-acc-test/") + qc.AddFilterSuffix(".png") + config.AddQueue(qc) + + updateConfig := notification.Configuration{} + updateQc := notification.NewConfig(arn) + updateQc.ID = "notification-queue" + updateQc.AddEvents(notification.ObjectCreatedAll, notification.ObjectRemovedDelete) + updateQc.AddFilterPrefix("tf-acc-test/") + updateQc.AddFilterSuffix(".mp4") + updateConfig.AddQueue(updateQc) + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviders, + CheckDestroy: testAccCheckMinioS3BucketDestroy, + Steps: []resource.TestStep{ + { + Config: testAccBucketNotificationConfig_queue(name, ".png"), + Check: resource.ComposeTestCheckFunc( + testAccCheckBucketHasNotification( + "minio_s3_bucket_notification.notification", + config, + ), + ), + }, + { + Config: testAccBucketNotificationConfig_queue(name, ".mp4"), + Check: resource.ComposeTestCheckFunc( + testAccCheckBucketHasNotification( + "minio_s3_bucket_notification.notification", + updateConfig, + ), + ), + }, + { + ResourceName: "minio_s3_bucket_notification.notification", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func testAccBucketNotificationConfig_queue(name string, suffix string) string { + return fmt.Sprintf(` +resource "minio_s3_bucket" "bucket" { + bucket = %[1]q +} + +resource "minio_s3_bucket_notification" "notification" { + bucket = minio_s3_bucket.bucket.id + + queue { + id = "notification-queue" + queue_arn = "arn:minio:sqs::primary:webhook" + + events = [ + "s3:ObjectCreated:*", + "s3:ObjectRemoved:Delete", + ] + + filter_prefix = "tf-acc-test/" + filter_suffix = %[2]q + } +} +`, name, suffix) +} + +func testAccCheckBucketHasNotification(n string, config notification.Configuration) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("not found: %s", n) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("no ID is set") + } + + minioC := testAccProvider.Meta().(*S3MinioClient).S3Client + actualConfig, err := minioC.GetBucketNotification(context.Background(), rs.Primary.ID) + + if err != nil { + return fmt.Errorf("error on GetBucketNotification: %v", err) + } + + if len(actualConfig.QueueConfigs) != len(config.QueueConfigs) { + return fmt.Errorf("non-equivalent queue configuration error:\n\nexpected len: %v\n\ngot: %v", len(actualConfig.QueueConfigs), len(config.QueueConfigs)) + } + + for _, actualQueueConfig := range actualConfig.QueueConfigs { + for _, queueConfig := range config.QueueConfigs { + if actualQueueConfig.Queue != queueConfig.Config.Arn.String() { + return fmt.Errorf("non-equivalent queue configuration error:\n\nexpected %s\n\ngot: %s", actualQueueConfig.Queue, queueConfig.Config.Arn.String()) + } + if !notificationConfigsEqual(actualQueueConfig.Config, queueConfig.Config) { + return fmt.Errorf("non-equivalent queue configuration error:\n\nexpected: %v\n\ngot: %v", queueConfig.Config, actualQueueConfig.Config) + } + } + } + + return nil + } +} + +func notificationConfigsEqual(a notification.Config, b notification.Config) bool { + return a.ID == b.ID && notification.EqualEventTypeList(a.Events, b.Events) && notification.EqualFilterRuleList(a.Filter.S3Key.FilterRules, b.Filter.S3Key.FilterRules) +}