From f9562cd8cf6febe625862a607868cfec5d80b22d Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Thu, 31 Jan 2019 11:33:13 -0500 Subject: [PATCH] service/s3: Add sweepers To prevent TooManyBuckets errors from failed acceptance testing. Output from test sweepers: ``` $ go test ./aws -v -sweep=us-west-2 -sweep-run=aws_s3_bucket -timeout 10h 2019/01/31 11:28:57 [DEBUG] Running Sweepers for region (us-west-2): ... 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket: 187416307283-awsmacietrail-dataevent 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket: aws-athena-query-results-187416307283-us-east-1 ... 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket (tf-acc-test-6417188599110687269destination) Objects: BucketRegionError: incorrect region, the bucket is not in 'us-west-2' region status code: 301, request id: , host id: 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket: tf-alex-preview 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket: tf-alex-preview2 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket: tf-deploy-alex 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket: tf-emr-bootstrap-5850549931442788494 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket: tf-emr-bootstrap-8212862313789127315 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket: tf-io-dev 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket: tf-k8s-acc-test 2019/01/31 11:28:59 [INFO] Skipping S3 Bucket: tf-macie-test-bucket-921738264640544542 2019/01/31 11:29:00 [INFO] Deleting S3 Bucket (tf-object-test-bucket-272994333124655711) Object (tf-testing-obj-272994333124655711-all-params) Delete Marker: YECgZ632K8z.xlufooF.aeFvafBA6kuz 2019/01/31 11:29:02 [INFO] Deleting S3 Bucket (tf-object-test-bucket-3609874408477041358) Object (tf-testing-obj-3609874408477041358-all-params) Delete Marker: hQ4eUTYHGqhZ6lV9ATtCbu2F3HNuqgwa 2019/01/31 11:29:03 [INFO] Deleting S3 Bucket (tf-object-test-bucket-3660475575348874992) Object (updateable-key) Delete Marker: JxNsmwLQGbCqmtpMgt4n.d60NeCtC3V_ 2019/01/31 11:29:04 [INFO] Deleting S3 Bucket (tf-object-test-bucket-3900350678782068183) Object (test-key) Delete Marker: auhlvd6dgHaEc4jv2sfi1yUKM_lkvh.. 2019/01/31 11:29:05 [INFO] Deleting S3 Bucket (tf-object-test-bucket-4138169415932139850) Object (updateable-key) Delete Marker: nA74rE0HYETVIrXvq2zSu7Z4AQGc7jR9 2019/01/31 11:29:07 [INFO] Deleting S3 Bucket (tf-object-test-bucket-6391031696293038021) Object (updateable-key) Delete Marker: Cc0CRHZZ6P5Z3RIGSolfZk1KfNqs4n1l 2019/01/31 11:29:08 [INFO] Skipping S3 Bucket: tf-redshift-logging-5085425770880693968 2019/01/31 11:29:08 [INFO] Skipping S3 Bucket: tf-redshift-logging-9209707503695989768 2019/01/31 11:29:08 [INFO] Skipping S3 Bucket: tf-spot-datafeed-3152131621798213916 2019/01/31 11:29:08 [INFO] Skipping S3 Bucket (tf-test-bucket-1173093638223895558) Objects: BucketRegionError: incorrect region, the bucket is not in 'us-west-2' region status code: 301, request id: , host id: 2019/01/31 11:29:09 [INFO] Skipping S3 Bucket (tf-test-bucket-6879147095700184482) Objects: BucketRegionError: incorrect region, the bucket is not in 'us-west-2' region status code: 301, request id: , host id: 2019/01/31 11:29:09 [DEBUG] Sweeper (aws_s3_bucket) has dependency (aws_s3_bucket_object), running.. 2019/01/31 11:29:09 [DEBUG] Sweeper (aws_s3_bucket_object) already ran in region (us-west-2) ... 2019/01/31 11:29:09 [INFO] Skipping S3 Bucket: 187416307283-awsmacietrail-dataevent 2019/01/31 11:29:09 [INFO] Skipping S3 Bucket: aws-athena-query-results-187416307283-us-east-1 ... 2019/01/31 11:29:09 [INFO] Deleting S3 Bucket: tf-acc-test-6417188599110687269destination 2019/01/31 11:29:09 [DEBUG] Waiting for state to become: [success] 2019/01/31 11:29:10 [INFO] Skipping S3 Bucket (tf-acc-test-6417188599110687269destination): BucketRegionError: incorrect region, the bucket is not in 'us-west-2' region status code: 301, request id: , host id: 2019/01/31 11:29:10 [INFO] Skipping S3 Bucket: tf-alex-preview 2019/01/31 11:29:10 [INFO] Skipping S3 Bucket: tf-alex-preview2 2019/01/31 11:29:10 [INFO] Skipping S3 Bucket: tf-deploy-alex 2019/01/31 11:29:10 [INFO] Skipping S3 Bucket: tf-emr-bootstrap-5850549931442788494 2019/01/31 11:29:10 [INFO] Skipping S3 Bucket: tf-emr-bootstrap-8212862313789127315 2019/01/31 11:29:10 [INFO] Skipping S3 Bucket: tf-io-dev 2019/01/31 11:29:10 [INFO] Skipping S3 Bucket: tf-k8s-acc-test 2019/01/31 11:29:10 [INFO] Skipping S3 Bucket: tf-macie-test-bucket-921738264640544542 2019/01/31 11:29:10 [INFO] Deleting S3 Bucket: tf-object-test-bucket-272994333124655711 2019/01/31 11:29:10 [DEBUG] Waiting for state to become: [success] 2019/01/31 11:29:10 [INFO] Deleting S3 Bucket: tf-object-test-bucket-3609874408477041358 2019/01/31 11:29:10 [DEBUG] Waiting for state to become: [success] 2019/01/31 11:29:11 [INFO] Deleting S3 Bucket: tf-object-test-bucket-3660475575348874992 2019/01/31 11:29:11 [DEBUG] Waiting for state to become: [success] 2019/01/31 11:29:11 [INFO] Deleting S3 Bucket: tf-object-test-bucket-3900350678782068183 2019/01/31 11:29:11 [DEBUG] Waiting for state to become: [success] 2019/01/31 11:29:12 [INFO] Deleting S3 Bucket: tf-object-test-bucket-4138169415932139850 2019/01/31 11:29:12 [DEBUG] Waiting for state to become: [success] 2019/01/31 11:29:13 [INFO] Deleting S3 Bucket: tf-object-test-bucket-6391031696293038021 2019/01/31 11:29:13 [DEBUG] Waiting for state to become: [success] 2019/01/31 11:29:13 [INFO] Deleting S3 Bucket: tf-object-test-bucket-7215878632290092620 2019/01/31 11:29:13 [DEBUG] Waiting for state to become: [success] 2019/01/31 11:29:14 [INFO] Skipping S3 Bucket: tf-redshift-logging-5085425770880693968 2019/01/31 11:29:14 [INFO] Skipping S3 Bucket: tf-redshift-logging-9209707503695989768 2019/01/31 11:29:14 [INFO] Skipping S3 Bucket: tf-spot-datafeed-3152131621798213916 2019/01/31 11:29:14 [INFO] Deleting S3 Bucket: tf-test-bucket-1173093638223895558 2019/01/31 11:29:14 [DEBUG] Waiting for state to become: [success] 2019/01/31 11:29:14 [INFO] Skipping S3 Bucket (tf-test-bucket-1173093638223895558): BucketRegionError: incorrect region, the bucket is not in 'us-west-2' region status code: 301, request id: , host id: 2019/01/31 11:29:14 [INFO] Deleting S3 Bucket: tf-test-bucket-6879147095700184482 2019/01/31 11:29:14 [DEBUG] Waiting for state to become: [success] 2019/01/31 11:29:14 [INFO] Skipping S3 Bucket (tf-test-bucket-6879147095700184482): BucketRegionError: incorrect region, the bucket is not in 'us-west-2' region status code: 301, request id: , host id: 2019/01/31 11:29:14 Sweeper Tests ran: - aws_s3_bucket_object - aws_s3_bucket ok github.com/terraform-providers/terraform-provider-aws/aws 19.735s ``` --- aws/resource_aws_s3_bucket_object_test.go | 134 +++++++++++++++++++++- aws/resource_aws_s3_bucket_test.go | 91 ++++++++++++++- 2 files changed, 216 insertions(+), 9 deletions(-) diff --git a/aws/resource_aws_s3_bucket_object_test.go b/aws/resource_aws_s3_bucket_object_test.go index d21f7cc9873..050bc16243c 100644 --- a/aws/resource_aws_s3_bucket_object_test.go +++ b/aws/resource_aws_s3_bucket_object_test.go @@ -4,19 +4,147 @@ import ( "encoding/base64" "fmt" "io/ioutil" + "log" "os" "reflect" "sort" + "strings" "testing" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" "github.com/hashicorp/terraform/helper/acctest" "github.com/hashicorp/terraform/helper/resource" "github.com/hashicorp/terraform/terraform" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" ) +func init() { + resource.AddTestSweepers("aws_s3_bucket_object", &resource.Sweeper{ + Name: "aws_s3_bucket_object", + F: testSweepS3BucketObjects, + }) +} + +func testSweepS3BucketObjects(region string) error { + client, err := sharedClientForRegion(region) + if err != nil { + return fmt.Errorf("error getting client: %s", err) + } + + conn := client.(*AWSClient).s3conn + input := &s3.ListBucketsInput{} + + output, err := conn.ListBuckets(input) + + if testSweepSkipSweepError(err) { + log.Printf("[WARN] Skipping S3 Bucket Objects sweep for %s: %s", region, err) + return nil + } + + if err != nil { + return fmt.Errorf("error listing S3 Bucket Objects: %s", err) + } + + if len(output.Buckets) == 0 { + log.Print("[DEBUG] No S3 Bucket Objects to sweep") + return nil + } + + for _, bucket := range output.Buckets { + bucketName := aws.StringValue(bucket.Name) + + if !strings.HasPrefix(bucketName, "tf-acc") && !strings.HasPrefix(bucketName, "tf-object-test") && !strings.HasPrefix(bucketName, "tf-test") { + log.Printf("[INFO] Skipping S3 Bucket: %s", bucketName) + continue + } + + input := &s3.ListObjectVersionsInput{ + Bucket: bucket.Name, + } + + err := conn.ListObjectVersionsPages(input, func(page *s3.ListObjectVersionsOutput, lastPage bool) bool { + if page == nil { + return !lastPage + } + + for _, objectVersion := range page.Versions { + input := &s3.DeleteObjectInput{ + Bucket: bucket.Name, + Key: objectVersion.Key, + VersionId: objectVersion.VersionId, + } + objectKey := aws.StringValue(objectVersion.Key) + objectVersionID := aws.StringValue(objectVersion.VersionId) + + log.Printf("[INFO] Deleting S3 Bucket (%s) Object (%s) Version: %s", bucketName, objectKey, objectVersionID) + _, err := conn.DeleteObject(input) + + if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") || isAWSErr(err, s3.ErrCodeNoSuchKey, "") { + continue + } + + if err != nil { + log.Printf("[ERROR] Error deleting S3 Bucket (%s) Object (%s) Version (%s): %s", bucketName, objectKey, objectVersionID, err) + } + } + + return !lastPage + }) + + if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") { + continue + } + + if isAWSErr(err, "BucketRegionError", "") { + log.Printf("[INFO] Skipping S3 Bucket (%s) Objects: %s", bucketName, err) + continue + } + + if err != nil { + return fmt.Errorf("error listing S3 Bucket (%s) Objects: %s", bucketName, err) + } + + err = conn.ListObjectVersionsPages(input, func(page *s3.ListObjectVersionsOutput, lastPage bool) bool { + if page == nil { + return !lastPage + } + + for _, deleteMarker := range page.DeleteMarkers { + input := &s3.DeleteObjectInput{ + Bucket: bucket.Name, + Key: deleteMarker.Key, + VersionId: deleteMarker.VersionId, + } + deleteMarkerKey := aws.StringValue(deleteMarker.Key) + deleteMarkerVersionID := aws.StringValue(deleteMarker.VersionId) + + log.Printf("[INFO] Deleting S3 Bucket (%s) Object (%s) Delete Marker: %s", bucketName, deleteMarkerKey, deleteMarkerVersionID) + _, err := conn.DeleteObject(input) + + if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") || isAWSErr(err, s3.ErrCodeNoSuchKey, "") { + continue + } + + if err != nil { + log.Printf("[ERROR] Error deleting S3 Bucket (%s) Object (%s) Version (%s): %s", bucketName, deleteMarkerKey, deleteMarkerVersionID, err) + } + } + + return !lastPage + }) + + if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") { + continue + } + + if err != nil { + return fmt.Errorf("error listing S3 Bucket (%s) Objects: %s", bucketName, err) + } + } + + return nil +} + func TestAccAWSS3BucketObject_source(t *testing.T) { var obj s3.GetObjectOutput resourceName := "aws_s3_bucket_object.object" diff --git a/aws/resource_aws_s3_bucket_test.go b/aws/resource_aws_s3_bucket_test.go index 70683d6c52e..67a92afda30 100644 --- a/aws/resource_aws_s3_bucket_test.go +++ b/aws/resource_aws_s3_bucket_test.go @@ -4,25 +4,104 @@ import ( "bytes" "encoding/json" "fmt" + "log" "reflect" "regexp" "sort" "strconv" + "strings" "testing" "text/template" - - "github.com/hashicorp/terraform/helper/acctest" - "github.com/hashicorp/terraform/helper/resource" - "github.com/hashicorp/terraform/terraform" - - "strings" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/s3" + "github.com/hashicorp/terraform/helper/acctest" + "github.com/hashicorp/terraform/helper/resource" "github.com/hashicorp/terraform/helper/schema" + "github.com/hashicorp/terraform/terraform" ) +func init() { + resource.AddTestSweepers("aws_s3_bucket", &resource.Sweeper{ + Name: "aws_s3_bucket", + F: testSweepS3Buckets, + Dependencies: []string{ + "aws_s3_bucket_object", + }, + }) +} + +func testSweepS3Buckets(region string) error { + client, err := sharedClientForRegion(region) + if err != nil { + return fmt.Errorf("error getting client: %s", err) + } + + conn := client.(*AWSClient).s3conn + input := &s3.ListBucketsInput{} + + output, err := conn.ListBuckets(input) + + if testSweepSkipSweepError(err) { + log.Printf("[WARN] Skipping S3 Buckets sweep for %s: %s", region, err) + return nil + } + + if err != nil { + return fmt.Errorf("error listing S3 Buckets: %s", err) + } + + if len(output.Buckets) == 0 { + log.Print("[DEBUG] No S3 Buckets to sweep") + return nil + } + + for _, bucket := range output.Buckets { + name := aws.StringValue(bucket.Name) + + if !strings.HasPrefix(name, "tf-acc") && !strings.HasPrefix(name, "tf-object-test") && !strings.HasPrefix(name, "tf-test") { + log.Printf("[INFO] Skipping S3 Bucket: %s", name) + continue + } + + input := &s3.DeleteBucketInput{ + Bucket: bucket.Name, + } + + log.Printf("[INFO] Deleting S3 Bucket: %s", name) + err := resource.Retry(1*time.Minute, func() *resource.RetryError { + _, err := conn.DeleteBucket(input) + + if isAWSErr(err, s3.ErrCodeNoSuchBucket, "") { + return nil + } + + if isAWSErr(err, "BucketRegionError", "") { + log.Printf("[INFO] Skipping S3 Bucket (%s): %s", name, err) + return nil + } + + if isAWSErr(err, "BucketNotEmpty", "") { + return resource.RetryableError(err) + } + + if err != nil { + return resource.NonRetryableError(err) + } + + return nil + }) + + if err != nil { + return fmt.Errorf("error deleting S3 Bucket (%s): %s", name, err) + } + } + + return nil +} + func TestAccAWSS3Bucket_importBasic(t *testing.T) { resourceName := "aws_s3_bucket.bucket" rInt := acctest.RandInt()