From d73a28641946303d190e3fc5ffc099bd0e82e252 Mon Sep 17 00:00:00 2001 From: Arnaud Meukam Date: Sun, 29 Dec 2024 00:14:38 +0100 Subject: [PATCH] Use ephemeral S3 buckets for E2E tests Use S3 buckets created during the lifecycle of a test instead of a static one and provide the capability to make them read-only public. Signed-off-by: Arnaud Meukam --- tests/e2e/kubetest2-kops/aws/s3.go | 172 ++++++++++++++++++ tests/e2e/kubetest2-kops/deployer/common.go | 17 +- tests/e2e/kubetest2-kops/deployer/deployer.go | 7 +- 3 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/kubetest2-kops/aws/s3.go diff --git a/tests/e2e/kubetest2-kops/aws/s3.go b/tests/e2e/kubetest2-kops/aws/s3.go new file mode 100644 index 0000000000000..dd1d9fd76a100 --- /dev/null +++ b/tests/e2e/kubetest2-kops/aws/s3.go @@ -0,0 +1,172 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + "k8s.io/klog/v2" +) + +// We need to pick some region to query the AWS APIs through, even if we are not running on AWS. +const defaultRegion = "us-east-2" + +// It contains S3Client, an Amazon S3 service client that is used to perform bucket +// and object actions. +type awsClient struct { + S3Client *s3.Client +} + +func NewAWSClient(ctx context.Context) (*awsClient, error) { + cfg, err := awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithRegion(defaultRegion)) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + return &awsClient{ + S3Client: s3.NewFromConfig(cfg), + }, nil +} + +// AWSBucketName constructs an unique bucket name using the AWS account ID on region us-east-2 +func AWSBucketName(ctx context.Context) (string, error) { + config, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(string(types.BucketLocationConstraintUsEast2))) + if err != nil { + return "", fmt.Errorf("failed to load AWS config: %w", err) + } + + stsSvc := sts.NewFromConfig(config) + callerIdentity, err := stsSvc.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + return "", fmt.Errorf("building AWS STS presigned request: %w", err) + } + + // Add timestamp suffix + timestamp := time.Now().Format("01022006") + bucket := fmt.Sprintf("k8s-infra-kops-%s", *callerIdentity.Account) + bucket = fmt.Sprintf("%s-%s", bucket, timestamp) + + bucket = strings.ToLower(bucket) + bucket = regexp.MustCompile("[^a-z0-9-]").ReplaceAllString(bucket, "") // Only allow lowercase letters, numbers, and hyphens + + if len(bucket) > 63 { + bucket = bucket[:63] // Max length is 63 + } + + return bucket, nil +} + +func (client awsClient) EnsureS3Bucket(ctx context.Context, bucketName string, publicRead bool) error { + _, err := client.S3Client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + CreateBucketConfiguration: &types.CreateBucketConfiguration{ + LocationConstraint: types.BucketLocationConstraintUsEast2, + }, + }) + + var exists *types.BucketAlreadyExists + if err != nil { + if errors.As(err, &exists) { + klog.Infof("Bucket %s already exists.\n", bucketName) + err = exists + } + } else { + err := s3.NewBucketExistsWaiter(client.S3Client).Wait( + ctx, &s3.HeadBucketInput{ + Bucket: aws.String(bucketName), + }, + time.Minute) + if err != nil { + klog.Infof("Failed attempt to wait for bucket %s to exist.", bucketName) + } + } + + klog.Infof("Bucket %s created successfully", bucketName) + + if err == nil && publicRead { + err = client.setPublicReadPolicy(ctx, bucketName) + if err != nil { + klog.Errorf("Failed to set public read policy on bucket %s: %v", bucketName, err) + return err + } + klog.Infof("Public read policy set on bucket %s", bucketName) + } + + return err +} + +func (client awsClient) DeleteS3Bucket(ctx context.Context, bucketName string) error { + _, err := client.S3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + if err != nil { + var noBucket *types.NoSuchBucket + if errors.As(err, &noBucket) { + klog.Infof("Bucket %s does not exits", bucketName) + err = noBucket + } else { + klog.Infof("Couldn't delete bucket %s. Reason: %v", bucketName, err) + } + } else { + err = s3.NewBucketNotExistsWaiter(client.S3Client).Wait( + ctx, &s3.HeadBucketInput{ + Bucket: aws.String(bucketName), + }, + time.Minute) + if err != nil { + klog.Infof("Failed attempt to wait for bucket %s to be deleted", bucketName) + } else { + klog.Infof("Bucket %s deleted", bucketName) + } + } + return err +} + +func (client awsClient) setPublicReadPolicy(ctx context.Context, bucketName string) error { + policy := fmt.Sprintf(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "PublicReadGetObject", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::%s/*" + } + ] + }`, bucketName) + + _, err := client.S3Client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: aws.String(bucketName), + Policy: aws.String(policy), + }) + if err != nil { + return fmt.Errorf("failed to put bucket policy for %s: %w", bucketName, err) + } + + return err +} diff --git a/tests/e2e/kubetest2-kops/deployer/common.go b/tests/e2e/kubetest2-kops/deployer/common.go index f8f8fb5b699b1..e202f8ccaf62e 100644 --- a/tests/e2e/kubetest2-kops/deployer/common.go +++ b/tests/e2e/kubetest2-kops/deployer/common.go @@ -17,6 +17,7 @@ limitations under the License. package deployer import ( + "context" "errors" "fmt" "os" @@ -24,6 +25,7 @@ import ( "strings" "k8s.io/klog/v2" + "k8s.io/kops/tests/e2e/kubetest2-kops/aws" "k8s.io/kops/tests/e2e/kubetest2-kops/gce" "k8s.io/kops/tests/e2e/pkg/target" "k8s.io/kops/tests/e2e/pkg/util" @@ -320,7 +322,20 @@ func (d *deployer) stateStore() string { if ss == "" { switch d.CloudProvider { case "aws": - ss = "s3://k8s-kops-prow" + ctx := context.TODO() + bucketName, err := aws.AWSBucketName(ctx) + if err != nil { + klog.Fatalf("Failed to generate bucket name: %v", err) + } + awsClient, err := aws.NewAWSClient(ctx) + if err != nil { + klog.Fatalf("failed to load client config: %v", err) + } + if err := awsClient.EnsureS3Bucket(ctx, bucketName, d.PublicReadOnlyBucket); err != nil { + klog.Fatalf("Failed to ensure S3 bucket exists: %v", err) + return "" + } + ss = "s3://" + bucketName case "gce": d.createBucket = true ss = "gs://" + gce.GCSBucketName(d.GCPProject, "state") diff --git a/tests/e2e/kubetest2-kops/deployer/deployer.go b/tests/e2e/kubetest2-kops/deployer/deployer.go index 15cd41bb210ad..8e9299015f33a 100644 --- a/tests/e2e/kubetest2-kops/deployer/deployer.go +++ b/tests/e2e/kubetest2-kops/deployer/deployer.go @@ -58,6 +58,7 @@ type deployer struct { KopsBinaryPath string `flag:"kops-binary-path" desc:"The path to kops executable used for testing"` KubernetesFeatureGates string `flag:"kubernetes-feature-gates" desc:"Feature Gates to enable on Kubernetes components"` createBucket bool `flag:"-"` + PublicReadOnlyBucket bool `flag:"-"` // ControlPlaneCount specifies the number of VMs in the control-plane. ControlPlaneCount int `flag:"control-plane-count" desc:"Number of control-plane instances"` @@ -106,8 +107,10 @@ type deployer struct { var _ types.NewDeployer = New // assert that deployer implements types.Deployer -var _ types.Deployer = &deployer{} -var _ types.DeployerWithPostTester = &deployer{} +var ( + _ types.Deployer = &deployer{} + _ types.DeployerWithPostTester = &deployer{} +) func (d *deployer) Provider() string { return Name