From 4288410bde2be37cfbaf0a68b7ae2201e66ce568 Mon Sep 17 00:00:00 2001 From: Dev Lakhia Date: Thu, 30 Nov 2023 20:55:04 +0000 Subject: [PATCH] Upgrading MP and adding tests for S3 Express --- .github/workflows/e2e-tests.yaml | 2 +- Dockerfile | 2 +- tests/e2e-kubernetes/go.mod | 21 +++++ tests/e2e-kubernetes/go.sum | 36 +++++++++ tests/e2e-kubernetes/scripts/kops-patch.yaml | 3 +- tests/e2e-kubernetes/scripts/run.sh | 6 ++ tests/e2e-kubernetes/testdriver.go | 80 ++++++++++++++++--- .../e2e-kubernetes/testsuites/mountoptions.go | 26 +++++- 8 files changed, 158 insertions(+), 18 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index a846e9c7..a9f8fb9a 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -61,7 +61,7 @@ jobs: needs: build strategy: matrix: - cluster-type: ["kops", "eksctl"] + cluster-type: ["eksctl", "kops"] arch: ["x86", "arm"] runs-on: ubuntu-latest permissions: diff --git a/Dockerfile b/Dockerfile index ef256f8a..bc1e83fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ #See the License for the specific language governing permissions and #limitations under the License. -ARG MOUNTPOINT_VERSION=1.1.1 +ARG MOUNTPOINT_VERSION=1.3.1 # Download the mountpoint tarball and produce an installable directory # Building on Amazon Linux 2 because it has an old libc version. libfuse from the os diff --git a/tests/e2e-kubernetes/go.mod b/tests/e2e-kubernetes/go.mod index d8ff0f40..3f2fe2d5 100644 --- a/tests/e2e-kubernetes/go.mod +++ b/tests/e2e-kubernetes/go.mod @@ -9,11 +9,32 @@ require ( k8s.io/kubernetes v1.28.3 ) +require ( + github.com/aws/aws-sdk-go-v2 v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.8 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.1 // indirect + github.com/aws/smithy-go v1.18.1 // indirect +) + require ( github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/aws/aws-sdk-go v1.46.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.25.10 + github.com/aws/aws-sdk-go-v2/service/s3 v1.47.1 github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect diff --git a/tests/e2e-kubernetes/go.sum b/tests/e2e-kubernetes/go.sum index 7718333a..2920c647 100644 --- a/tests/e2e-kubernetes/go.sum +++ b/tests/e2e-kubernetes/go.sum @@ -50,6 +50,42 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.46.1 h1:U26quvBWFZMQuultLw5tloW4GnmWaChEwMZNq8uYatw= github.com/aws/aws-sdk-go v1.46.1/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.23.4 h1:2P20ZjH0ouSAu/6yZep8oCmTReathLuEu6dwoqEgjts= +github.com/aws/aws-sdk-go-v2 v1.23.4/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 h1:Zx9+31KyB8wQna6SXFWOewlgoY5uGdDAu6PTOEU3OQI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3/go.mod h1:zxbEJhRdKTH1nqS2qu6UJ7zGe25xaHxZXaC2CvuQFnA= +github.com/aws/aws-sdk-go-v2/config v1.25.10 h1:qw/e8emDtNufTkrAU86DlQ18DruMyyM7ttW6Lgwp4v0= +github.com/aws/aws-sdk-go-v2/config v1.25.10/go.mod h1:203YiAtb6XyoGxXMPsUVwEcuxCiTQY/r8P27IDjfvMc= +github.com/aws/aws-sdk-go-v2/credentials v1.16.8 h1:phw9nRLy/77bPk6Mfu2SHCOnHwfVB7WWrOa5rZIY2Fc= +github.com/aws/aws-sdk-go-v2/credentials v1.16.8/go.mod h1:MrS4SOin6adbO6wgWhdifyPiq+TX7fPPwyA/ZLC1F5M= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.8 h1:tQZLSPC2Zj2CqZHonLmWEvCsbpMX5tQvaYJWHadcPek= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.8/go.mod h1:5+YpvTHDFffykWr5qAGjqwoh8oVYZOddL3sSrEN7lws= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.7 h1:eMqD7ku6WGdmcWWXPYun9m6yk6feSULLhJlAtN6rYG4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.7/go.mod h1:0oBIfcDV6LScxEW0VgOqxT3e4aqKRp+SYhB9wAd5E3Q= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.7 h1:+XYhWhgWs5F3Zx8oa49CXzNvfXrItaDjZB/M172fcHQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.7/go.mod h1:L6tcSRyCGxcKfDWUrmv2jv8G1cLDU7d0FUpEFpG9bVE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.7 h1:3VaUNB1LclLomv82VnP5QnxAfowG+Ro4m82+af9wjZ4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.7/go.mod h1:D5i0c+qvEY0LV5F4elFZd+mYnvHQbufCLHNHoBfQR2g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.7 h1:Mft1tmIK1fkFS9l9sYVYiN+OdgXeOcQ9ZS3SxKOh3A4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.7/go.mod h1:QWI83fhocxDaN3b74N8rrvET60CBaike5lQ+5sm3OcE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.7 h1:dU+ZyhvqMB/T/TxjGagHMCdyUiqaThRIaMu3YvKiSQI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.7/go.mod h1:SGORuNqoXyWfTvTp/gBGJfv8jRvW/+nha0XhnIXVI+o= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.7 h1:ybtGXm0qFVFi0hFUF7eFAVnL3ntl9MO7lrxhhGP7KYU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.7/go.mod h1:BUyWJUKAnNqoEq1LfyQxy+Eh4U8Y3c5w2C6m21f3yvI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.1 h1:0/W5F+LlXzKZ7KTsRcD8pugasVnsrjUWmhOsN/LdSFY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.1/go.mod h1:TqThLn4bRCn/UYf960hNZgPPjmxc17fQcwmjfuG6D5k= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.1 h1:V40g2daNO3l1J94JYwqfkyvQMYXi5I25fs3fNQW8iDs= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.1/go.mod h1:0ZWQJP/mBOUxkCvZKybZNz1XmdUKSBxoF0dzgfxtvDs= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.1 h1:uQrj7SpUNC3r55vc1CDh3qV9wJC66lz546xM9dhSo5s= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.1/go.mod h1:oyaTk5xEAOuPXX1kCD7HmIeuLqdj3Bk5yGkqGXtGi14= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.1 h1:K33V7L0XDdb23FMOZySr8bon1jou5SHn1fiv7NJ1SUg= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.1/go.mod h1:YtXUl/sfnS06VksYhr855hTQf2HphfT1Xv/EwuzbPjg= +github.com/aws/smithy-go v1.18.1 h1:pOdBTUfXNazOlxLrgeYalVnuTpKreACHtc62xLwIB3c= +github.com/aws/smithy-go v1.18.1/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/tests/e2e-kubernetes/scripts/kops-patch.yaml b/tests/e2e-kubernetes/scripts/kops-patch.yaml index 41f1d4cb..92ae1637 100644 --- a/tests/e2e-kubernetes/scripts/kops-patch.yaml +++ b/tests/e2e-kubernetes/scripts/kops-patch.yaml @@ -5,7 +5,8 @@ spec: { "Effect": "Allow", "Action": [ - "s3:*" + "s3:*", + "s3express:*" ], "Resource": "*" } diff --git a/tests/e2e-kubernetes/scripts/run.sh b/tests/e2e-kubernetes/scripts/run.sh index bda91e58..60827db2 100755 --- a/tests/e2e-kubernetes/scripts/run.sh +++ b/tests/e2e-kubernetes/scripts/run.sh @@ -74,6 +74,12 @@ function kubectl_install() { sudo install -o root -g root -m 0755 kubectl ${KUBECTL_INSTALL_PATH}/kubectl } +function print_cluster_info() { + $KUBECTL_BIN logs -l app=s3-csi-node -n kube-system --kubeconfig ${KUBECONFIG} + $KUBECTL_BIN version --kubeconfig ${KUBECONFIG} + $KUBECTL_BIN get nodes -o wide --kubeconfig ${KUBECONFIG} +} + function install_tools() { kubectl_install diff --git a/tests/e2e-kubernetes/testdriver.go b/tests/e2e-kubernetes/testdriver.go index 43559a3c..592e76d2 100644 --- a/tests/e2e-kubernetes/testdriver.go +++ b/tests/e2e-kubernetes/testdriver.go @@ -3,11 +3,14 @@ package e2e import ( "context" "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + custom_testsuites "github.com/awslabs/aws-s3-csi-driver/tests/e2e-kubernetes/testsuites" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/storage/names" @@ -16,6 +19,10 @@ import ( "k8s.io/kubernetes/test/e2e/storage/framework" ) +const ( + maxS3ExpressBucketNameLength = 63 +) + var ( CommitId string BucketRegion string // assumed to be the same as k8s cluster's region @@ -80,8 +87,44 @@ func (d *s3Driver) CreateVolume(ctx context.Context, config *framework.PerTestCo bucketName := names.SimpleNameGenerator.GenerateName(fmt.Sprintf("%s-e2e-kubernetes-%s-", BucketPrefix, CommitId)) input := &s3.CreateBucketInput{ Bucket: aws.String(bucketName), + // note: you need this if testing in non us-east-1 regions + // CreateBucketConfiguration: &types.CreateBucketConfiguration{ + // LocationConstraint: types.BucketLocationConstraint(BucketRegion), + // }, + } + + if config.Prefix == custom_testsuites.S3ExpressTestIdentifier { + // assume us-east-1 since that's where our integration tests currently do their work + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-networking.html + regionAz := "use1-az4" + if BucketRegion == "us-west-2" { + regionAz = "usw2-az1" + } + // refer to s3 express bucket naming conventions + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-naming-rules.html + suffix := fmt.Sprintf("--%s--x-s3", regionAz) + // s3 express doesn't allow non-virtually routable names + bucketName = strings.Replace(bucketName, ".", "", -1) + if len(bucketName)+len(suffix) > maxS3ExpressBucketNameLength { + bucketName = strings.TrimRight(bucketName[:maxS3ExpressBucketNameLength-len(suffix)], "-") + } + bucketName = fmt.Sprintf("%s%s", bucketName, suffix) + input = &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + CreateBucketConfiguration: &types.CreateBucketConfiguration{ + Location: &types.LocationInfo{ + Name: aws.String(regionAz), + Type: types.LocationTypeAvailabilityZone, + }, + Bucket: &types.BucketInfo{ + DataRedundancy: types.DataRedundancySingleAvailabilityZone, + Type: types.BucketTypeDirectory, + }, + }, + } } - _, err := newS3Client().CreateBucket(input) + f.Logf("Attempting to create bucket: %s", bucketName) + _, err := newS3Client().CreateBucket(context.TODO(), input) f.ExpectNoError(err) f.Logf("Created bucket: %s", bucketName) return &s3Volume{bucketName: bucketName} @@ -100,23 +143,36 @@ func (d *s3Driver) GetPersistentVolumeSource(readOnly bool, fsType string, testV func (v *s3Volume) DeleteVolume(ctx context.Context) { s3Client := newS3Client() - // delete all objects from a bucket - iter := s3manager.NewDeleteListIterator(s3Client, &s3.ListObjectsInput{ + objects, err := s3Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ Bucket: aws.String(v.bucketName), }) - err := s3manager.NewBatchDeleteWithClient(s3Client).Delete(aws.BackgroundContext(), iter) f.ExpectNoError(err) + var objectIds []types.ObjectIdentifier + // get all object keys in the s3 bucket + for _, obj := range objects.Contents { + objectIds = append(objectIds, types.ObjectIdentifier{Key: obj.Key}) + } + // delete all objects from the bucket + if len(objectIds) > 0 { + _, err = s3Client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{ + Bucket: aws.String(v.bucketName), + Delete: &types.Delete{Objects: objectIds}, + }) + f.ExpectNoError(err) + } // finally delete the bucket input := &s3.DeleteBucketInput{ Bucket: aws.String(v.bucketName), } - _, err = s3Client.DeleteBucket(input) + _, err = s3Client.DeleteBucket(context.TODO(), input) f.ExpectNoError(err) f.Logf("Deleted bucket: %s", v.bucketName) } -func newS3Client() *s3.S3 { - session, err := session.NewSession(&aws.Config{Region: aws.String(BucketRegion)}) +func newS3Client() *s3.Client { + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(BucketRegion), + ) f.ExpectNoError(err) - return s3.New(session) + return s3.NewFromConfig(cfg) } diff --git a/tests/e2e-kubernetes/testsuites/mountoptions.go b/tests/e2e-kubernetes/testsuites/mountoptions.go index 147cf389..1dce0d40 100644 --- a/tests/e2e-kubernetes/testsuites/mountoptions.go +++ b/tests/e2e-kubernetes/testsuites/mountoptions.go @@ -33,6 +33,10 @@ import ( "k8s.io/utils/pointer" ) +const ( + S3ExpressTestIdentifier = "express" +) + type s3CSIMountOptionsTestSuite struct { tsInfo storageframework.TestSuiteInfo } @@ -79,8 +83,9 @@ func (t *s3CSIMountOptionsTestSuite) DefineTests(driver storageframework.TestDri l.config = driver.PrepareTest(ctx, f) ginkgo.DeferCleanup(cleanup) }) - ginkgo.It("should access volume as a non-root user", func(ctx context.Context) { - resource := createVolumeResourceWithMountOptions(ctx, l.config, pattern, []string{"uid=1000", "gid=2000", "allow-other"}) + + validateWriteToVolume := func(ctx context.Context) { + resource := createVolumeResourceWithMountOptions(ctx, l.config, pattern, []string{"uid=1000", "gid=2000", "allow-other", "debug", "debug-crt"}) l.resources = append(l.resources, resource) ginkgo.By("Creating pod with a volume") pod := e2epod.MakePod(f.Namespace.Name, nil, []*v1.PersistentVolumeClaim{resource.Pvc}, admissionapi.LevelRestricted, "") @@ -105,8 +110,16 @@ func (t *s3CSIMountOptionsTestSuite) DefineTests(driver storageframework.TestDri e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("stat -L -c '%%a %%g %%u' %s | grep '755 2000 1000'", volPath)) ginkgo.By("Checking pod identity") e2evolume.VerifyExecInPodSucceed(f, pod, "id | grep 'uid=1000 gid=2000 groups=2000'") + } + ginkgo.It("should access volume as a non-root user", func(ctx context.Context) { + validateWriteToVolume(ctx) }) - ginkgo.It("should not be able to access volume as a non-root user", func(ctx context.Context) { + ginkgo.It("S3 express -- should access volume as a non-root user", func(ctx context.Context) { + l.config.Prefix = S3ExpressTestIdentifier + validateWriteToVolume(ctx) + }) + + accessVolAsNonRootUser := func(ctx context.Context) { resource := createVolumeResourceWithMountOptions(ctx, l.config, pattern, []string{}) l.resources = append(l.resources, resource) ginkgo.By("Creating pod with a volume") @@ -123,5 +136,12 @@ func (t *s3CSIMountOptionsTestSuite) DefineTests(driver storageframework.TestDri _, stderr, err := e2evolume.PodExec(f, pod, fmt.Sprintf("ls %s", volPath)) gomega.Expect(err).To(gomega.HaveOccurred()) gomega.Expect(stderr).To(gomega.ContainSubstring("Permission denied")) + } + ginkgo.It("should not be able to access volume as a non-root user", func(ctx context.Context) { + accessVolAsNonRootUser(ctx) + }) + ginkgo.It("S3 express -- should not be able to access volume as a non-root user", func(ctx context.Context) { + l.config.Prefix = S3ExpressTestIdentifier + accessVolAsNonRootUser(ctx) }) }