Upgrading MP and adding tests for S3 Express

.github/workflows/e2e-tests.yaml

@@ -61,7 +61,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        cluster-type: ["kops", "eksctl"]
+        cluster-type: ["eksctl", "kops"]
         arch: ["x86", "arm"]
     runs-on: ubuntu-latest
     permissions:

Dockerfile

@@ -12,7 +12,7 @@
 #See the License for the specific language governing permissions and
 #limitations under the License.
 
-ARG MOUNTPOINT_VERSION=1.1.1
+ARG MOUNTPOINT_VERSION=1.3.1
 
 # Download the mountpoint tarball and produce an installable directory
 # Building on Amazon Linux 2 because it has an old libc version. libfuse from the os

tests/e2e-kubernetes/go.mod

@@ -9,11 +9,32 @@ require (
 	k8s.io/kubernetes v1.28.3
 )
 
+require (
+	github.com/aws/aws-sdk-go-v2 v1.23.4 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.8 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.8 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.1 // indirect
+	github.com/aws/smithy-go v1.18.1 // indirect
+)
+
 require (
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/aws/aws-sdk-go v1.46.1 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.25.10
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.1
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect

tests/e2e-kubernetes/go.sum

@@ -50,6 +50,42 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.46.1 h1:U26quvBWFZMQuultLw5tloW4GnmWaChEwMZNq8uYatw=
 github.com/aws/aws-sdk-go v1.46.1/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go-v2 v1.23.4 h1:2P20ZjH0ouSAu/6yZep8oCmTReathLuEu6dwoqEgjts=
+github.com/aws/aws-sdk-go-v2 v1.23.4/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 h1:Zx9+31KyB8wQna6SXFWOewlgoY5uGdDAu6PTOEU3OQI=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3/go.mod h1:zxbEJhRdKTH1nqS2qu6UJ7zGe25xaHxZXaC2CvuQFnA=
+github.com/aws/aws-sdk-go-v2/config v1.25.10 h1:qw/e8emDtNufTkrAU86DlQ18DruMyyM7ttW6Lgwp4v0=
+github.com/aws/aws-sdk-go-v2/config v1.25.10/go.mod h1:203YiAtb6XyoGxXMPsUVwEcuxCiTQY/r8P27IDjfvMc=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.8 h1:phw9nRLy/77bPk6Mfu2SHCOnHwfVB7WWrOa5rZIY2Fc=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.8/go.mod h1:MrS4SOin6adbO6wgWhdifyPiq+TX7fPPwyA/ZLC1F5M=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.8 h1:tQZLSPC2Zj2CqZHonLmWEvCsbpMX5tQvaYJWHadcPek=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.8/go.mod h1:5+YpvTHDFffykWr5qAGjqwoh8oVYZOddL3sSrEN7lws=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.7 h1:eMqD7ku6WGdmcWWXPYun9m6yk6feSULLhJlAtN6rYG4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.7/go.mod h1:0oBIfcDV6LScxEW0VgOqxT3e4aqKRp+SYhB9wAd5E3Q=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.7 h1:+XYhWhgWs5F3Zx8oa49CXzNvfXrItaDjZB/M172fcHQ=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.7/go.mod h1:L6tcSRyCGxcKfDWUrmv2jv8G1cLDU7d0FUpEFpG9bVE=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.7 h1:3VaUNB1LclLomv82VnP5QnxAfowG+Ro4m82+af9wjZ4=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.7/go.mod h1:D5i0c+qvEY0LV5F4elFZd+mYnvHQbufCLHNHoBfQR2g=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.7 h1:Mft1tmIK1fkFS9l9sYVYiN+OdgXeOcQ9ZS3SxKOh3A4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.7/go.mod h1:QWI83fhocxDaN3b74N8rrvET60CBaike5lQ+5sm3OcE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.7 h1:dU+ZyhvqMB/T/TxjGagHMCdyUiqaThRIaMu3YvKiSQI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.7/go.mod h1:SGORuNqoXyWfTvTp/gBGJfv8jRvW/+nha0XhnIXVI+o=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.7 h1:ybtGXm0qFVFi0hFUF7eFAVnL3ntl9MO7lrxhhGP7KYU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.7/go.mod h1:BUyWJUKAnNqoEq1LfyQxy+Eh4U8Y3c5w2C6m21f3yvI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.47.1 h1:0/W5F+LlXzKZ7KTsRcD8pugasVnsrjUWmhOsN/LdSFY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.47.1/go.mod h1:TqThLn4bRCn/UYf960hNZgPPjmxc17fQcwmjfuG6D5k=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.1 h1:V40g2daNO3l1J94JYwqfkyvQMYXi5I25fs3fNQW8iDs=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.1/go.mod h1:0ZWQJP/mBOUxkCvZKybZNz1XmdUKSBxoF0dzgfxtvDs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.1 h1:uQrj7SpUNC3r55vc1CDh3qV9wJC66lz546xM9dhSo5s=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.1/go.mod h1:oyaTk5xEAOuPXX1kCD7HmIeuLqdj3Bk5yGkqGXtGi14=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.1 h1:K33V7L0XDdb23FMOZySr8bon1jou5SHn1fiv7NJ1SUg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.1/go.mod h1:YtXUl/sfnS06VksYhr855hTQf2HphfT1Xv/EwuzbPjg=
+github.com/aws/smithy-go v1.18.1 h1:pOdBTUfXNazOlxLrgeYalVnuTpKreACHtc62xLwIB3c=
+github.com/aws/smithy-go v1.18.1/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

tests/e2e-kubernetes/scripts/kops-patch.yaml

@@ -5,7 +5,8 @@ spec:
         {
           "Effect": "Allow",
           "Action": [
-            "s3:*"
+            "s3:*",
+            "s3express:*"
           ],
           "Resource": "*"
         }

tests/e2e-kubernetes/scripts/run.sh

@@ -74,6 +74,12 @@ function kubectl_install() {
   sudo install -o root -g root -m 0755 kubectl ${KUBECTL_INSTALL_PATH}/kubectl
 }
 
+function print_cluster_info() {
+  $KUBECTL_BIN logs -l app=s3-csi-node -n kube-system --kubeconfig ${KUBECONFIG}
+  $KUBECTL_BIN version --kubeconfig ${KUBECONFIG}
+  $KUBECTL_BIN get nodes -o wide --kubeconfig ${KUBECONFIG}
+}
+
 function install_tools() {
   kubectl_install
 

tests/e2e-kubernetes/testdriver.go

@@ -3,11 +3,14 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	custom_testsuites "github.com/awslabs/aws-s3-csi-driver/tests/e2e-kubernetes/testsuites"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/storage/names"
@@ -16,6 +19,10 @@ import (
 	"k8s.io/kubernetes/test/e2e/storage/framework"
 )
 
+const (
+	maxS3ExpressBucketNameLength = 63
+)
+
 var (
 	CommitId     string
 	BucketRegion string // assumed to be the same as k8s cluster's region
@@ -80,8 +87,44 @@ func (d *s3Driver) CreateVolume(ctx context.Context, config *framework.PerTestCo
 	bucketName := names.SimpleNameGenerator.GenerateName(fmt.Sprintf("%s-e2e-kubernetes-%s-", BucketPrefix, CommitId))
 	input := &s3.CreateBucketInput{
 		Bucket: aws.String(bucketName),
+		// note: you need this if testing in non us-east-1 regions
+		// CreateBucketConfiguration: &types.CreateBucketConfiguration{
+		// 	LocationConstraint: types.BucketLocationConstraint(BucketRegion),
+		// },
+	}
+
+	if config.Prefix == custom_testsuites.S3ExpressTestIdentifier {
+		// assume us-east-1 since that's where our integration tests currently do their work
+		// https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-networking.html
+		regionAz := "use1-az4"
+		if BucketRegion == "us-west-2" {
+			regionAz = "usw2-az1"
+		}
+		// refer to s3 express bucket naming conventions
+		// https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-naming-rules.html
+		suffix := fmt.Sprintf("--%s--x-s3", regionAz)
+		// s3 express doesn't allow non-virtually routable names
+		bucketName = strings.Replace(bucketName, ".", "", -1)
+		if len(bucketName)+len(suffix) > maxS3ExpressBucketNameLength {
+			bucketName = strings.TrimRight(bucketName[:maxS3ExpressBucketNameLength-len(suffix)], "-")
+		}
+		bucketName = fmt.Sprintf("%s%s", bucketName, suffix)
+		input = &s3.CreateBucketInput{
+			Bucket: aws.String(bucketName),
+			CreateBucketConfiguration: &types.CreateBucketConfiguration{
+				Location: &types.LocationInfo{
+					Name: aws.String(regionAz),
+					Type: types.LocationTypeAvailabilityZone,
+				},
+				Bucket: &types.BucketInfo{
+					DataRedundancy: types.DataRedundancySingleAvailabilityZone,
+					Type:           types.BucketTypeDirectory,
+				},
+			},
+		}
 	}
-	_, err := newS3Client().CreateBucket(input)
+	f.Logf("Attempting to create bucket: %s", bucketName)
+	_, err := newS3Client().CreateBucket(context.TODO(), input)
 	f.ExpectNoError(err)
 	f.Logf("Created bucket: %s", bucketName)
 	return &s3Volume{bucketName: bucketName}
@@ -100,23 +143,36 @@ func (d *s3Driver) GetPersistentVolumeSource(readOnly bool, fsType string, testV
 
 func (v *s3Volume) DeleteVolume(ctx context.Context) {
 	s3Client := newS3Client()
-	// delete all objects from a bucket
-	iter := s3manager.NewDeleteListIterator(s3Client, &s3.ListObjectsInput{
+	objects, err := s3Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
 		Bucket: aws.String(v.bucketName),
 	})
-	err := s3manager.NewBatchDeleteWithClient(s3Client).Delete(aws.BackgroundContext(), iter)
 	f.ExpectNoError(err)
+	var objectIds []types.ObjectIdentifier
+	// get all object keys in the s3 bucket
+	for _, obj := range objects.Contents {
+		objectIds = append(objectIds, types.ObjectIdentifier{Key: obj.Key})
+	}
+	// delete all objects from the bucket
+	if len(objectIds) > 0 {
+		_, err = s3Client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
+			Bucket: aws.String(v.bucketName),
+			Delete: &types.Delete{Objects: objectIds},
+		})
+		f.ExpectNoError(err)
+	}
 	// finally delete the bucket
 	input := &s3.DeleteBucketInput{
 		Bucket: aws.String(v.bucketName),
 	}
-	_, err = s3Client.DeleteBucket(input)
+	_, err = s3Client.DeleteBucket(context.TODO(), input)
 	f.ExpectNoError(err)
 	f.Logf("Deleted bucket: %s", v.bucketName)
 }
 
-func newS3Client() *s3.S3 {
-	session, err := session.NewSession(&aws.Config{Region: aws.String(BucketRegion)})
+func newS3Client() *s3.Client {
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(BucketRegion),
+	)
 	f.ExpectNoError(err)
-	return s3.New(session)
+	return s3.NewFromConfig(cfg)
 }

tests/e2e-kubernetes/testsuites/mountoptions.go

@@ -33,6 +33,10 @@ import (
 	"k8s.io/utils/pointer"
 )
 
+const (
+	S3ExpressTestIdentifier = "express"
+)
+
 type s3CSIMountOptionsTestSuite struct {
 	tsInfo storageframework.TestSuiteInfo
 }
@@ -79,8 +83,9 @@ func (t *s3CSIMountOptionsTestSuite) DefineTests(driver storageframework.TestDri
 		l.config = driver.PrepareTest(ctx, f)
 		ginkgo.DeferCleanup(cleanup)
 	})
-	ginkgo.It("should access volume as a non-root user", func(ctx context.Context) {
-		resource := createVolumeResourceWithMountOptions(ctx, l.config, pattern, []string{"uid=1000", "gid=2000", "allow-other"})
+
+	validateWriteToVolume := func(ctx context.Context) {
+		resource := createVolumeResourceWithMountOptions(ctx, l.config, pattern, []string{"uid=1000", "gid=2000", "allow-other", "debug", "debug-crt"})
 		l.resources = append(l.resources, resource)
 		ginkgo.By("Creating pod with a volume")
 		pod := e2epod.MakePod(f.Namespace.Name, nil, []*v1.PersistentVolumeClaim{resource.Pvc}, admissionapi.LevelRestricted, "")
@@ -105,8 +110,16 @@ func (t *s3CSIMountOptionsTestSuite) DefineTests(driver storageframework.TestDri
 		e2evolume.VerifyExecInPodSucceed(f, pod, fmt.Sprintf("stat -L -c '%%a %%g %%u' %s | grep '755 2000 1000'", volPath))
 		ginkgo.By("Checking pod identity")
 		e2evolume.VerifyExecInPodSucceed(f, pod, "id | grep 'uid=1000 gid=2000 groups=2000'")
+	}
+	ginkgo.It("should access volume as a non-root user", func(ctx context.Context) {
+		validateWriteToVolume(ctx)
 	})
-	ginkgo.It("should not be able to access volume as a non-root user", func(ctx context.Context) {
+	ginkgo.It("S3 express -- should access volume as a non-root user", func(ctx context.Context) {
+		l.config.Prefix = S3ExpressTestIdentifier
+		validateWriteToVolume(ctx)
+	})
+
+	accessVolAsNonRootUser := func(ctx context.Context) {
 		resource := createVolumeResourceWithMountOptions(ctx, l.config, pattern, []string{})
 		l.resources = append(l.resources, resource)
 		ginkgo.By("Creating pod with a volume")
@@ -123,5 +136,12 @@ func (t *s3CSIMountOptionsTestSuite) DefineTests(driver storageframework.TestDri
 		_, stderr, err := e2evolume.PodExec(f, pod, fmt.Sprintf("ls %s", volPath))
 		gomega.Expect(err).To(gomega.HaveOccurred())
 		gomega.Expect(stderr).To(gomega.ContainSubstring("Permission denied"))
+	}
+	ginkgo.It("should not be able to access volume as a non-root user", func(ctx context.Context) {
+		accessVolAsNonRootUser(ctx)
+	})
+	ginkgo.It("S3 express -- should not be able to access volume as a non-root user", func(ctx context.Context) {
+		l.config.Prefix = S3ExpressTestIdentifier
+		accessVolAsNonRootUser(ctx)
 	})
 }