Merge pull request #1292 from seanzatzdev-amazon/master

Add static & dynamic provisioning options for using efs-utils crossac…

examples/kubernetes/cross_account_mount/README.md

@@ -1,7 +1,7 @@
 ## Dynamic Provisioning
 This example shows how to create a dynamically provisioned volume created through [EFS access points](https://docs.aws.amazon.com/efs/latest/ug/efs-access-points.html) and Persistent Volume Claim (PVC) and consume it from a pod.
 
-**Note**: this example requires Kubernetes v1.17+ and driver version >= 1.2.0.
+**Note**: this example requires Kubernetes v1.17+ and driver version >= 1.8.0.
 
 ### Edit [StorageClass](./specs/storageclass.yaml)
 
@@ -30,7 +30,7 @@ Lets say you have an EKS cluster in aws account `A` & you wish to mount your fil
 1. Perform [vpc-peering](https://docs.aws.amazon.com/vpc/latest/peering/working-with-vpc-peering.html) between EKS cluster `vpc` in aws account `A` and EFS `vpc` in another aws account `B`.
 2. Create an IAM role, say `EFSCrossAccountAccessRole` in Account `B` which has a [trust relationship](./iam-policy-examples/trust-relationship-example.json) with Account `A` and add an inline EFS policy with [permissions](./iam-policy-examples/describe-mount-target-example.json) to call `DescribeMountTargets`. This role will be used by CSI-Driver's Controller service running on EKS cluster in account `A` to determine the mount targets for your file system in account `B`. 
 3. In aws account `A`, attach an inline policy to IAM role of efs-csi-driver's controller service account with necessary [permissions](./iam-policy-examples/cross-account-assume-policy-example.json) to perform `sts assume role` on the IAM role created in step 2.  
-4. Create a kubernetes secret with `awsRoleArn` as the key and the role from step 2 as the value. For example, `kubectl create secret generic x-account --namespace=default --from-literal=awsRoleArn='arn:aws:iam::123456789012:role/EFSCrossAccountAccessRole'`.
+4. Create a kubernetes secret with `awsRoleArn` as the key and the role from step 2 as the value. For example, `kubectl create secret generic x-account --namespace=default --from-literal=awsRoleArn='arn:aws:iam::123456789012:role/EFSCrossAccountAccessRole'`. If you would like to ensure that your EFS Mount Target is in the same availability zone as your EKS Node, then ensure you have completed the [prerequisites for cross-account DNS resolution](https://github.com/aws/efs-utils?tab=readme-ov-file#crossaccount-option-prerequisites) and include the `crossaccount` key with value `true`. For example, `kubectl create secret generic x-account --namespace=kube-system --from-literal=awsRoleArn='arn:aws:iam::123456789012:role/EFSCrossAccountAccessRole' --from-literal=crossaccount='true'` instead.
 5. Create an IAM role for service accounts for EKS cluster in account `A` with required [permissions](./iam-policy-examples/node-deamonset-iam-policy-example.json) for EFS client mount. Alternatively, you can find this policy under AWS managed policy as `AmazonElasticFileSystemClientFullAccess`.  
 6. Attach the service account from step 5 to node daemonset.
 7. Create a [file system policy](https://docs.aws.amazon.com/efs/latest/ug/iam-access-control-nfs-efs.html#file-sys-policy-examples) for file system in account `B` which allows account `A` to perform mount on it.
@@ -58,3 +58,55 @@ Also you can verify that data is written onto EFS filesystem:
 >> kubectl exec -ti efs-app -- tail -f /data/out
 ```
 
+## Static Provisioning
+This example shows how to perform cross-account static provisioning.
+
+**Note**: this example requires Kubernetes v1.17+ and driver version >= 1.8.0.
+
+### Edit [PersistentVolume](./specs/pv.yaml) config file volumeAttributes
+
+```
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: efs-pv
+spec:
+  capacity:
+    storage: 5Gi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: efs-sc
+  persistentVolumeReclaimPolicy: Retain
+  csi:
+    driver: efs.csi.aws.com
+    volumeHandle: [Filesystem ID]
+    volumeAttributes:
+      crossaccount: "true"
+```
+Replace [Filesystem ID] with the corresponding EFS filesystem ID.
+
+### Prerequisite setup
+Complete the [efs-utils crossaccount mount option setup](https://github.com/aws/efs-utils?tab=readme-ov-file#crossaccount-option-prerequisites).
+
+### Deploy the Example Application
+Create PV and persistent volume claim (PVC):
+```sh
+>> kubectl apply -f examples/kubernetes/cross_account_mount/specs/storageclass-static-prov.yaml
+>> kubectl apply -f examples/kubernetes/cross_account_mount/specs/pv.yaml
+>> kubectl apply -f examples/kubernetes/cross_account_mount/specs/claim.yaml
+>> kubectl apply -f examples/kubernetes/cross_account_mount/specs/pod-static-prov.yaml
+```
+
+### Check EFS filesystem is used
+After the objects are created, verify that pod is running:
+
+```sh
+>> kubectl get pods
+```
+
+Also you can verify that data is written onto EFS filesystem:
+
+```sh
+>> kubectl exec -ti efs-app -- tail -f /data/out.txt
+```

examples/kubernetes/cross_account_mount/specs/claim.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: efs-claim
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: efs-sc
+  resources:
+    requests:
+      storage: 5Gi

examples/kubernetes/cross_account_mount/specs/pod-static-prov.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: efs-app
+spec:
+  containers:
+  - name: app
+    image: centos
+    command: ["/bin/sh"]
+    args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"]
+    volumeMounts:
+    - name: persistent-storage
+      mountPath: /data
+  volumes:
+  - name: persistent-storage
+    persistentVolumeClaim:
+      claimName: efs-claim

examples/kubernetes/cross_account_mount/specs/pv.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: efs-pv
+spec:
+  capacity:
+    storage: 5Gi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: efs-sc
+  persistentVolumeReclaimPolicy: Retain
+  csi:
+    driver: efs.csi.aws.com
+    volumeHandle: [Filesystem ID]
+    volumeAttributes:
+      crossaccount: "true"

examples/kubernetes/cross_account_mount/specs/storageclass-static-prov.yaml

@@ -0,0 +1,5 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: efs-sc
+provisioner: efs.csi.aws.com

pkg/driver/controller.go

@@ -20,13 +20,14 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
-	"github.com/google/uuid"
 	"os"
 	"path"
 	"sort"
 	"strconv"
 	"strings"
 
+	"github.com/google/uuid"
+
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/kubernetes-sigs/aws-efs-csi-driver/pkg/cloud"
 	"google.golang.org/grpc/codes"
@@ -59,6 +60,7 @@ const (
 	Uid                   = "uid"
 	ReuseAccessPointKey   = "reuseAccessPoint"
 	PvcNameKey            = "csi.storage.k8s.io/pvc/name"
+	CrossAccount          = "crossaccount"
 )
 
 var (
@@ -117,15 +119,16 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
 	}
 
 	var (
-		azName           string
-		basePath         string
-		gid              int64
-		gidMin           int64
-		gidMax           int64
-		localCloud       cloud.Cloud
-		provisioningMode string
-		roleArn          string
-		uid              int64
+		azName                 string
+		basePath               string
+		gid                    int64
+		gidMin                 int64
+		gidMax                 int64
+		localCloud             cloud.Cloud
+		provisioningMode       string
+		roleArn                string
+		uid                    int64
+		crossAccountDNSEnabled bool
 	)
 
 	//Parse parameters
@@ -236,7 +239,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
 		azName = value
 	}
 
-	localCloud, roleArn, err = getCloud(req.GetSecrets(), d)
+	localCloud, roleArn, crossAccountDNSEnabled, err = getCloud(req.GetSecrets(), d)
 	if err != nil {
 		return nil, err
 	}
@@ -326,13 +329,22 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
 
 	volContext := map[string]string{}
 
-	// Fetch mount target Ip for cross-account mount
+	// Enable cross-account dns resolution or fetch mount target Ip for cross-account mount
 	if roleArn != "" {
-		mountTarget, err := localCloud.DescribeMountTargets(ctx, accessPointsOptions.FileSystemId, azName)
-		if err != nil {
-			klog.Warningf("Failed to describe mount targets for file system %v. Skip using `mounttargetip` mount option: %v", accessPointsOptions.FileSystemId, err)
+		if crossAccountDNSEnabled {
+			// This option indicates the customer would like to use DNS to resolve
+			// the cross-account mount target ip address (in order to mount to
+			// the same AZ-ID as the client instance); mounttargetip should
+			// not be used as a mount option in this case.
+			volContext[CrossAccount] = strconv.FormatBool(true)
 		} else {
-			volContext[MountTargetIp] = mountTarget.IPAddress
+			mountTarget, err := localCloud.DescribeMountTargets(ctx, accessPointsOptions.FileSystemId, azName)
+			if err != nil {
+				klog.Warningf("Failed to describe mount targets for file system %v. Skip using `mounttargetip` mount option: %v", accessPointsOptions.FileSystemId, err)
+			} else {
+				volContext[MountTargetIp] = mountTarget.IPAddress
+			}
+
 		}
 	}
 
@@ -347,12 +359,13 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
 
 func (d *Driver) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
 	var (
-		localCloud cloud.Cloud
-		roleArn    string
-		err        error
+		localCloud             cloud.Cloud
+		roleArn                string
+		crossAccountDNSEnabled bool
+		err                    error
 	)
 
-	localCloud, roleArn, err = getCloud(req.GetSecrets(), d)
+	localCloud, roleArn, crossAccountDNSEnabled, err = getCloud(req.GetSecrets(), d)
 	if err != nil {
 		return nil, err
 	}
@@ -392,12 +405,16 @@ func (d *Driver) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest)
 			//Mount File System at it root and delete access point root directory
 			mountOptions := []string{"tls", "iam"}
 			if roleArn != "" {
-				mountTarget, err := localCloud.DescribeMountTargets(ctx, fileSystemId, "")
-
-				if err == nil {
-					mountOptions = append(mountOptions, MountTargetIp+"="+mountTarget.IPAddress)
+				if crossAccountDNSEnabled {
+					// Connect via dns rather than mounttargetip
+					mountOptions = append(mountOptions, CrossAccount)
 				} else {
-					klog.Warningf("Failed to describe mount targets for file system %v. Skip using `mounttargetip` mount option: %v", fileSystemId, err)
+					mountTarget, err := localCloud.DescribeMountTargets(ctx, fileSystemId, "")
+					if err == nil {
+						mountOptions = append(mountOptions, MountTargetIp+"="+mountTarget.IPAddress)
+					} else {
+						klog.Warningf("Failed to describe mount targets for file system %v. Skip using `mounttargetip` mount option: %v", fileSystemId, err)
+					}
 				}
 			}
 
@@ -520,28 +537,37 @@ func (d *Driver) ControllerGetVolume(ctx context.Context, req *csi.ControllerGet
 	return nil, status.Error(codes.Unimplemented, "")
 }
 
-func getCloud(secrets map[string]string, driver *Driver) (cloud.Cloud, string, error) {
+func getCloud(secrets map[string]string, driver *Driver) (cloud.Cloud, string, bool, error) {
 
 	var localCloud cloud.Cloud
 	var roleArn string
+	var crossAccountDNSEnabled bool
 	var err error
 
 	// Fetch aws role ARN for cross account mount from CSI secrets. Link to CSI secrets below
 	// https://kubernetes-csi.github.io/docs/secrets-and-credentials.html#csi-operation-secrets
 	if value, ok := secrets[RoleArn]; ok {
 		roleArn = value
 	}
+	if value, ok := secrets[CrossAccount]; ok {
+		crossAccountDNSEnabled, err = strconv.ParseBool(value)
+		if err != nil {
+			return nil, "", false, status.Error(codes.InvalidArgument, "crossaccount parameter must have boolean value.")
+		}
+	} else {
+		crossAccountDNSEnabled = false
+	}
 
 	if roleArn != "" {
 		localCloud, err = cloud.NewCloudWithRole(roleArn)
 		if err != nil {
-			return nil, "", status.Errorf(codes.Unauthenticated, "Unable to initialize aws cloud: %v. Please verify role has the correct AWS permissions for cross account mount", err)
+			return nil, "", false, status.Errorf(codes.Unauthenticated, "Unable to initialize aws cloud: %v. Please verify role has the correct AWS permissions for cross account mount", err)
 		}
 	} else {
 		localCloud = driver.cloud
 	}
 
-	return localCloud, roleArn, nil
+	return localCloud, roleArn, crossAccountDNSEnabled, nil
 }
 
 func interpolateRootDirectoryName(rootDirectoryPath string, volumeParams map[string]string) (string, error) {

pkg/driver/node.go

@@ -78,6 +78,7 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu
 	// TODO when CreateVolume is implemented, it must use the same key names
 	subpath := "/"
 	encryptInTransit := true
+	crossAccountDNSEnabled := false
 	volContext := req.GetVolumeContext()
 	for k, v := range volContext {
 		switch strings.ToLower(k) {
@@ -99,8 +100,14 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu
 		case MountTargetIp:
 			ipAddr := volContext[MountTargetIp]
 			mountOptions = append(mountOptions, MountTargetIp+"="+ipAddr)
+		case CrossAccount:
+			var err error
+			crossAccountDNSEnabled, err = strconv.ParseBool(v)
+			if err != nil {
+				return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Volume context property %q must be a boolean value: %v", k, err))
+			}
 		default:
-			return nil, status.Errorf(codes.InvalidArgument, "Volume context property %s not supported", k)
+			return nil, status.Errorf(codes.InvalidArgument, "Volume context property %s not supported.", k)
 		}
 	}
 
@@ -133,6 +140,10 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu
 		}
 	}
 
+	if crossAccountDNSEnabled {
+		mountOptions = append(mountOptions, CrossAccount)
+	}
+
 	if req.GetReadonly() {
 		mountOptions = append(mountOptions, "ro")
 	}

pkg/driver/node_test.go

@@ -498,7 +498,7 @@ func TestNodePublishVolume(t *testing.T) {
 			expectMakeDir: false,
 			expectError: errtyp{
 				code:    "InvalidArgument",
-				message: "Volume context property asdf not supported",
+				message: "Volume context property asdf not supported.",
 			},
 		},
 		{