Allow using IRSA for EBS CSI Driver

pkg/model/components/addonmanifests/awsebscsidriver/iam.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awsebscsidriver
+
+import (
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kops/pkg/model/iam"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// ServiceAccount represents the service-account used by the dns-controller.
+// It implements iam.Subject to get AWS IAM permissions.
+type ServiceAccount struct {
+}
+
+var _ iam.Subject = &ServiceAccount{}
+
+// BuildAWSPolicy generates a custom policy for a ServiceAccount IAM role.
+func (r *ServiceAccount) BuildAWSPolicy(b *iam.PolicyBuilder) (*iam.Policy, error) {
+	p := &iam.Policy{
+		Version: iam.PolicyDefaultVersion,
+	}
+
+	addSnapshotControllerPermissions := b.Cluster.Spec.SnapshotController != nil && fi.BoolValue(b.Cluster.Spec.SnapshotController.Enabled)
+	iam.AddAWSEBSCSIDriverPermissions(p, b.Cluster.ObjectMeta.Name, addSnapshotControllerPermissions)
+
+	return p, nil
+}
+
+// ServiceAccount returns the kubernetes service account used.
+func (r *ServiceAccount) ServiceAccount() (types.NamespacedName, bool) {
+	return types.NamespacedName{
+		Namespace: "kube-system",
+		Name:      "ebs-csi-controller-sa",
+	}, true
+}

pkg/model/components/addonmanifests/dnscontroller/remap.go

@@ -69,9 +69,8 @@ func Remap(context *model.KopsModelContext, addon *addonsapi.AddonSpec, objects
 	if len(containers) != 1 {
 		return fmt.Errorf("expected exactly one container in dns-controller Deployment, found %d", len(containers))
 	}
-	container := &containers[0]
 
-	if err := iam.AddServiceAccountRole(&context.IAMModelContext, podSpec, container, &ServiceAccount{}); err != nil {
+	if err := iam.AddServiceAccountRole(&context.IAMModelContext, podSpec, &ServiceAccount{}); err != nil {
 		return err
 	}
 

pkg/model/components/addonmanifests/remap.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/pkg/model"
+	"k8s.io/kops/pkg/model/components/addonmanifests/awsebscsidriver"
 	"k8s.io/kops/pkg/model/components/addonmanifests/awsloadbalancercontroller"
 	"k8s.io/kops/pkg/model/components/addonmanifests/clusterautoscaler"
 	"k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller"
@@ -96,18 +97,16 @@ func addServiceAccountRole(context *model.KopsModelContext, objects kubemanifest
 		if err := object.Reparse(podSpec, "spec", "template", "spec"); err != nil {
 			return fmt.Errorf("failed to parse spec.template.spec from Deployment: %v", err)
 		}
-		containers := podSpec.Containers
 		sa := podSpec.ServiceAccountName
 		subject := getWellknownServiceAccount(sa)
 		if subject == nil {
 			continue
 		}
-		for k, container := range containers {
-			if err := iam.AddServiceAccountRole(&context.IAMModelContext, podSpec, &container, subject); err != nil {
-				return err
-			}
-			containers[k] = container
+
+		if err := iam.AddServiceAccountRole(&context.IAMModelContext, podSpec, subject); err != nil {
+			return err
 		}
+
 		if err := object.Set(podSpec, "spec", "template", "spec"); err != nil {
 			return fmt.Errorf("failed to set object: %w", err)
 		}
@@ -122,6 +121,8 @@ func getWellknownServiceAccount(name string) iam.Subject {
 		return &awsloadbalancercontroller.ServiceAccount{}
 	case "cluster-autoscaler":
 		return &clusterautoscaler.ServiceAccount{}
+	case "ebs-csi-controller-sa":
+		return &awsebscsidriver.ServiceAccount{}
 	default:
 		return nil
 	}

pkg/model/iam/iam_builder.go

@@ -283,6 +283,7 @@ func (r *NodeRoleAPIServer) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) {
 // BuildAWSPolicy generates a custom policy for a Kubernetes master.
 func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) {
 	resource := createResource(b)
+	clusterName := b.Cluster.GetName()
 
 	p := &Policy{
 		Version: PolicyDefaultVersion,
@@ -303,6 +304,12 @@ func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) {
 		addKMSIAMPolicies(p, stringorslice.Slice(b.KMSKeys))
 	}
 
+	if !fi.BoolValue(b.Cluster.Spec.CloudConfig.AWSEBSCSIDriver.Enabled) {
+		esc := b.Cluster.Spec.SnapshotController != nil &&
+			fi.BoolValue(b.Cluster.Spec.SnapshotController.Enabled)
+		AddAWSEBSCSIDriverPermissions(p, clusterName, esc)
+	}
+
 	// Protokube needs dns-controller permissions in instance role even if UseServiceAccountIAM.
 	AddDNSControllerPermissions(b, p)
 
@@ -790,6 +797,117 @@ func AddClusterAutoscalerPermissions(p *Policy, clusterName string) {
 	)
 }
 
+// AddAWSEBSCSIDriverPermissions appens policy statements that the AWS EBS CSI Driver needs to operate.
+func AddAWSEBSCSIDriverPermissions(p *Policy, clusterName string, appendSnapshotPermissions bool) {
+
+	everything := stringorslice.String("*")
+
+	if appendSnapshotPermissions {
+		addSnapshotPersmissions(p, clusterName)
+	}
+
+	p.Statement = append(p.Statement,
+		&Statement{
+			Effect: StatementEffectAllow,
+			Action: stringorslice.Slice([]string{
+				"ec2:DescribeAccountAttributes",    // aws.go
+				"ec2:DescribeInstances",            // aws.go
+				"ec2:DescribeVolumes",              // aws.go
+				"ec2:DescribeVolumesModifications", // aws.go
+				"ec2:DescribeTags",                 // aws.go
+			}),
+			Resource: everything,
+		},
+		&Statement{
+			Effect: StatementEffectAllow,
+			Action: stringorslice.Slice([]string{
+				"ec2:CreateVolume", // aws.go
+			}),
+
+			Resource: everything,
+			Condition: Condition{
+				"StringEquals": map[string]string{
+					"aws:RequestTag/KubernetesCluster": clusterName,
+				},
+			},
+		},
+
+		&Statement{
+			Effect: StatementEffectAllow,
+			Action: stringorslice.Slice([]string{
+				"ec2:ModifyVolume",            // aws.go
+				"ec2:ModifyInstanceAttribute", // aws.go
+				"ec2:AttachVolume",            // aws.go
+				"ec2:DeleteVolume",            // aws.go
+				"ec2:DetachVolume",            // aws.go
+			}),
+
+			Resource: everything,
+			Condition: Condition{
+				"StringEquals": map[string]string{
+					"aws:ResourceTag/KubernetesCluster": clusterName,
+				},
+			},
+		},
+
+		&Statement{
+			Effect: StatementEffectAllow,
+			Action: stringorslice.String(
+				"ec2:CreateTags", // aws.go, tag.go
+			),
+
+			Resource: stringorslice.Slice(
+				[]string{
+					"arn:aws:ec2:*:*:volume/*",
+					"arn:aws:ec2:*:*:snapshot/*",
+				},
+			),
+			Condition: Condition{
+				"StringEquals": map[string]interface{}{
+					"ec2:CreateAction": []string{
+						"CreateVolume",
+						"CreateSnapshot",
+					},
+				},
+			},
+		},
+
+		&Statement{
+			Effect: StatementEffectAllow,
+			Action: stringorslice.String(
+				"ec2:DeleteTags", // aws.go, tag.go
+			),
+			Resource: stringorslice.Slice(
+				[]string{
+					"arn:aws:ec2:*:*:volume/*",
+					"arn:aws:ec2:*:*:snapshot/*",
+				},
+			),
+			Condition: Condition{
+				"StringEquals": map[string]string{
+					"ec2:ResourceTag/KubernetesCluster": clusterName,
+				},
+			},
+		},
+
+		&Statement{
+			Effect: StatementEffectAllow,
+			Action: stringorslice.Of(
+				"ec2:AttachVolume",               // aws.go
+				"ec2:DeleteVolume",               // aws.go
+				"ec2:DetachVolume",               // aws.go
+				"ec2:RevokeSecurityGroupIngress", // aws.go
+			),
+			Resource: everything,
+			Condition: Condition{
+				"StringEquals": map[string]string{
+					"ec2:ResourceTag/KubernetesCluster": clusterName,
+				},
+			},
+		},
+	)
+}
+
 func addSnapshotPersmissions(p *Policy, clusterName string) {
 	p.Statement = append(p.Statement, &Statement{
 		Effect: StatementEffectAllow,
@@ -907,10 +1025,8 @@ func addMasterEC2Policies(p *Policy, resource stringorslice.StringOrSlice, clust
 			Action: stringorslice.Slice([]string{
 				"ec2:CreateSecurityGroup",          // aws.go
 				"ec2:CreateTags",                   // aws.go, tag.go
-				"ec2:CreateVolume",                 // aws.go
 				"ec2:DescribeVolumesModifications", // aws.go
 				"ec2:ModifyInstanceAttribute",      // aws.go
-				"ec2:ModifyVolume",                 // aws.go
 			}),
 			Resource: resource,
 		},
@@ -922,8 +1038,6 @@ func addMasterEC2Policies(p *Policy, resource stringorslice.StringOrSlice, clust
 				"ec2:CreateRoute",                   // aws.go
 				"ec2:DeleteRoute",                   // aws.go
 				"ec2:DeleteSecurityGroup",           // aws.go
-				"ec2:DeleteVolume",                  // aws.go
-				"ec2:DetachVolume",                  // aws.go
 				"ec2:RevokeSecurityGroupIngress",    // aws.go
 			),
 			Resource: resource,
@@ -938,11 +1052,7 @@ func addMasterEC2Policies(p *Policy, resource stringorslice.StringOrSlice, clust
 			Action: stringorslice.Of(
 				"ec2:AttachVolume",                  // aws.go
 				"ec2:AuthorizeSecurityGroupIngress", // aws.go
-				"ec2:CreateRoute",                   // aws.go
-				"ec2:DeleteRoute",                   // aws.go
 				"ec2:DeleteSecurityGroup",           // aws.go
-				"ec2:DeleteVolume",                  // aws.go
-				"ec2:DetachVolume",                  // aws.go
 				"ec2:RevokeSecurityGroupIngress",    // aws.go
 			),
 			Resource: resource,

pkg/model/iam/iam_builder_test.go

@@ -182,6 +182,11 @@ func TestPolicyGeneration(t *testing.T) {
 							},
 						},
 					},
+					CloudConfig: &kops.CloudConfiguration{
+						AWSEBSCSIDriver: &kops.AWSEBSCSIDriver{
+							Enabled: fi.Bool(true),
+						},
+					},
 				},
 			},
 			Role: x.Role,

pkg/model/iam/subject.go

@@ -156,18 +156,18 @@ func supportsPublicJWKS(clusterSpec *kops.ClusterSpec) bool {
 }
 
 // AddServiceAccountRole adds the appropriate mounts / env vars to enable a pod to use a service-account role
-func AddServiceAccountRole(context *IAMModelContext, podSpec *corev1.PodSpec, container *corev1.Container, serviceAccountRole Subject) error {
+func AddServiceAccountRole(context *IAMModelContext, podSpec *corev1.PodSpec, serviceAccountRole Subject) error {
 	cloudProvider := kops.CloudProviderID(context.Cluster.Spec.CloudProvider)
 
 	switch cloudProvider {
 	case kops.CloudProviderAWS:
-		return addServiceAccountRoleForAWS(context, podSpec, container, serviceAccountRole)
+		return addServiceAccountRoleForAWS(context, podSpec, serviceAccountRole)
 	default:
 		return fmt.Errorf("ServiceAccount-level IAM is not yet supported on cloud %T", cloudProvider)
 	}
 }
 
-func addServiceAccountRoleForAWS(context *IAMModelContext, podSpec *corev1.PodSpec, container *corev1.Container, serviceAccountRole Subject) error {
+func addServiceAccountRoleForAWS(context *IAMModelContext, podSpec *corev1.PodSpec, serviceAccountRole Subject) error {
 	roleName, err := context.IAMNameForServiceAccountRole(serviceAccountRole)
 	if err != nil {
 		return err
@@ -197,21 +197,26 @@ func addServiceAccountRoleForAWS(context *IAMModelContext, podSpec *corev1.PodSp
 	}
 	podSpec.Volumes = append(podSpec.Volumes, volume)
 
-	container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
-		MountPath: tokenDir,
-		Name:      volume.Name,
-		ReadOnly:  true,
-	})
-
-	container.Env = append(container.Env, corev1.EnvVar{
-		Name:  "AWS_ROLE_ARN",
-		Value: awsRoleARN,
-	})
-
-	container.Env = append(container.Env, corev1.EnvVar{
-		Name:  "AWS_WEB_IDENTITY_TOKEN_FILE",
-		Value: tokenDir + tokenName,
-	})
+	containers := podSpec.Containers
+	for k, container := range containers {
+
+		container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
+			MountPath: tokenDir,
+			Name:      volume.Name,
+			ReadOnly:  true,
+		})
+
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "AWS_ROLE_ARN",
+			Value: awsRoleARN,
+		})
+
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "AWS_WEB_IDENTITY_TOKEN_FILE",
+			Value: tokenDir + tokenName,
+		})
+		containers[k] = container
+	}
 
 	// Set securityContext.fsGroup to enable file to be read
 	// background: https://github.com/kubernetes/enhancements/pull/1598