Merge pull request kubernetes#103920 from msau42/automated-cherry-pic…

…k-of-#100637-kubernetes#102212-upstream-release-1.21

Automated cherry pick of kubernetes#100637: storage e2e: automate hostpath YAML updates, hostpath kubernetes#102212: Update pd csi driver images to use v1 images (CSINode,

test/e2e/storage/drivers/csi.go

@@ -47,7 +47,9 @@ import (
 
 	"github.com/onsi/ginkgo"
 	"google.golang.org/grpc/codes"
+	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -142,7 +144,12 @@ func InitHostPathCSIDriver() storageframework.TestDriver {
 		storageframework.CapPVCDataSource:       true,
 		storageframework.CapControllerExpansion: true,
 		storageframework.CapSingleNodeVolume:    true,
-		storageframework.CapVolumeLimits:        true,
+
+		// This is needed for the
+		// testsuites/volumelimits.go `should support volume limits`
+		// test. --maxvolumespernode=10 gets
+		// added when patching the deployment.
+		storageframework.CapVolumeLimits: true,
 	}
 	return initHostPathCSIDriver("csi-hostpath",
 		capabilities,
@@ -152,7 +159,8 @@ func InitHostPathCSIDriver() storageframework.TestDriver {
 		},
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
-		"test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml",
@@ -220,16 +228,47 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*storageframewo
 	}
 
 	o := utils.PatchCSIOptions{
-		OldDriverName:            h.driverInfo.Name,
-		NewDriverName:            config.GetUniqueDriverName(),
-		DriverContainerName:      "hostpath",
-		DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName()},
+		OldDriverName:       h.driverInfo.Name,
+		NewDriverName:       config.GetUniqueDriverName(),
+		DriverContainerName: "hostpath",
+		DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName(),
+			// This is needed for the
+			// testsuites/volumelimits.go `should support volume limits`
+			// test.
+			"--maxvolumespernode=10",
+		},
 		ProvisionerContainerName: "csi-provisioner",
 		SnapshotterContainerName: "csi-snapshotter",
 		NodeName:                 node.Name,
 	}
 	cleanup, err := utils.CreateFromManifests(config.Framework, driverNamespace, func(item interface{}) error {
-		return utils.PatchCSIDeployment(config.Framework, o, item)
+		if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil {
+			return err
+		}
+
+		// Remove csi-external-health-monitor-agent and
+		// csi-external-health-monitor-controller
+		// containers. They are not needed for any of the
+		// tests and in practice apparently caused enough
+		// overhead that even unrelated tests timed out. For
+		// example, in the pull-kubernetes-e2e-kind test, 43
+		// out of 5771 tests failed, including tests from
+		// sig-node, sig-cli, sig-api-machinery, sig-network.
+		switch item := item.(type) {
+		case *appsv1.StatefulSet:
+			var containers []v1.Container
+			for _, container := range item.Spec.Template.Spec.Containers {
+				switch container.Name {
+				case "csi-external-health-monitor-agent", "csi-external-health-monitor-controller":
+					// Remove these containers.
+				default:
+					// Keep the others.
+					containers = append(containers, container)
+				}
+			}
+			item.Spec.Template.Spec.Containers = containers
+		}
+		return nil
 	}, h.manifests...)
 
 	if err != nil {
@@ -408,7 +447,7 @@ func InitMockCSIDriver(driverOpts CSIMockDriverOpts) MockCSITestDriver {
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml",
-		"test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml",
 		"test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml",
 	}
@@ -611,7 +650,25 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*storageframework.P
 		FSGroupPolicy:     m.fsGroupPolicy,
 	}
 	cleanup, err := utils.CreateFromManifests(f, m.driverNamespace, func(item interface{}) error {
-		return utils.PatchCSIDeployment(f, o, item)
+		if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil {
+			return err
+		}
+
+		switch item := item.(type) {
+		case *rbacv1.ClusterRole:
+			if strings.HasPrefix(item.Name, "external-snapshotter-runner") {
+				// Re-enable access to secrets for the snapshotter sidecar for
+				// https://github.com/kubernetes/kubernetes/blob/6ede5ca95f78478fa627ecfea8136e0dff34436b/test/e2e/storage/csi_mock_volume.go#L1539-L1548
+				// It was disabled in https://github.com/kubernetes-csi/external-snapshotter/blob/501cc505846c03ee665355132f2da0ce7d5d747d/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml#L26-L32
+				item.Rules = append(item.Rules, rbacv1.PolicyRule{
+					APIGroups: []string{""},
+					Resources: []string{"secrets"},
+					Verbs:     []string{"get", "list"},
+				})
+			}
+		}
+
+		return nil
 	}, m.manifests...)
 
 	if err != nil {

test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml

@@ -1,3 +1,7 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-attacher/raw/v3.1.0/deploy/kubernetes//rbac.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
 # This YAML file contains all RBAC objects that are necessary to run external
 # CSI attacher.
 #
@@ -16,24 +20,21 @@ metadata:
   namespace: default
 
 ---
-# Attacher must be able to work with PVs, nodes and VolumeAttachments
+# Attacher must be able to work with PVs, CSINodes and VolumeAttachments
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: external-attacher-runner
 rules:
   - apiGroups: [""]
     resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
+    verbs: ["get", "list", "watch", "patch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["csinodes"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments"]
-    verbs: ["get", "list", "watch", "update", "patch"]
+    verbs: ["get", "list", "watch", "patch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments/status"]
     verbs: ["patch"]

test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-agent/rbac.yaml

@@ -0,0 +1,60 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-health-monitor/raw/v0.2.0/deploy/kubernetes/external-health-monitor-agent/rbac.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
+# This YAML file contains all RBAC objects that are necessary to run external
+# CSI health monitor agent.
+#
+# In production, each CSI driver deployment has to be customized:
+# - to avoid conflicts, use non-default namespace and different names
+#   for non-namespaced entities like the ClusterRole
+# - decide whether the deployment replicates the external CSI
+#   health monitor agent, in which case leadership election must be enabled;
+#   this influences the RBAC setup, see below
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-external-health-monitor-agent
+  # replace with non-default namespace name
+  namespace: default
+
+---
+# Health monitor agent must be able to work with PVs, PVCs, Nodes and Pods
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-health-monitor-agent-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-external-health-monitor-agent-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-agent
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-health-monitor-agent-runner
+  apiGroup: rbac.authorization.k8s.io
+
+

test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml

@@ -0,0 +1,89 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-health-monitor/raw/v0.2.0/deploy/kubernetes/external-health-monitor-controller/rbac.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
+# This YAML file contains all RBAC objects that are necessary to run external
+# CSI health monitor controller.
+#
+# In production, each CSI driver deployment has to be customized:
+# - to avoid conflicts, use non-default namespace and different names
+#   for non-namespaced entities like the ClusterRole
+# - decide whether the deployment replicates the external CSI
+#   health monitor controller, in which case leadership election must be enabled;
+#   this influences the RBAC setup, see below
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-external-health-monitor-controller
+  # replace with non-default namespace name
+  namespace: default
+
+---
+# Health monitor controller must be able to work with PVs, PVCs, Nodes and Pods
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-health-monitor-controller-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-external-health-monitor-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-controller
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-health-monitor-controller-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# Health monitor controller must be able to work with configmaps or leases in the current namespace
+# if (and only if) leadership election is enabled
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  # replace with non-default namespace name
+  namespace: default
+  name: external-health-monitor-controller-cfg
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-external-health-monitor-controller-role-cfg
+  # replace with non-default namespace name
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: csi-external-health-monitor-controller
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: Role
+  name: external-health-monitor-controller-cfg
+  apiGroup: rbac.authorization.k8s.io

test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml

@@ -1,3 +1,7 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-provisioner/raw/v2.1.1/deploy/kubernetes//rbac.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
 # This YAML file contains all RBAC objects that are necessary to run external
 # CSI provisioner.
 #
@@ -50,6 +54,13 @@ rules:
   - apiGroups: [""]
     resources: ["nodes"]
     verbs: ["get", "list", "watch"]
+  # Access to volumeattachments is only needed when the CSI driver
+  # has the PUBLISH_UNPUBLISH_VOLUME controller capability.
+  # In that case, external-provisioner will watch volumeattachments
+  # to determine when it is safe to delete a volume.
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch"]
 
 ---
 kind: ClusterRoleBinding
@@ -84,6 +95,21 @@ rules:
 - apiGroups: ["coordination.k8s.io"]
   resources: ["leases"]
   verbs: ["get", "watch", "list", "delete", "update", "create"]
+# Permissions for CSIStorageCapacity are only needed enabling the publishing
+# of storage capacity information.
+- apiGroups: ["storage.k8s.io"]
+  resources: ["csistoragecapacities"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+# The GET permissions below are needed for walking up the ownership chain
+# for CSIStorageCapacity. They are sufficient for deployment via
+# StatefulSet (only needs to get Pod) and Deployment (needs to get
+# Pod and then ReplicaSet to find the Deployment).
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get"]
+- apiGroups: ["apps"]
+  resources: ["replicasets"]
+  verbs: ["get"]
 
 ---
 kind: RoleBinding

test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml

@@ -1,3 +1,7 @@
+# Do not edit, downloaded from https://github.com/kubernetes-csi/external-resizer/raw/v1.1.0/deploy/kubernetes//rbac.yaml
+# for csi-driver-host-path v1.6.2
+# by test/e2e/testing-manifests/storage-csi/update-hostpath.sh
+#
 # This YAML file contains all RBAC objects that are necessary to run external
 # CSI resizer.
 #
@@ -29,13 +33,16 @@ rules:
   #   verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "update", "patch"]
+    verbs: ["get", "list", "watch", "patch"]
   - apiGroups: [""]
     resources: ["persistentvolumeclaims"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["persistentvolumeclaims/status"]
-    verbs: ["update", "patch"]
+    verbs: ["patch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]