Enable container level resource requests/limits via cluster spec (#252)

api/v1/storageoscluster_types.go

@@ -82,6 +82,8 @@ type StorageOSClusterSpec struct {
 
 	// Resources is to set the resource requirements of the storageos containers.
 	//+operator-sdk:csv:customresourcedefinitions:type=spec
+	// Deprecated: Set resource requests for individual containers via
+	// ContainerResources field in spec.
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 
 	// Disable Pod Fencing.  With StatefulSets, Pods are only re-scheduled if
@@ -173,6 +175,10 @@ type StorageOSClusterSpec struct {
 
 	// Disable StorageOS CLI deployment.
 	DisableCLI bool `json:"disableCLI,omitempty"`
+
+	// ContainerResources is to set the resource requirements of each individual
+	// container managed by the operator.
+	ContainerResources ContainerResources `json:"containerResources,omitempty"`
 }
 
 // ContainerImages contains image names of all the containers used by the operator.
@@ -197,6 +203,24 @@ type ContainerImages struct {
 	CLIContainer                       string `json:"cliContainer,omitempty"`
 }
 
+// ContainerResources contains resources of all the containers used by the operator.
+type ContainerResources struct {
+	NodeContainer                   corev1.ResourceRequirements `json:"nodeContainer,omitempty"`
+	InitContainer                   corev1.ResourceRequirements `json:"initContainer,omitempty"`
+	CSINodeDriverRegistrarContainer corev1.ResourceRequirements `json:"csiNodeDriverRegistrarContainer,omitempty"`
+	CSILivenessProbeContainer       corev1.ResourceRequirements `json:"csiLivenessProbeContainer,omitempty"`
+	CSIExternalProvisionerContainer corev1.ResourceRequirements `json:"csiExternalProvisionerContainer,omitempty"`
+	CSIExternalAttacherContainer    corev1.ResourceRequirements `json:"csiExternalAttacherContainer,omitempty"`
+	CSIExternalResizerContainer     corev1.ResourceRequirements `json:"csiExternalResizerContainer,omitempty"`
+	CSIExternalSnapshotterContainer corev1.ResourceRequirements `json:"csiExternalSnapshotterContainer,omitempty"`
+	KubeSchedulerContainer          corev1.ResourceRequirements `json:"kubeSchedulerContainer,omitempty"`
+	APIManagerContainer             corev1.ResourceRequirements `json:"apiManagerContainer,omitempty"`
+	NodeManagerContainer            corev1.ResourceRequirements `json:"nodeManagerContainer,omitempty"`
+	PortalManagerContainer          corev1.ResourceRequirements `json:"portalManagerContainer,omitempty"`
+	MetricsExporterContainer        corev1.ResourceRequirements `json:"metricsExporterContainer,omitempty"`
+	CLIContainer                    corev1.ResourceRequirements `json:"cliContainer,omitempty"`
+}
+
 // StorageOSClusterCSI contains CSI configurations.
 type StorageOSClusterCSI struct {
 	Enable                       bool            `json:"enable,omitempty"`

bundle/manifests/storageosoperator.clusterserviceversion.yaml

@@ -68,8 +68,9 @@ spec:
           node affinity requiredDuringSchedulingIgnoredDuringExecution.
         displayName: Node Selector Terms
         path: nodeSelectorTerms
-      - description: Resources is to set the resource requirements of the storageos
-          containers.
+      - description: 'Resources is to set the resource requirements of the storageos
+          containers. Deprecated: Set resource requests for individual containers
+          via ContainerResources field in spec.'
         displayName: Resources
         path: resources
       - description: SecretRefName is the name of the secret object that contains

config/manifests/bases/storageosoperator.clusterserviceversion.yaml

@@ -66,8 +66,9 @@ spec:
           node affinity requiredDuringSchedulingIgnoredDuringExecution.
         displayName: Node Selector Terms
         path: nodeSelectorTerms
-      - description: Resources is to set the resource requirements of the storageos
-          containers.
+      - description: 'Resources is to set the resource requirements of the storageos
+          containers. Deprecated: Set resource requests for individual containers
+          via ContainerResources field in spec.'
         displayName: Resources
         path: resources
       - description: SecretRefName is the name of the secret object that contains

controllers/storageoscluster/api-manager_operand.go

@@ -28,6 +28,9 @@ const (
 	// operand.
 	apiManagerPackage = "api-manager"
 
+	// API Manager container name.
+	apiManagerContainer = "api-manager"
+
 	// Kustomize image name for container image.
 	kImageAPIManager = "api-manager"
 
@@ -119,6 +122,11 @@ func getAPIManagerBuilder(fs filesys.FileSystem, obj client.Object, kcl kubectl.
 	// Create deployment transforms.
 	deploymentTransforms := []transform.TransformFunc{}
 
+	// Set resource requests and limits.
+	if cluster.Spec.ContainerResources.APIManagerContainer.Limits != nil || cluster.Spec.ContainerResources.APIManagerContainer.Requests != nil {
+		deploymentTransforms = append(deploymentTransforms, stransform.SetPodTemplateContainerResourceFunc(apiManagerContainer, cluster.Spec.ContainerResources.APIManagerContainer))
+	}
+
 	// Add secret volume transform.
 	apiSecretVolTF := stransform.SetPodTemplateSecretVolumeFunc("api-secret", cluster.Spec.SecretRefName, nil)
 

controllers/storageoscluster/cli_operand.go

@@ -30,6 +30,9 @@ const (
 	// Kustomize image name for container image.
 	kImageCLI = "cli"
 
+	// CLI container name.
+	cliContainer = "cli"
+
 	// Related image environment variable.
 	cliImageEnvVar = "RELATED_IMAGE_CLI"
 )
@@ -143,6 +146,11 @@ func getCliBuilder(fs filesys.FileSystem, obj client.Object, kcl kubectl.Kubectl
 	// CLI transforms.
 	cliTransforms := []transform.TransformFunc{}
 
+	// Set resource requests and limits.
+	if cluster.Spec.ContainerResources.CLIContainer.Limits != nil || cluster.Spec.ContainerResources.CLIContainer.Requests != nil {
+		cliTransforms = append(cliTransforms, stransform.SetPodTemplateContainerResourceFunc(cliContainer, cluster.Spec.ContainerResources.CLIContainer))
+	}
+
 	// Set the CLI deployment namespace.
 	namespaceTF := stransform.SetMetadataNamespaceFunc(cluster.GetNamespace())
 

controllers/storageoscluster/csi_operand.go

@@ -30,6 +30,12 @@ const (
 	// csiPackage contains the resource manifests for csi operand.
 	csiPackage = "csi"
 
+	// Name of each container as represented in deployment spec.
+	csiProvisionerContainer = "provisioner"
+	csiAttacherContainer    = "attacher"
+	csiResizerContainer     = "resizer"
+	csiSnapshotterContainer = "snapshotter"
+
 	// Kustomize image name for container image.
 	kImageCSIProvisioner = "csi-provisioner"
 	kImageCSIAttacher    = "csi-attacher"
@@ -230,6 +236,20 @@ func getCSIBuilder(fs filesys.FileSystem, obj client.Object, kcl kubectl.Kubectl
 		deploymentTransforms = append(deploymentTransforms, stransform.SetPodTemplateHostPathVolumeFunc("plugin-dir", filepath.Join(cluster.Spec.CSI.KubeletDir, "plugins_registry/storageos"), &dirOrCreate))
 	}
 
+	// Set resource requests and limits for individual containers.
+	if cluster.Spec.ContainerResources.CSIExternalProvisionerContainer.Limits != nil || cluster.Spec.ContainerResources.CSIExternalProvisionerContainer.Requests != nil {
+		deploymentTransforms = append(deploymentTransforms, stransform.SetPodTemplateContainerResourceFunc(csiProvisionerContainer, cluster.Spec.ContainerResources.CSIExternalProvisionerContainer))
+	}
+	if cluster.Spec.ContainerResources.CSIExternalAttacherContainer.Limits != nil || cluster.Spec.ContainerResources.CSIExternalAttacherContainer.Requests != nil {
+		deploymentTransforms = append(deploymentTransforms, stransform.SetPodTemplateContainerResourceFunc(csiAttacherContainer, cluster.Spec.ContainerResources.CSIExternalAttacherContainer))
+	}
+	if cluster.Spec.ContainerResources.CSIExternalResizerContainer.Limits != nil || cluster.Spec.ContainerResources.CSIExternalResizerContainer.Requests != nil {
+		deploymentTransforms = append(deploymentTransforms, stransform.SetPodTemplateContainerResourceFunc(csiResizerContainer, cluster.Spec.ContainerResources.CSIExternalResizerContainer))
+	}
+	if cluster.Spec.ContainerResources.CSIExternalSnapshotterContainer.Limits != nil || cluster.Spec.ContainerResources.CSIExternalSnapshotterContainer.Requests != nil {
+		deploymentTransforms = append(deploymentTransforms, stransform.SetPodTemplateContainerResourceFunc(csiSnapshotterContainer, cluster.Spec.ContainerResources.CSIExternalSnapshotterContainer))
+	}
+
 	return declarative.NewBuilder(csiPackage, fs,
 		declarative.WithManifestTransform(transform.ManifestTransform{
 			"csi/deployment.yaml": deploymentTransforms}),

controllers/storageoscluster/metrics-exporter_operand.go

@@ -211,6 +211,11 @@ func (c *MetricsExporterOperand) getMetricsExporterBuilder(
 	}
 	daemonsetTransforms = append(daemonsetTransforms, stransform.SetPodTemplateTolerationFunc(tolerations))
 
+	// Set resource requests and limits.
+	if cluster.Spec.ContainerResources.MetricsExporterContainer.Limits != nil || cluster.Spec.ContainerResources.MetricsExporterContainer.Requests != nil {
+		daemonsetTransforms = append(daemonsetTransforms, stransform.SetPodTemplateContainerResourceFunc(metricsExporterName, cluster.Spec.ContainerResources.MetricsExporterContainer))
+	}
+
 	patches, err := getConfigPatches(c.client.Scheme(), &cluster.Spec.Metrics)
 	if err != nil {
 		log.Error(err, "failed to create metrics-exporter configmap patches")

controllers/storageoscluster/node-manager_operand.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ondat/operator-toolkit/declarative"
 	"github.com/ondat/operator-toolkit/declarative/kubectl"
 	"github.com/ondat/operator-toolkit/declarative/kustomize"
+	"github.com/ondat/operator-toolkit/declarative/transform"
 	eventv1 "github.com/ondat/operator-toolkit/event/v1"
 	"github.com/ondat/operator-toolkit/operator/v1/operand"
 	appsv1 "k8s.io/api/apps/v1"
@@ -29,13 +30,17 @@ import (
 
 	storageoscomv1 "github.com/storageos/operator/api/v1"
 	"github.com/storageos/operator/internal/image"
+	stransform "github.com/storageos/operator/internal/transform"
 	"github.com/storageos/operator/watchers"
 )
 
 const (
 	// nodeManagerPackage contains the resource manifests for nodeManager operand.
 	nodeManagerPackage = "node-manager"
 
+	// Node Manager container name.
+	nodeManagerContainer = "manager"
+
 	// Kustomize image name for container image.
 	kImageKubeNodeManager = "controller"
 
@@ -366,7 +371,18 @@ func (c *NodeManagerOperand) getNodeManagerBuilder(fs filesys.FileSystem, obj cl
 		}
 	}
 
+	// Create deployment transforms.
+	deploymentTransforms := []transform.TransformFunc{}
+
+	// Set resource requests and limits.
+	if cluster.Spec.ContainerResources.NodeManagerContainer.Limits != nil || cluster.Spec.ContainerResources.NodeManagerContainer.Requests != nil {
+		deploymentTransforms = append(deploymentTransforms, stransform.SetPodTemplateContainerResourceFunc(nodeManagerContainer, cluster.Spec.ContainerResources.NodeManagerContainer))
+	}
+
 	return declarative.NewBuilder(nodeManagerPackage, fs,
+		declarative.WithManifestTransform(transform.ManifestTransform{
+			"node-manager/deployment-storageos-node-manager.yaml": deploymentTransforms,
+		}),
 		declarative.WithKustomizeMutationFunc(mutators),
 		declarative.WithKubectlClient(kcl),
 	)

controllers/storageoscluster/node_operand.go

@@ -37,6 +37,12 @@ const (
 	// initContainer is the name of the storageos init container.
 	initContainer = "init"
 
+	// csiLivenessProbeContainer is the name of the liveness probe container.
+	csiLivenessProbeContainer = "csi-liveness-probe"
+
+	// initContainer is the name of the storageos init container.
+	csiNodeDriverRegistrarContainer = "csi-driver-registrar"
+
 	// storageosService is the default name of the storageos service.
 	storageosService = "storageos"
 
@@ -265,9 +271,25 @@ func getNodeBuilder(fs filesys.FileSystem, obj client.Object, kcl kubectl.Kubect
 	daemonsetTransforms = append(daemonsetTransforms, stransform.SetPodTemplateTolerationFunc(tolerations))
 
 	// If any resources are defined, set container resource requirements.
+	// This field is now deprecated, but keep check to maintain backward compatibility.
 	if cluster.Spec.Resources.Limits != nil || cluster.Spec.Resources.Requests != nil {
 		daemonsetTransforms = append(daemonsetTransforms, stransform.SetPodTemplateContainerResourceFunc(storageosContainer, cluster.Spec.Resources))
 	}
+	// ContainerResources is now the preferred method of configuring resource settings.
+	// Set individual container resource requests and limits if specified.
+	// If set for node container, overwrite above 'Resources' setting.
+	if cluster.Spec.ContainerResources.NodeContainer.Limits != nil || cluster.Spec.ContainerResources.NodeContainer.Requests != nil {
+		daemonsetTransforms = append(daemonsetTransforms, stransform.SetPodTemplateContainerResourceFunc(storageosContainer, cluster.Spec.ContainerResources.NodeContainer))
+	}
+	if cluster.Spec.ContainerResources.InitContainer.Limits != nil || cluster.Spec.ContainerResources.InitContainer.Requests != nil {
+		daemonsetTransforms = append(daemonsetTransforms, stransform.SetPodTemplateContainerResourceFunc(initContainer, cluster.Spec.ContainerResources.InitContainer))
+	}
+	if cluster.Spec.ContainerResources.CSILivenessProbeContainer.Limits != nil || cluster.Spec.ContainerResources.CSILivenessProbeContainer.Requests != nil {
+		daemonsetTransforms = append(daemonsetTransforms, stransform.SetPodTemplateContainerResourceFunc(csiLivenessProbeContainer, cluster.Spec.ContainerResources.CSILivenessProbeContainer))
+	}
+	if cluster.Spec.ContainerResources.CSINodeDriverRegistrarContainer.Limits != nil || cluster.Spec.ContainerResources.CSINodeDriverRegistrarContainer.Requests != nil {
+		daemonsetTransforms = append(daemonsetTransforms, stransform.SetPodTemplateContainerResourceFunc(csiNodeDriverRegistrarContainer, cluster.Spec.ContainerResources.CSINodeDriverRegistrarContainer))
+	}
 
 	// Always set the kubelet registration path, using the default if none is defined in the cluster spec.
 	kubeletRegistrationPath := defaultKubeletRegPath

controllers/storageoscluster/portal-manager_operand.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ondat/operator-toolkit/declarative"
 	"github.com/ondat/operator-toolkit/declarative/kubectl"
 	"github.com/ondat/operator-toolkit/declarative/kustomize"
+	"github.com/ondat/operator-toolkit/declarative/transform"
 	eventv1 "github.com/ondat/operator-toolkit/event/v1"
 	"github.com/ondat/operator-toolkit/operator/v1/operand"
 	appsv1 "k8s.io/api/apps/v1"
@@ -21,12 +22,16 @@ import (
 
 	storageoscomv1 "github.com/storageos/operator/api/v1"
 	"github.com/storageos/operator/internal/image"
+	stransform "github.com/storageos/operator/internal/transform"
 )
 
 const (
 	// portalManagerPackage contains the resource manifests for portalManager operand.
 	portalManagerPackage = "portal-manager"
 
+	// Portal Manager container name.
+	portalManagerContainer = "manager"
+
 	// Kustomize image name for container image.
 	kImageKubePortalManager = "controller"
 
@@ -177,7 +182,18 @@ func getPortalManagerBuilder(fs filesys.FileSystem, obj client.Object, kcl kubec
 		images = append(images, *img)
 	}
 
+	// Create deployment transforms.
+	deploymentTransforms := []transform.TransformFunc{}
+
+	// Set resource requests and limits.
+	if cluster.Spec.ContainerResources.PortalManagerContainer.Limits != nil || cluster.Spec.ContainerResources.PortalManagerContainer.Requests != nil {
+		deploymentTransforms = append(deploymentTransforms, stransform.SetPodTemplateContainerResourceFunc(portalManagerContainer, cluster.Spec.ContainerResources.PortalManagerContainer))
+	}
+
 	return declarative.NewBuilder(portalManagerPackage, fs,
+		declarative.WithManifestTransform(transform.ManifestTransform{
+			"portal-manager/deployment-storageos-portal-manager.yaml": deploymentTransforms,
+		}),
 		declarative.WithKustomizeMutationFunc([]kustomize.MutateFunc{
 			kustomize.AddNamespace(cluster.GetNamespace()),
 			kustomize.AddImages(images),

controllers/storageoscluster/scheduler_operand.go

@@ -27,6 +27,9 @@ const (
 	// schedulerPackage contains the resource manifests for scheduler operand.
 	schedulerPackage = "scheduler"
 
+	// Scheduler container name.
+	kubeSchedulerContainer = "storageos-scheduler"
+
 	// Kustomize image name for container image.
 	kImageKubeScheduler = "kube-scheduler"
 
@@ -165,9 +168,18 @@ func getSchedulerBuilder(fs filesys.FileSystem, cluster *storageoscomv1.StorageO
 
 	configTransforms = append(configTransforms, rnsTF)
 
+	// Create deployment transforms.
+	deploymentTransforms := []transform.TransformFunc{}
+
+	// Set resource requests and limits.
+	if cluster.Spec.ContainerResources.KubeSchedulerContainer.Limits != nil || cluster.Spec.ContainerResources.KubeSchedulerContainer.Requests != nil {
+		deploymentTransforms = append(deploymentTransforms, stransform.SetPodTemplateContainerResourceFunc(kubeSchedulerContainer, cluster.Spec.ContainerResources.KubeSchedulerContainer))
+	}
+
 	return declarative.NewBuilder(schedulerPackage, fs,
 		declarative.WithManifestTransform(transform.ManifestTransform{
-			"scheduler/config.yaml": configTransforms,
+			"scheduler/config.yaml":     configTransforms,
+			"scheduler/deployment.yaml": deploymentTransforms,
 		}),
 		declarative.WithKustomizeMutationFunc([]kustomize.MutateFunc{
 			kustomize.AddNamespace(cluster.GetNamespace()),