Namespace pod tolerations

pkg/constants/metadata.go

@@ -102,4 +102,12 @@ const (
 	// NamespacedConfigLabelKey is a label applied to configmaps to mark them as a configuration for all DevWorkspaces in
 	// the current namespace.
 	NamespacedConfigLabelKey = "controller.devfile.io/namespaced-config"
+
+	// NamespacePodTolerationsAnnotation is an annotation applied to a namespace to configure pod tolerations for all workspaces
+	// in that namespace. Value should be json-encoded []corev1.Toleration struct.
+	NamespacePodTolerationsAnnotation = "controller.devfile.io/pod-tolerations"
+
+	// NamespaceNodeSelectorAnnotation is an annotation applied to a namespace to configure the node selector for all workspaces
+	// in that namespace. Value should be json-encoded map[string]string
+	NamespaceNodeSelectorAnnotation = "controller.devfile.io/node-selector"
 )

pkg/provision/config/config.go

@@ -16,12 +16,14 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
 	"strings"
 
 	"github.com/devfile/devworkspace-operator/pkg/provision/sync"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/devfile/devworkspace-operator/pkg/constants"
@@ -72,3 +74,32 @@ func ReadNamespacedConfig(namespace string, api sync.ClusterAPI) (*NamespacedCon
 		CommonPVCSize: cm.Data[commonPVCSizeKey],
 	}, nil
 }
+
+// GetNamespacePodTolerationsAndNodeSelector gets pod tolerations and the node selector that should be applied to all pods created
+// for workspaces in a given namespace. Tolerations and node selector are unmarshalled from json-formatted annotations on the namespace
+// itself. Returns an error if annotations are not valid JSON.
+func GetNamespacePodTolerationsAndNodeSelector(namespace string, api sync.ClusterAPI) ([]corev1.Toleration, map[string]string, error) {
+	ns := &corev1.Namespace{}
+	err := api.Client.Get(api.Ctx, types.NamespacedName{Name: namespace}, ns)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var podTolerations []corev1.Toleration
+	podTolerationsAnnot, ok := ns.Annotations[constants.NamespacePodTolerationsAnnotation]
+	if ok && podTolerationsAnnot != "" {
+		if err := json.Unmarshal([]byte(podTolerationsAnnot), &podTolerations); err != nil {
+			return nil, nil, fmt.Errorf("failed to parse %s annotation: %w", constants.NamespacePodTolerationsAnnotation, err)
+		}
+	}
+
+	nodeSelector := map[string]string{}
+	nodeSelectorAnnot, ok := ns.Annotations[constants.NamespaceNodeSelectorAnnotation]
+	if ok && nodeSelectorAnnot != "" {
+		if err := json.Unmarshal([]byte(nodeSelectorAnnot), &nodeSelector); err != nil {
+			return nil, nil, fmt.Errorf("failed to parse %s annotation: %w", constants.NamespaceNodeSelectorAnnotation, err)
+		}
+	}
+
+	return podTolerations, nodeSelector, nil
+}

pkg/provision/storage/asyncstorage/deployment.go

@@ -17,6 +17,7 @@ package asyncstorage
 
 import (
 	"github.com/devfile/devworkspace-operator/internal/images"
+	nsconfig "github.com/devfile/devworkspace-operator/pkg/provision/config"
 	"github.com/devfile/devworkspace-operator/pkg/provision/sync"
 	wsprovision "github.com/devfile/devworkspace-operator/pkg/provision/workspace"
 	appsv1 "k8s.io/api/apps/v1"
@@ -27,7 +28,12 @@ import (
 )
 
 func SyncWorkspaceSyncDeploymentToCluster(namespace string, sshConfigMap *corev1.ConfigMap, storage *corev1.PersistentVolumeClaim, clusterAPI sync.ClusterAPI) (*appsv1.Deployment, error) {
-	specDeployment := getWorkspaceSyncDeploymentSpec(namespace, sshConfigMap, storage)
+	podTolerations, nodeSelector, err := nsconfig.GetNamespacePodTolerationsAndNodeSelector(namespace, clusterAPI)
+	if err != nil {
+		return nil, err
+	}
+
+	specDeployment := getWorkspaceSyncDeploymentSpec(namespace, sshConfigMap, storage, podTolerations, nodeSelector)
 	clusterObj, err := sync.SyncObjectWithCluster(specDeployment, clusterAPI)
 	switch err.(type) {
 	case nil:
@@ -47,7 +53,13 @@ func SyncWorkspaceSyncDeploymentToCluster(namespace string, sshConfigMap *corev1
 	return nil, NotReadyError
 }
 
-func getWorkspaceSyncDeploymentSpec(namespace string, sshConfigMap *corev1.ConfigMap, storage *corev1.PersistentVolumeClaim) *appsv1.Deployment {
+func getWorkspaceSyncDeploymentSpec(
+	namespace string,
+	sshConfigMap *corev1.ConfigMap,
+	storage *corev1.PersistentVolumeClaim,
+	tolerations []corev1.Toleration,
+	nodeSelector map[string]string) *appsv1.Deployment {
+
 	replicas := int32(1)
 	terminationGracePeriod := int64(1)
 	modeReadOnly := int32(0640)
@@ -140,6 +152,15 @@ func getWorkspaceSyncDeploymentSpec(namespace string, sshConfigMap *corev1.Confi
 			},
 		},
 	}
+
+	if tolerations != nil && len(tolerations) > 0 {
+		deployment.Spec.Template.Spec.Tolerations = tolerations
+	}
+
+	if nodeSelector != nil && len(nodeSelector) > 0 {
+		deployment.Spec.Template.Spec.NodeSelector = nodeSelector
+	}
+
 	return deployment
 }
 

pkg/provision/storage/cleanup.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
+	nsconfig "github.com/devfile/devworkspace-operator/pkg/provision/config"
 	"github.com/devfile/devworkspace-operator/pkg/provision/sync"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -106,6 +107,7 @@ func getSpecCommonPVCCleanupJob(workspace *dw.DevWorkspace, clusterAPI sync.Clus
 	if restrictedAccess, needsRestrictedAccess := workspace.Annotations[constants.DevWorkspaceRestrictedAccessAnnotation]; needsRestrictedAccess {
 		jobLabels[constants.DevWorkspaceRestrictedAccessAnnotation] = restrictedAccess
 	}
+
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      common.PVCCleanupJobName(workspaceId),
@@ -161,10 +163,20 @@ func getSpecCommonPVCCleanupJob(workspace *dw.DevWorkspace, clusterAPI sync.Clus
 		},
 	}
 
-	err := controllerutil.SetControllerReference(workspace, job, clusterAPI.Scheme)
+	podTolerations, nodeSelector, err := nsconfig.GetNamespacePodTolerationsAndNodeSelector(workspace.Namespace, clusterAPI)
 	if err != nil {
 		return nil, err
 	}
+	if podTolerations != nil && len(podTolerations) > 0 {
+		job.Spec.Template.Spec.Tolerations = podTolerations
+	}
+	if nodeSelector != nil && len(nodeSelector) > 0 {
+		job.Spec.Template.Spec.NodeSelector = nodeSelector
+	}
+
+	if err := controllerutil.SetControllerReference(workspace, job, clusterAPI.Scheme); err != nil {
+		return nil, err
+	}
 	return job, nil
 }
 

pkg/provision/workspace/deployment.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"strings"
 
+	nsconfig "github.com/devfile/devworkspace-operator/pkg/provision/config"
 	"github.com/devfile/devworkspace-operator/pkg/provision/sync"
 	"k8s.io/apimachinery/pkg/fields"
 
@@ -113,9 +114,20 @@ func SyncDeploymentToCluster(
 		envFromSourceAdditions = append(envFromSourceAdditions, automountEnv...)
 	}
 
+	podTolerations, nodeSelector, err := nsconfig.GetNamespacePodTolerationsAndNodeSelector(workspace.Namespace, clusterAPI)
+	if err != nil {
+		return DeploymentProvisioningStatus{
+			ProvisioningStatus{
+				Message:     "failed to read pod tolerations and node selector from namespace",
+				Err:         err,
+				FailStartup: true,
+			},
+		}
+	}
+
 	// [design] we have to pass components and routing pod additions separately because we need mountsources from each
 	// component.
-	specDeployment, err := getSpecDeployment(workspace, podAdditions, envFromSourceAdditions, saName, clusterAPI.Scheme)
+	specDeployment, err := getSpecDeployment(workspace, podAdditions, envFromSourceAdditions, saName, podTolerations, nodeSelector, clusterAPI.Scheme)
 	if err != nil {
 		return DeploymentProvisioningStatus{
 			ProvisioningStatus{
@@ -228,6 +240,8 @@ func getSpecDeployment(
 	podAdditionsList []v1alpha1.PodAdditions,
 	envFromSourceAdditions []corev1.EnvFromSource,
 	saName string,
+	podTolerations []corev1.Toleration,
+	nodeSelector map[string]string,
 	scheme *runtime.Scheme) (*appsv1.Deployment, error) {
 	replicas := int32(1)
 	terminationGracePeriod := int64(10)
@@ -294,6 +308,13 @@ func getSpecDeployment(
 		},
 	}
 
+	if podTolerations != nil && len(podTolerations) > 0 {
+		deployment.Spec.Template.Spec.Tolerations = podTolerations
+	}
+	if nodeSelector != nil && len(nodeSelector) > 0 {
+		deployment.Spec.Template.Spec.NodeSelector = nodeSelector
+	}
+
 	if needsPVCWorkaround(podAdditions) {
 		// Kubernetes creates directories in a PVC to support subpaths such that only the leaf directory has g+rwx permissions.
 		// This means that mounting the subpath e.g. <workspace-id>/plugins will result in the <workspace-id> directory being