Skip to content

Commit

Permalink
AdAdd Trident Controller plugin pod scheduling options to trident ope…
Browse files Browse the repository at this point in the history 
…rator and helm chart

This commit adds options on the TridentOrchestrator CRD to specify nodeSelectors and tolerations for the Trident controller service pod.
  • Loading branch information
@ameade
ameade authored Jan 14, 2022
1 parent aa688c0 commit c7c167d
Show file tree
Hide file tree
Showing 7 changed files with 205 additions and 42 deletions.
40 changes: 21 additions & 19 deletions cli/k8s_client/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,25 +140,27 @@ type KubernetesClient interface {
}

type DeploymentYAMLArguments struct {
DeploymentName string `json:"deploymentName"`
TridentImage string `json:"tridentImage"`
AutosupportImage string `json:"autosupportImage"`
AutosupportProxy string `json:"autosupportProxy"`
AutosupportCustomURL string `json:"autosupportCustomURL"`
AutosupportSerialNumber string `json:"autosupportSerialNumber"`
AutosupportHostname string `json:"autosupportHostname"`
ImageRegistry string `json:"imageRegistry"`
LogFormat string `json:"logFormat"`
SnapshotCRDVersion string `json:"snapshotCRDVersion"`
ImagePullSecrets []string `json:"imagePullSecrets"`
Labels map[string]string `json:"labels"`
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
Debug bool `json:"debug"`
UseIPv6 bool `json:"useIPv6"`
SilenceAutosupport bool `json:"silenceAutosupport"`
Version *utils.Version `json:"version"`
TopologyEnabled bool `json:"topologyEnabled"`
HTTPRequestTimeout string `json:"httpRequestTimeout"`
DeploymentName string `json:"deploymentName"`
TridentImage string `json:"tridentImage"`
AutosupportImage string `json:"autosupportImage"`
AutosupportProxy string `json:"autosupportProxy"`
AutosupportCustomURL string `json:"autosupportCustomURL"`
AutosupportSerialNumber string `json:"autosupportSerialNumber"`
AutosupportHostname string `json:"autosupportHostname"`
ImageRegistry string `json:"imageRegistry"`
LogFormat string `json:"logFormat"`
SnapshotCRDVersion string `json:"snapshotCRDVersion"`
ImagePullSecrets []string `json:"imagePullSecrets"`
Labels map[string]string `json:"labels"`
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
Debug bool `json:"debug"`
UseIPv6 bool `json:"useIPv6"`
SilenceAutosupport bool `json:"silenceAutosupport"`
Version *utils.Version `json:"version"`
TopologyEnabled bool `json:"topologyEnabled"`
HTTPRequestTimeout string `json:"httpRequestTimeout"`
NodeSelector map[string]string `json:"nodeSelector"`
Tolerations []map[string]string `json:"tolerations"`
}

type DaemonsetYAMLArguments struct {
Expand Down
6 changes: 6 additions & 0 deletions cli/k8s_client/yaml_factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,8 @@ func GetCSIDeploymentYAML(args *DeploymentYAMLArguments) string {
deploymentYAML = replaceMultilineYAMLTag(deploymentYAML, "LABELS", constructLabels(args.Labels))
deploymentYAML = replaceMultilineYAMLTag(deploymentYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
deploymentYAML = replaceMultilineYAMLTag(deploymentYAML, "IMAGE_PULL_SECRETS", constructImagePullSecrets(args.ImagePullSecrets))
deploymentYAML = replaceMultilineYAMLTag(deploymentYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
deploymentYAML = replaceMultilineYAMLTag(deploymentYAML, "NODE_TOLERATIONS", constructTolerations(args.Tolerations))

return deploymentYAML
}
Expand Down Expand Up @@ -527,6 +529,8 @@ spec:
nodeSelector:
kubernetes.io/os: linux
kubernetes.io/arch: amd64
{NODE_SELECTOR}
{NODE_TOLERATIONS}
volumes:
- name: socket-dir
emptyDir:
Expand Down Expand Up @@ -682,6 +686,8 @@ spec:
nodeSelector:
kubernetes.io/os: linux
kubernetes.io/arch: amd64
{NODE_SELECTOR}
{NODE_TOLERATIONS}
volumes:
- name: socket-dir
emptyDir:
Expand Down
115 changes: 115 additions & 0 deletions cli/k8s_client/yaml_factory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,121 @@ func TestValidateGetCSIDeploymentYAMLFail(t *testing.T) {
}
}

func TestGetCSIDeploymentYAML(t *testing.T) {
versions := []string{"1.17.0", "1.18.0", "1.19.1", "1.21.0"}

for _, versionString := range versions {
version := utils.MustParseSemantic(versionString)
deploymentArgs := &DeploymentYAMLArguments{Version: version}

yamlData := GetCSIDeploymentYAML(deploymentArgs)
_, err := yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
t.Fatalf("expected valid YAML for version %s", versionString)
}
}
}

func TestGetCSIDeploymentYAML_NodeSelectors(t *testing.T) {
deploymentArgs := &DeploymentYAMLArguments{
NodeSelector: map[string]string{"foo": "bar"},
}
expectedNodeSelectorString := `
nodeSelector:
kubernetes.io/os: linux
kubernetes.io/arch: amd64
foo: bar
`

yamlData := GetCSIDeploymentYAML(deploymentArgs)
_, err := yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
t.Fatalf("expected valid YAML, got %s", yamlData)
}
assert.Contains(t, yamlData, expectedNodeSelectorString, fmt.Sprintf("expected nodeSelector in final YAML: %s", yamlData))

// Defaults
deploymentArgs = &DeploymentYAMLArguments{}
expectedNodeSelectorString = `
nodeSelector:
kubernetes.io/os: linux
kubernetes.io/arch: amd64
`

yamlData = GetCSIDeploymentYAML(deploymentArgs)
_, err = yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
t.Fatalf("expected valid YAML, got %s", yamlData)
}
assert.Contains(t, yamlData, expectedNodeSelectorString, fmt.Sprintf("expected nodeSelector in final YAML: %s", yamlData))

// Defaults used when empty list specified
deploymentArgs = &DeploymentYAMLArguments{
NodeSelector: map[string]string{},
}
expectedNodeSelectorString = `
nodeSelector:
kubernetes.io/os: linux
kubernetes.io/arch: amd64
`

yamlData = GetCSIDeploymentYAML(deploymentArgs)
_, err = yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
t.Fatalf("expected valid YAML, got %s", yamlData)
}
assert.Contains(t, yamlData, expectedNodeSelectorString, fmt.Sprintf("expected nodeSelector in final YAML: %s", yamlData))
}

func TestGetCSIDeploymentYAMLTolerations(t *testing.T) {
deploymentArgs := &DeploymentYAMLArguments{
Tolerations: []map[string]string{
{"key": "foo", "value": "bar", "operator": "Exists", "effect": "NoSchedule"},
{"key": "foo2", "value": "bar2", "operator": "Equals", "effect": "NoExecute", "tolerationSeconds": "20"},
},
}
expectedTolerationString := `
tolerations:
- key: "foo"
value: "bar"
effect: "NoSchedule"
operator: "Exists"
- key: "foo2"
value: "bar2"
effect: "NoExecute"
operator: "Equals"
tolerationSeconds: 20
`

yamlData := GetCSIDeploymentYAML(deploymentArgs)
_, err := yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
t.Fatalf("expected valid YAML, got %s", yamlData)
}
assert.Contains(t, yamlData, expectedTolerationString, fmt.Sprintf("expected toleration in final YAML: %s", yamlData))

// Test empty tolerations specified
deploymentArgs = &DeploymentYAMLArguments{Tolerations: []map[string]string{}}
expectedTolerationString = `
tolerations: []
`
defaultTolerationString := `
tolerations:
- effect: "NoExecute"
operator: "Exists"
- effect: "NoSchedule"
operator: "Exists"
`

yamlData = GetCSIDeploymentYAML(deploymentArgs)
_, err = yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
t.Fatalf("expected valid YAML, got %s", yamlData)
}
assert.Contains(t, yamlData, expectedTolerationString, fmt.Sprintf("expected toleration in final YAML: %s", yamlData))
assert.NotContains(t, yamlData, defaultTolerationString, fmt.Sprintf("expected default tolerations to not appear in final YAML: %s", yamlData))
}

func TestGetCSIDaemonSetYAML(t *testing.T) {
versions := []string{"1.17.0", "1.18.0", "1.21.0"}

Expand Down
14 changes: 14 additions & 0 deletions helm/trident-operator/templates/tridentorchestrator.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,20 @@ spec:
{{- toYaml . | nindent 2 }}
{{- end }}
enableNodePrep: {{ include "trident.enableNodePrep" $ }}
{{- if .Values.tridentControllerPluginNodeSelector }}
controllerPluginNodeSelector:
{{- range $key, $value := .Values.tridentControllerPluginNodeSelector }}
{{ $key }}: "{{ $value }}"
{{- end}}
{{- end }}
{{- if .Values.tridentControllerPluginTolerations }}
controllerPluginTolerations:
{{- range $value := .Values.tridentControllerPluginTolerations }}
- {{- range $k, $v := $value }}
{{ $k }}: "{{ $v }}"
{{- end}}
{{- end}}
{{- end }}
{{- if .Values.tridentNodePluginNodeSelector }}
nodePluginNodeSelector:
{{- range $key, $value := .Values.tridentNodePluginNodeSelector }}
Expand Down
6 changes: 6 additions & 0 deletions helm/trident-operator/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ tolerations: []
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
affinity: {}

# tridentControllerPluginNodeSelector additional nodeSelectors for the Pod running the Trident Controller CSI Plugin
# tridentControllerPluginNodeSelector : {}

# tridentControllerPluginTolerations overrides tolerations for the Pod running the Trident Controler CSI Plugin
# tridentControllerPluginTolerations: []

# tridentNodePluginNodeSelector additional nodeSelectors for Pods running the Trident Node CSI Plugin
# tridentNodePluginNodeSelector : {}

Expand Down
44 changes: 23 additions & 21 deletions operator/controllers/orchestrator/apis/netapp/v1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,27 +33,29 @@ type TridentOrchestratorList struct {

// TridentOrchestratorSpec defines the desired state of TridentOrchestrator
type TridentOrchestratorSpec struct {
Debug bool `json:"debug"`
Namespace string `json:"namespace"`
IPv6 bool `json:"IPv6,omitempty"`
K8sTimeout int `json:"k8sTimeout,omitempty"`
HTTPRequestTimeout string `json:"httpRequestTimeout,omitempty"`
SilenceAutosupport bool `json:"silenceAutosupport,omitempty"`
AutosupportImage string `json:"autosupportImage,omitempty"`
AutosupportProxy string `json:"autosupportProxy,omitempty"`
AutosupportSerialNumber string `json:"autosupportSerialNumber,omitempty"`
AutosupportHostname string `json:"autosupportHostname,omitempty"`
Uninstall bool `json:"uninstall,omitempty"`
LogFormat string `json:"logFormat,omitempty"`
ProbePort *int64 `json:"probePort,omitempty"`
TridentImage string `json:"tridentImage,omitempty"`
ImageRegistry string `json:"imageRegistry,omitempty"`
KubeletDir string `json:"kubeletDir,omitempty"`
Wipeout []string `json:"wipeout,omitempty"`
ImagePullSecrets []string `json:"imagePullSecrets,omitempty"`
EnableNodePrep bool `json:"enableNodePrep,omitempty"`
NodePluginNodeSelector map[string]string `json:"nodePluginNodeSelector,omitempty"`
NodePluginTolerations []Toleration `json:"nodePluginTolerations,omitempty"`
Debug bool `json:"debug"`
Namespace string `json:"namespace"`
IPv6 bool `json:"IPv6,omitempty"`
K8sTimeout int `json:"k8sTimeout,omitempty"`
HTTPRequestTimeout string `json:"httpRequestTimeout,omitempty"`
SilenceAutosupport bool `json:"silenceAutosupport,omitempty"`
AutosupportImage string `json:"autosupportImage,omitempty"`
AutosupportProxy string `json:"autosupportProxy,omitempty"`
AutosupportSerialNumber string `json:"autosupportSerialNumber,omitempty"`
AutosupportHostname string `json:"autosupportHostname,omitempty"`
Uninstall bool `json:"uninstall,omitempty"`
LogFormat string `json:"logFormat,omitempty"`
ProbePort *int64 `json:"probePort,omitempty"`
TridentImage string `json:"tridentImage,omitempty"`
ImageRegistry string `json:"imageRegistry,omitempty"`
KubeletDir string `json:"kubeletDir,omitempty"`
Wipeout []string `json:"wipeout,omitempty"`
ImagePullSecrets []string `json:"imagePullSecrets,omitempty"`
EnableNodePrep bool `json:"enableNodePrep,omitempty"`
ControllerPluginNodeSelector map[string]string `json:"controllerPluginNodeSelector,omitempty"`
ControllerPluginTolerations []Toleration `json:"controllerPluginTolerations,omitempty"`
NodePluginNodeSelector map[string]string `json:"nodePluginNodeSelector,omitempty"`
NodePluginTolerations []Toleration `json:"nodePluginTolerations,omitempty"`
}

// Toleration
Expand Down
22 changes: 20 additions & 2 deletions operator/controllers/orchestrator/installer/installer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,10 @@ var (
appLabelKey string
appLabelValue string

nodePluginNodeSelector map[string]string
nodePluginTolerations []netappv1.Toleration
controllerPluginNodeSelector map[string]string
controllerPluginTolerations []netappv1.Toleration
nodePluginNodeSelector map[string]string
nodePluginTolerations []netappv1.Toleration

CRDnames = []string{
BackendCRDName,
Expand Down Expand Up @@ -342,6 +344,12 @@ func (i *Installer) setInstallationParams(
appLabelKey = TridentCSILabelKey
appLabelValue = TridentCSILabelValue

if cr.Spec.ControllerPluginNodeSelector != nil {
controllerPluginNodeSelector = cr.Spec.ControllerPluginNodeSelector
}
if cr.Spec.ControllerPluginTolerations != nil {
controllerPluginTolerations = cr.Spec.ControllerPluginTolerations
}
if cr.Spec.NodePluginNodeSelector != nil {
nodePluginNodeSelector = cr.Spec.NodePluginNodeSelector
}
Expand Down Expand Up @@ -1047,6 +1055,14 @@ func (i *Installer) createOrPatchTridentDeployment(
createDeployment = true
}

var tolerations []map[string]string
if controllerPluginTolerations != nil {
tolerations = make([]map[string]string, 0)
for _, t := range controllerPluginTolerations {
tolerations = append(tolerations, t.GetMap())
}
}

if err = i.client.RemoveMultipleDeployments(unwantedDeployments); err != nil {
return fmt.Errorf("failed to remove unwanted Trident deployments; %v", err)
}
Expand All @@ -1073,6 +1089,8 @@ func (i *Installer) createOrPatchTridentDeployment(
Version: i.client.ServerVersion(),
TopologyEnabled: topologyEnabled,
HTTPRequestTimeout: httpTimeout,
NodeSelector: controllerPluginNodeSelector,
Tolerations: tolerations,
}

newDeploymentYAML := k8sclient.GetCSIDeploymentYAML(deploymentArgs)
Expand Down

0 comments on commit c7c167d

Please sign in to comment.