From e8acbeac55b7f7987767370cd321c91900bb4b85 Mon Sep 17 00:00:00 2001 From: Omar Nahhas Sanchez Date: Mon, 8 Apr 2019 16:39:28 +0100 Subject: [PATCH] Adding arbitrary flags to core k8s components. (#1564) --- .../files/userdata/cloud-config-controller | 13 +++++- builtin/files/userdata/cloud-config-worker | 7 +++- pkg/api/kubernetes.go | 10 ++++- pkg/api/types.go | 15 ++++--- pkg/model/compiler.go | 9 +++-- pkg/model/config.go | 14 ++++--- pkg/model/node_pool_config.go | 3 +- pkg/model/stack_new.go | 6 +++ plugin/clusterextension/extras.go | 35 ++++++++++++++++ test/integration/plugin_test.go | 40 +++++++++++++++++++ 10 files changed, 129 insertions(+), 23 deletions(-) diff --git a/builtin/files/userdata/cloud-config-controller b/builtin/files/userdata/cloud-config-controller index e85f37b0e..925a91a16 100644 --- a/builtin/files/userdata/cloud-config-controller +++ b/builtin/files/userdata/cloud-config-controller @@ -366,9 +366,9 @@ coreos: {{ else }}--cluster-dns={{.DNSServiceIP}} \ {{ end }}--cluster-domain=cluster.local \ --cloud-provider=aws \ - {{if .ControllerFeatureGates.Enabled -}} + {{- if .ControllerFeatureGates.Enabled }} --feature-gates={{.ControllerFeatureGates.String}} \ - {{end -}}\ + {{- end }} {{- if .Kubelet.SystemReservedResources }} --system-reserved={{ .Kubelet.SystemReservedResources }} \ {{- end }} @@ -379,6 +379,9 @@ coreos: --node-ip=$$(curl http://169.254.169.254/latest/meta-data/local-ipv4) \ --max-pods=$$(/opt/bin/aws-k8s-cni-max-pods) \ {{- end }} + {{- range $f := .Kubelet.Flags}} + --{{$f.Name}}={{$f.Value}} \ + {{- end }} $KUBELET_OPTS \ " Restart=always @@ -2969,6 +2972,9 @@ write_files: apiVersion: {{if checkVersion ">=1.9" .K8sVer}}kubeproxy.config.k8s.io{{else}}componentconfig{{end}}/v1alpha1 kind: KubeProxyConfiguration bindAddress: 0.0.0.0 + {{range $flag,$value := .KubeProxy.Config -}} + {{$flag}}: {{$value}} + {{ end -}} clientConnection: kubeconfig: /etc/kubernetes/kubeconfig/kube-proxy.yaml clusterCIDR: {{.PodCIDR}} @@ -3347,6 +3353,9 @@ write_files: - scheduler - --kubeconfig=/etc/kubernetes/kubeconfig/kube-scheduler.yaml - --leader-elect=true + {{- range $f := .KubeSchedulerFlags }} + - --{{$f.Name}}={{$f.Value}} + {{- end }} {{- if .ControllerFeatureGates.Enabled }} - --feature-gates={{.ControllerFeatureGates.String}} {{- end }} diff --git a/builtin/files/userdata/cloud-config-worker b/builtin/files/userdata/cloud-config-worker index 1ff2a653d..1d57b87fa 100644 --- a/builtin/files/userdata/cloud-config-worker +++ b/builtin/files/userdata/cloud-config-worker @@ -388,9 +388,12 @@ coreos: --node-ip=$$(curl http://169.254.169.254/latest/meta-data/local-ipv4) \ --max-pods=$$(/opt/bin/aws-k8s-cni-max-pods) \ {{- end }} - {{if checkVersion "<1.10" .K8sVer -}} + {{- if checkVersion "<1.10" .K8sVer }} --require-kubeconfig \ - {{end -}} + {{- end }} + {{- range $f := .Kubelet.Flags }} + --{{$f.Name}}={{$f.Value}} \ + {{- end }} $KUBELET_OPTS" Restart=always RestartSec=10 diff --git a/pkg/api/kubernetes.go b/pkg/api/kubernetes.go index aeefeaffb..ae1a63373 100644 --- a/pkg/api/kubernetes.go +++ b/pkg/api/kubernetes.go @@ -5,8 +5,11 @@ type Kubernetes struct { EncryptionAtRest EncryptionAtRest `yaml:"encryptionAtRest"` Networking Networking `yaml:"networking,omitempty"` ControllerManager ControllerManager `yaml:"controllerManager,omitempty"` + KubeScheduler KubeScheduler `yaml:"kubeScheduler,omitempty"` + KubeProxy KubeProxy `yaml:"kubeProxy,omitempty"` + Kubelet Kubelet `yaml:"kubelet,omitempty"` + APIServer KubernetesAPIServer `yaml:"apiserver,omitempty"` - APIServer KubernetesAPIServer `yaml:"apiserver,omitempty"` // Manifests is a list of manifests to be installed to the cluster. // Note that the list is sorted by their names by kube-aws so that it won't result in unnecessarily node replacements. Manifests KubernetesManifests `yaml:"manifests,omitempty"` @@ -17,6 +20,11 @@ type ControllerManager struct { Flags CommandLineFlags `yaml:"flags,omitempty"` } +type KubeScheduler struct { + ComputeResources ComputeResources `yaml:"resources,omitempty"` + Flags CommandLineFlags `yaml:"flags,omitempty"` +} + type ComputeResources struct { Requests ResourceQuota `yaml:"requests,omitempty"` Limits ResourceQuota `yaml:"limits,omitempty"` diff --git a/pkg/api/types.go b/pkg/api/types.go index c509edd8e..2ae63e491 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -11,11 +11,12 @@ type Worker struct { // Kubelet options type Kubelet struct { - RotateCerts RotateCerts `yaml:"rotateCerts"` - SystemReservedResources string `yaml:"systemReserved"` - KubeReservedResources string `yaml:"kubeReserved"` - Kubeconfig string `yaml:"kubeconfig"` - Mounts []ContainerVolumeMount `yaml:"mounts"` + RotateCerts RotateCerts `yaml:"rotateCerts,omitempty"` + SystemReservedResources string `yaml:"systemReserved,omitempty"` + KubeReservedResources string `yaml:"kubeReserved,omitempty"` + Kubeconfig string `yaml:"kubeconfig,omitempty"` + Mounts []ContainerVolumeMount `yaml:"mounts,omitempty"` + Flags CommandLineFlags `yaml:"flags,omitempty"` } type Experimental struct { @@ -229,7 +230,9 @@ type TargetGroup struct { } type KubeProxy struct { - IPVSMode IPVSMode `yaml:"ipvsMode"` + IPVSMode IPVSMode `yaml:"ipvsMode"` + ComputeResources ComputeResources `yaml:"resources,omitempty"` + Config map[string]interface{} `yaml:"config,omitempty"` } type IPVSMode struct { diff --git a/pkg/model/compiler.go b/pkg/model/compiler.go index 4b06cef8b..3d9acbd3f 100644 --- a/pkg/model/compiler.go +++ b/pkg/model/compiler.go @@ -16,10 +16,11 @@ func Compile(cfgRef *api.Cluster, opts api.ClusterOptions) (*Config, error) { c.SetDefaults() config := Config{ - Cluster: c, - APIServerFlags: api.CommandLineFlags{}, - APIServerVolumes: api.APIServerVolumes{}, - ControllerFlags: api.CommandLineFlags{}, + Cluster: c, + APIServerFlags: api.CommandLineFlags{}, + APIServerVolumes: api.APIServerVolumes{}, + ControllerFlags: api.CommandLineFlags{}, + KubeSchedulerFlags: api.CommandLineFlags{}, } if c.AmiId == "" { diff --git a/pkg/model/config.go b/pkg/model/config.go index 62a2727ff..7352a9dec 100644 --- a/pkg/model/config.go +++ b/pkg/model/config.go @@ -2,14 +2,15 @@ package model import ( "fmt" + "path/filepath" + "strings" + "unicode/utf8" + "github.com/kubernetes-incubator/kube-aws/builtin" "github.com/kubernetes-incubator/kube-aws/gzipcompressor" "github.com/kubernetes-incubator/kube-aws/logger" "github.com/kubernetes-incubator/kube-aws/pkg/api" "github.com/kubernetes-incubator/kube-aws/provisioner" - "path/filepath" - "strings" - "unicode/utf8" ) const ( @@ -28,9 +29,10 @@ type Config struct { // This is used to simplify templating of the control-plane stack template. EtcdNodes []EtcdNode - APIServerVolumes api.APIServerVolumes - APIServerFlags api.CommandLineFlags - ControllerFlags api.CommandLineFlags + APIServerVolumes api.APIServerVolumes + APIServerFlags api.CommandLineFlags + ControllerFlags api.CommandLineFlags + KubeSchedulerFlags api.CommandLineFlags KubernetesManifestFiles []*provisioner.RemoteFile HelmReleaseFilesets []api.HelmReleaseFileset diff --git a/pkg/model/node_pool_config.go b/pkg/model/node_pool_config.go index 9d56632f3..67a1ae485 100644 --- a/pkg/model/node_pool_config.go +++ b/pkg/model/node_pool_config.go @@ -21,8 +21,7 @@ type NodePoolConfig struct { // APIEndpoint is the k8s api endpoint to which worker nodes in this node pool communicate APIEndpoint APIEndpoint api.UnknownKeys `yaml:",inline"` - - AMI string + AMI string } type MainClusterSettings struct { diff --git a/pkg/model/stack_new.go b/pkg/model/stack_new.go index f836a81ae..aadade5c8 100644 --- a/pkg/model/stack_new.go +++ b/pkg/model/stack_new.go @@ -82,6 +82,9 @@ func NewControlPlaneStack(conf *Config, opts api.StackTemplateOptions, extras cl conf.Kubelet.Mounts = append(conf.Kubelet.Mounts, extraController.KubeletVolumeMounts...) conf.APIServerFlags = append(conf.APIServerFlags, extraController.APIServerFlags...) conf.ControllerFlags = append(conf.ControllerFlags, extraController.ControllerFlags...) + conf.KubeSchedulerFlags = append(conf.KubeSchedulerFlags, extraController.KubeSchedulerFlags...) + conf.KubeProxy.Config = extraController.KubeProxyConfig + conf.Kubelet.Flags = append(conf.Kubelet.Flags, extraController.KubeletFlags...) conf.APIServerVolumes = append(conf.APIServerVolumes, extraController.APIServerVolumes...) conf.Controller.CustomSystemdUnits = append(conf.Controller.CustomSystemdUnits, extraController.SystemdUnits...) conf.Controller.CustomFiles = append(conf.Controller.CustomFiles, extraController.Files...) @@ -192,6 +195,7 @@ func NewEtcdStack(conf *Config, opts api.StackTemplateOptions, extras clusterext } func NewWorkerStack(conf *Config, npconf *NodePoolConfig, opts api.StackTemplateOptions, extras clusterextension.ClusterExtension, assetsConfig *credential.CompactAssets) (*Stack, error) { + return newStack( npconf.StackName(), conf, @@ -219,6 +223,8 @@ func NewWorkerStack(conf *Config, npconf *NodePoolConfig, opts api.StackTemplate if len(npconf.Kubelet.Kubeconfig) == 0 { npconf.Kubelet.Kubeconfig = extraWorker.Kubeconfig } + + npconf.Kubelet.Flags = conf.Kubelet.Flags npconf.Kubelet.Mounts = append(conf.Kubelet.Mounts, extraWorker.KubeletVolumeMounts...) npconf.CustomSystemdUnits = append(npconf.CustomSystemdUnits, extraWorker.SystemdUnits...) npconf.CustomFiles = append(npconf.CustomFiles, extraWorker.Files...) diff --git a/plugin/clusterextension/extras.go b/plugin/clusterextension/extras.go index ab76f9933..9dd3e23bd 100644 --- a/plugin/clusterextension/extras.go +++ b/plugin/clusterextension/extras.go @@ -83,6 +83,9 @@ type controller struct { APIServerFlags api.CommandLineFlags APIServerVolumes api.APIServerVolumes ControllerFlags api.CommandLineFlags + KubeProxyConfig map[string]interface{} + KubeSchedulerFlags api.CommandLineFlags + KubeletFlags api.CommandLineFlags CfnInitConfigSets map[string]interface{} Files []api.CustomFile SystemdUnits []api.CustomSystemdUnit @@ -281,6 +284,10 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er apiServerFlags := api.CommandLineFlags{} apiServerVolumes := api.APIServerVolumes{} controllerFlags := api.CommandLineFlags{} + kubeProxyConfig := map[string]interface{}{} + kubeletFlags := api.CommandLineFlags{} + kubeSchedulerFlags := api.CommandLineFlags{} + systemdUnits := []api.CustomSystemdUnit{} files := []api.CustomFile{} iamStatements := api.IAMPolicyStatements{} @@ -321,6 +328,31 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er } controllerFlags = append(controllerFlags, newFlag) } + for key, value := range p.Spec.Cluster.Kubernetes.KubeProxy.Config { + kubeProxyConfig[key] = value + } + for _, f := range p.Spec.Cluster.Kubernetes.KubeScheduler.Flags { + v, err := render.String(f.Value) + if err != nil { + return nil, fmt.Errorf("failed to load Kube Scheduler flags: %v", err) + } + newFlag := api.CommandLineFlag{ + Name: f.Name, + Value: v, + } + kubeSchedulerFlags = append(kubeSchedulerFlags, newFlag) + } + for _, f := range p.Spec.Cluster.Kubernetes.Kubelet.Flags { + v, err := render.String(f.Value) + if err != nil { + return nil, fmt.Errorf("failed to load kubelet flags: %v", err) + } + newFlag := api.CommandLineFlag{ + Name: f.Name, + Value: v, + } + kubeletFlags = append(kubeletFlags, newFlag) + } } apiServerVolumes = append(apiServerVolumes, p.Spec.Cluster.Kubernetes.APIServer.Volumes...) @@ -468,6 +500,9 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er ArchivedFiles: archivedFiles, APIServerFlags: apiServerFlags, ControllerFlags: controllerFlags, + KubeSchedulerFlags: kubeSchedulerFlags, + KubeProxyConfig: kubeProxyConfig, + KubeletFlags: kubeletFlags, APIServerVolumes: apiServerVolumes, Files: files, SystemdUnits: systemdUnits, diff --git a/test/integration/plugin_test.go b/test/integration/plugin_test.go index 15e0e5935..57047388f 100644 --- a/test/integration/plugin_test.go +++ b/test/integration/plugin_test.go @@ -143,6 +143,21 @@ spec: } } kubernetes: + controllerManager: + flags: + - name: "secure-port" + value: "11257" + kubelet: + flags: + - name: "healthz-bind-address" + value: "0.0.0.0" + kubeScheduler: + flags: + - name: "secure-port" + value: "11259" + kubeProxy: + config: + metricsBindAddress: "0.0.0.0" apiserver: flags: - name: "oidc-issuer-url" @@ -492,6 +507,31 @@ spec: if !strings.Contains(controllerUserdataS3Part, `--oidc-issuer-url=https://login.example.com/`) { t.Errorf("missing apiserver flag: --oidc-issuer-url=https://login.example.com/") } + + // A kube-aws plugin can add flags to the kube-controllermanager + if !strings.Contains(controllerUserdataS3Part, `--secure-port=11259`) { + t.Errorf("missing kube-controllermanager flag: --secure-port=11259") + } + + // A kube-aws plugin can add flags to the kubescheduler + if !strings.Contains(controllerUserdataS3Part, `- --secure-port=11259`) { + t.Errorf("missing kubescheduler flag: --secure-port=11259") + } + + // A kube-aws plugin can add flags to the kubelet in the controller + if !strings.Contains(controllerUserdataS3Part, `--healthz-bind-address=0.0.0.0`) { + t.Errorf("missing kubelet flag: --healthz-bind-address=0.0.0.0") + } + + // A kube-aws plugin can add flags to the kubelet in the workers + if !strings.Contains(workerUserdataS3Part, `--healthz-bind-address=0.0.0.0`) { + t.Errorf("missing kubelet flag: --healthz-bind-address=0.0.0.0") + } + + // A kube-aws plugin can add flags to the kubeproxy + if !strings.Contains(controllerUserdataS3Part, `metricsBindAddress: 0.0.0.0`) { + t.Errorf("missing kubeproxy config item: metricsBindAddress: 0.0.0.0") + } }, }, },