Adding arbitrary flags to core k8s components. (#1564)

builtin/files/userdata/cloud-config-controller

@@ -366,9 +366,9 @@ coreos:
         {{ else }}--cluster-dns={{.DNSServiceIP}} \
         {{ end }}--cluster-domain=cluster.local \
         --cloud-provider=aws \
-        {{if .ControllerFeatureGates.Enabled -}}
+        {{- if .ControllerFeatureGates.Enabled }}
         --feature-gates={{.ControllerFeatureGates.String}} \
-        {{end -}}\
+        {{- end }}
         {{- if .Kubelet.SystemReservedResources }}
         --system-reserved={{ .Kubelet.SystemReservedResources }} \
         {{- end }}
@@ -379,6 +379,9 @@ coreos:
         --node-ip=$$(curl http://169.254.169.254/latest/meta-data/local-ipv4) \
         --max-pods=$$(/opt/bin/aws-k8s-cni-max-pods) \
         {{- end }}
+        {{- range $f := .Kubelet.Flags}}
+        --{{$f.Name}}={{$f.Value}} \
+        {{- end }}
         $KUBELET_OPTS \
         "
         Restart=always
@@ -2969,6 +2972,9 @@ write_files:
           apiVersion: {{if checkVersion ">=1.9" .K8sVer}}kubeproxy.config.k8s.io{{else}}componentconfig{{end}}/v1alpha1
           kind: KubeProxyConfiguration
           bindAddress: 0.0.0.0
+          {{range $flag,$value := .KubeProxy.Config -}}
+          {{$flag}}: {{$value}}
+          {{ end -}}
           clientConnection:
             kubeconfig: /etc/kubernetes/kubeconfig/kube-proxy.yaml
           clusterCIDR: {{.PodCIDR}}
@@ -3347,6 +3353,9 @@ write_files:
           - scheduler
           - --kubeconfig=/etc/kubernetes/kubeconfig/kube-scheduler.yaml
           - --leader-elect=true
+          {{- range $f := .KubeSchedulerFlags }}
+          - --{{$f.Name}}={{$f.Value}}
+          {{- end }}
           {{- if .ControllerFeatureGates.Enabled }}
           - --feature-gates={{.ControllerFeatureGates.String}}
           {{- end }}

builtin/files/userdata/cloud-config-worker

@@ -388,9 +388,12 @@ coreos:
         --node-ip=$$(curl http://169.254.169.254/latest/meta-data/local-ipv4) \
         --max-pods=$$(/opt/bin/aws-k8s-cni-max-pods) \
         {{- end }}
-        {{if checkVersion "<1.10" .K8sVer -}}
+        {{- if checkVersion "<1.10" .K8sVer }}
         --require-kubeconfig \
-        {{end -}}
+        {{- end }}
+        {{- range $f := .Kubelet.Flags }}
+        --{{$f.Name}}={{$f.Value}} \
+        {{- end }}
         $KUBELET_OPTS"
         Restart=always
         RestartSec=10

pkg/api/kubernetes.go

@@ -5,8 +5,11 @@ type Kubernetes struct {
 	EncryptionAtRest  EncryptionAtRest         `yaml:"encryptionAtRest"`
 	Networking        Networking               `yaml:"networking,omitempty"`
 	ControllerManager ControllerManager        `yaml:"controllerManager,omitempty"`
+	KubeScheduler     KubeScheduler            `yaml:"kubeScheduler,omitempty"`
+	KubeProxy         KubeProxy                `yaml:"kubeProxy,omitempty"`
+	Kubelet           Kubelet                  `yaml:"kubelet,omitempty"`
+	APIServer         KubernetesAPIServer      `yaml:"apiserver,omitempty"`
 
-	APIServer KubernetesAPIServer `yaml:"apiserver,omitempty"`
 	// Manifests is a list of manifests to be installed to the cluster.
 	// Note that the list is sorted by their names by kube-aws so that it won't result in unnecessarily node replacements.
 	Manifests KubernetesManifests `yaml:"manifests,omitempty"`
@@ -17,6 +20,11 @@ type ControllerManager struct {
 	Flags            CommandLineFlags `yaml:"flags,omitempty"`
 }
 
+type KubeScheduler struct {
+	ComputeResources ComputeResources `yaml:"resources,omitempty"`
+	Flags            CommandLineFlags `yaml:"flags,omitempty"`
+}
+
 type ComputeResources struct {
 	Requests ResourceQuota `yaml:"requests,omitempty"`
 	Limits   ResourceQuota `yaml:"limits,omitempty"`

pkg/api/types.go

@@ -11,11 +11,12 @@ type Worker struct {
 
 // Kubelet options
 type Kubelet struct {
-	RotateCerts             RotateCerts            `yaml:"rotateCerts"`
-	SystemReservedResources string                 `yaml:"systemReserved"`
-	KubeReservedResources   string                 `yaml:"kubeReserved"`
-	Kubeconfig              string                 `yaml:"kubeconfig"`
-	Mounts                  []ContainerVolumeMount `yaml:"mounts"`
+	RotateCerts             RotateCerts            `yaml:"rotateCerts,omitempty"`
+	SystemReservedResources string                 `yaml:"systemReserved,omitempty"`
+	KubeReservedResources   string                 `yaml:"kubeReserved,omitempty"`
+	Kubeconfig              string                 `yaml:"kubeconfig,omitempty"`
+	Mounts                  []ContainerVolumeMount `yaml:"mounts,omitempty"`
+	Flags                   CommandLineFlags       `yaml:"flags,omitempty"`
 }
 
 type Experimental struct {
@@ -229,7 +230,9 @@ type TargetGroup struct {
 }
 
 type KubeProxy struct {
-	IPVSMode IPVSMode `yaml:"ipvsMode"`
+	IPVSMode         IPVSMode               `yaml:"ipvsMode"`
+	ComputeResources ComputeResources       `yaml:"resources,omitempty"`
+	Config           map[string]interface{} `yaml:"config,omitempty"`
 }
 
 type IPVSMode struct {

pkg/model/compiler.go

@@ -16,10 +16,11 @@ func Compile(cfgRef *api.Cluster, opts api.ClusterOptions) (*Config, error) {
 	c.SetDefaults()
 
 	config := Config{
-		Cluster:          c,
-		APIServerFlags:   api.CommandLineFlags{},
-		APIServerVolumes: api.APIServerVolumes{},
-		ControllerFlags:  api.CommandLineFlags{},
+		Cluster:            c,
+		APIServerFlags:     api.CommandLineFlags{},
+		APIServerVolumes:   api.APIServerVolumes{},
+		ControllerFlags:    api.CommandLineFlags{},
+		KubeSchedulerFlags: api.CommandLineFlags{},
 	}
 
 	if c.AmiId == "" {

pkg/model/config.go

@@ -2,14 +2,15 @@ package model
 
 import (
 	"fmt"
+	"path/filepath"
+	"strings"
+	"unicode/utf8"
+
 	"github.com/kubernetes-incubator/kube-aws/builtin"
 	"github.com/kubernetes-incubator/kube-aws/gzipcompressor"
 	"github.com/kubernetes-incubator/kube-aws/logger"
 	"github.com/kubernetes-incubator/kube-aws/pkg/api"
 	"github.com/kubernetes-incubator/kube-aws/provisioner"
-	"path/filepath"
-	"strings"
-	"unicode/utf8"
 )
 
 const (
@@ -28,9 +29,10 @@ type Config struct {
 	// This is used to simplify templating of the control-plane stack template.
 	EtcdNodes []EtcdNode
 
-	APIServerVolumes api.APIServerVolumes
-	APIServerFlags   api.CommandLineFlags
-	ControllerFlags  api.CommandLineFlags
+	APIServerVolumes   api.APIServerVolumes
+	APIServerFlags     api.CommandLineFlags
+	ControllerFlags    api.CommandLineFlags
+	KubeSchedulerFlags api.CommandLineFlags
 
 	KubernetesManifestFiles []*provisioner.RemoteFile
 	HelmReleaseFilesets     []api.HelmReleaseFileset

pkg/model/node_pool_config.go

@@ -21,8 +21,7 @@ type NodePoolConfig struct {
 	// APIEndpoint is the k8s api endpoint to which worker nodes in this node pool communicate
 	APIEndpoint     APIEndpoint
 	api.UnknownKeys `yaml:",inline"`
-
-	AMI string
+	AMI             string
 }
 
 type MainClusterSettings struct {

pkg/model/stack_new.go

@@ -82,6 +82,9 @@ func NewControlPlaneStack(conf *Config, opts api.StackTemplateOptions, extras cl
 			conf.Kubelet.Mounts = append(conf.Kubelet.Mounts, extraController.KubeletVolumeMounts...)
 			conf.APIServerFlags = append(conf.APIServerFlags, extraController.APIServerFlags...)
 			conf.ControllerFlags = append(conf.ControllerFlags, extraController.ControllerFlags...)
+			conf.KubeSchedulerFlags = append(conf.KubeSchedulerFlags, extraController.KubeSchedulerFlags...)
+			conf.KubeProxy.Config = extraController.KubeProxyConfig
+			conf.Kubelet.Flags = append(conf.Kubelet.Flags, extraController.KubeletFlags...)
 			conf.APIServerVolumes = append(conf.APIServerVolumes, extraController.APIServerVolumes...)
 			conf.Controller.CustomSystemdUnits = append(conf.Controller.CustomSystemdUnits, extraController.SystemdUnits...)
 			conf.Controller.CustomFiles = append(conf.Controller.CustomFiles, extraController.Files...)
@@ -192,6 +195,7 @@ func NewEtcdStack(conf *Config, opts api.StackTemplateOptions, extras clusterext
 }
 
 func NewWorkerStack(conf *Config, npconf *NodePoolConfig, opts api.StackTemplateOptions, extras clusterextension.ClusterExtension, assetsConfig *credential.CompactAssets) (*Stack, error) {
+
 	return newStack(
 		npconf.StackName(),
 		conf,
@@ -219,6 +223,8 @@ func NewWorkerStack(conf *Config, npconf *NodePoolConfig, opts api.StackTemplate
 			if len(npconf.Kubelet.Kubeconfig) == 0 {
 				npconf.Kubelet.Kubeconfig = extraWorker.Kubeconfig
 			}
+
+			npconf.Kubelet.Flags = conf.Kubelet.Flags
 			npconf.Kubelet.Mounts = append(conf.Kubelet.Mounts, extraWorker.KubeletVolumeMounts...)
 			npconf.CustomSystemdUnits = append(npconf.CustomSystemdUnits, extraWorker.SystemdUnits...)
 			npconf.CustomFiles = append(npconf.CustomFiles, extraWorker.Files...)

plugin/clusterextension/extras.go

@@ -83,6 +83,9 @@ type controller struct {
 	APIServerFlags      api.CommandLineFlags
 	APIServerVolumes    api.APIServerVolumes
 	ControllerFlags     api.CommandLineFlags
+	KubeProxyConfig     map[string]interface{}
+	KubeSchedulerFlags  api.CommandLineFlags
+	KubeletFlags        api.CommandLineFlags
 	CfnInitConfigSets   map[string]interface{}
 	Files               []api.CustomFile
 	SystemdUnits        []api.CustomSystemdUnit
@@ -281,6 +284,10 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er
 	apiServerFlags := api.CommandLineFlags{}
 	apiServerVolumes := api.APIServerVolumes{}
 	controllerFlags := api.CommandLineFlags{}
+	kubeProxyConfig := map[string]interface{}{}
+	kubeletFlags := api.CommandLineFlags{}
+	kubeSchedulerFlags := api.CommandLineFlags{}
+
 	systemdUnits := []api.CustomSystemdUnit{}
 	files := []api.CustomFile{}
 	iamStatements := api.IAMPolicyStatements{}
@@ -321,6 +328,31 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er
 					}
 					controllerFlags = append(controllerFlags, newFlag)
 				}
+				for key, value := range p.Spec.Cluster.Kubernetes.KubeProxy.Config {
+					kubeProxyConfig[key] = value
+				}
+				for _, f := range p.Spec.Cluster.Kubernetes.KubeScheduler.Flags {
+					v, err := render.String(f.Value)
+					if err != nil {
+						return nil, fmt.Errorf("failed to load Kube Scheduler flags: %v", err)
+					}
+					newFlag := api.CommandLineFlag{
+						Name:  f.Name,
+						Value: v,
+					}
+					kubeSchedulerFlags = append(kubeSchedulerFlags, newFlag)
+				}
+				for _, f := range p.Spec.Cluster.Kubernetes.Kubelet.Flags {
+					v, err := render.String(f.Value)
+					if err != nil {
+						return nil, fmt.Errorf("failed to load kubelet flags: %v", err)
+					}
+					newFlag := api.CommandLineFlag{
+						Name:  f.Name,
+						Value: v,
+					}
+					kubeletFlags = append(kubeletFlags, newFlag)
+				}
 			}
 
 			apiServerVolumes = append(apiServerVolumes, p.Spec.Cluster.Kubernetes.APIServer.Volumes...)
@@ -468,6 +500,9 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er
 		ArchivedFiles:           archivedFiles,
 		APIServerFlags:          apiServerFlags,
 		ControllerFlags:         controllerFlags,
+		KubeSchedulerFlags:      kubeSchedulerFlags,
+		KubeProxyConfig:         kubeProxyConfig,
+		KubeletFlags:            kubeletFlags,
 		APIServerVolumes:        apiServerVolumes,
 		Files:                   files,
 		SystemdUnits:            systemdUnits,

test/integration/plugin_test.go

@@ -143,6 +143,21 @@ spec:
                 }
               }
     kubernetes:
+      controllerManager:
+        flags:
+          - name: "secure-port"
+            value: "11257"
+      kubelet:
+        flags:
+          - name: "healthz-bind-address"
+            value: "0.0.0.0"
+      kubeScheduler:
+        flags:
+          - name: "secure-port"
+            value: "11259"
+      kubeProxy:
+        config:
+          metricsBindAddress: "0.0.0.0"
       apiserver:
         flags:
         - name: "oidc-issuer-url"
@@ -492,6 +507,31 @@ spec:
 					if !strings.Contains(controllerUserdataS3Part, `--oidc-issuer-url=https://login.example.com/`) {
 						t.Errorf("missing apiserver flag: --oidc-issuer-url=https://login.example.com/")
 					}
+
+					// A kube-aws plugin can add flags to the kube-controllermanager
+					if !strings.Contains(controllerUserdataS3Part, `--secure-port=11259`) {
+						t.Errorf("missing kube-controllermanager flag: --secure-port=11259")
+					}
+
+					// A kube-aws plugin can add flags to the kubescheduler
+					if !strings.Contains(controllerUserdataS3Part, `- --secure-port=11259`) {
+						t.Errorf("missing kubescheduler flag: --secure-port=11259")
+					}
+
+					// A kube-aws plugin can add flags to the kubelet in the controller
+					if !strings.Contains(controllerUserdataS3Part, `--healthz-bind-address=0.0.0.0`) {
+						t.Errorf("missing kubelet flag: --healthz-bind-address=0.0.0.0")
+					}
+
+					// A kube-aws plugin can add flags to the kubelet in the workers
+					if !strings.Contains(workerUserdataS3Part, `--healthz-bind-address=0.0.0.0`) {
+						t.Errorf("missing kubelet flag: --healthz-bind-address=0.0.0.0")
+					}
+
+					// A kube-aws plugin can add flags to the kubeproxy
+					if !strings.Contains(controllerUserdataS3Part, `metricsBindAddress: 0.0.0.0`) {
+						t.Errorf("missing kubeproxy config item: metricsBindAddress: 0.0.0.0")
+					}
 				},
 			},
 		},