Merge pull request #46254 from mtaufen/dkcfg

Automatic merge from submit-queue (batch tested with PRs 50016, 49583, 49930, 46254, 50337)

Alpha Dynamic Kubelet Configuration

Feature: kubernetes/enhancements#281

This proposal contains the alpha implementation of the Dynamic Kubelet Configuration feature proposed in ~#29459~ [community/contributors/design-proposals/dynamic-kubelet-configuration.md](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/dynamic-kubelet-configuration.md). 

Please note:
- ~The proposal doc is not yet up to date with this implementation, there are some subtle differences and some more significant ones. I will update the proposal doc to match by tomorrow afternoon.~
- ~This obviously needs more tests. I plan to write several O(soon). Since it's alpha and feature-gated, I'm decoupling this review from the review of the tests.~ I've beefed up the unit tests, though there is still plenty of testing to be done.
- ~I'm temporarily holding off on updating the generated docs, api specs, etc, for the sake of my reviewers 😄~ these files now live in a separate commit; the first commit is the one to review.

/cc @dchen1107 @vishh @bgrant0607 @thockin @derekwaynecarr 

```release-note
Adds (alpha feature) the ability to dynamically configure Kubelets by enabling the DynamicKubeletConfig feature gate, posting a ConfigMap to the API server, and setting the spec.configSource field on Node objects. See the proposal at https://github.com/kubernetes/community/blob/master/contributors/design-proposals/dynamic-kubelet-configuration.md for details.
```

api/openapi-spec/swagger.json

@@ -54865,6 +54865,29 @@
      }
     }
    },
+   "io.k8s.api.core.v1.NodeConfigSource": {
+    "description": "NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil.",
+    "properties": {
+     "apiVersion": {
+      "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources",
+      "type": "string"
+     },
+     "configMapRef": {
+      "$ref": "#/definitions/io.k8s.api.core.v1.ObjectReference"
+     },
+     "kind": {
+      "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds",
+      "type": "string"
+     }
+    },
+    "x-kubernetes-group-version-kind": [
+     {
+      "group": "",
+      "kind": "NodeConfigSource",
+      "version": "v1"
+     }
+    ]
+   },
    "io.k8s.api.core.v1.NodeDaemonEndpoints": {
     "description": "NodeDaemonEndpoints lists ports opened by daemons running on the Node.",
     "properties": {
@@ -54967,6 +54990,10 @@
    "io.k8s.api.core.v1.NodeSpec": {
     "description": "NodeSpec describes the attributes that a node is created with.",
     "properties": {
+     "configSource": {
+      "description": "If specified, the source to get node configuration from The DynamicKubeletConfig feature gate must be enabled for the Kubelet to use this field",
+      "$ref": "#/definitions/io.k8s.api.core.v1.NodeConfigSource"
+     },
      "externalID": {
       "description": "External ID of the node assigned by some machine database (e.g. a cloud provider). Deprecated.",
       "type": "string"

api/swagger-spec/v1.json

@@ -18318,6 +18318,10 @@
        "$ref": "v1.Taint"
       },
       "description": "If specified, the node's taints."
+     },
+     "configSource": {
+      "$ref": "v1.NodeConfigSource",
+      "description": "If specified, the source to get node configuration from The DynamicKubeletConfig feature gate must be enabled for the Kubelet to use this field"
      }
     }
    },
@@ -18347,6 +18351,23 @@
      }
     }
    },
+   "v1.NodeConfigSource": {
+    "id": "v1.NodeConfigSource",
+    "description": "NodeConfigSource specifies a source of node configuration. Exactly one subfield (excluding metadata) must be non-nil.",
+    "properties": {
+     "kind": {
+      "type": "string",
+      "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds"
+     },
+     "apiVersion": {
+      "type": "string",
+      "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources"
+     },
+     "configMapRef": {
+      "$ref": "v1.ObjectReference"
+     }
+    }
+   },
    "v1.NodeStatus": {
     "id": "v1.NodeStatus",
     "description": "NodeStatus is information about the current status of a node.",

cmd/hyperkube/kubelet.go

@@ -23,8 +23,12 @@ import (
 
 // NewKubelet creates a new hyperkube Server object that includes the
 // description and flags.
-func NewKubelet() *Server {
-	s := options.NewKubeletServer()
+func NewKubelet() (*Server, error) {
+	s, err := options.NewKubeletServer()
+	if err != nil {
+		return nil, err
+	}
+
 	hks := Server{
 		name:        "kubelet",
 		SimpleUsage: "kubelet",
@@ -39,5 +43,5 @@ func NewKubelet() *Server {
 		},
 	}
 	s.AddFlags(hks.Flags())
-	return &hks
+	return &hks, nil
 }

cmd/hyperkube/main.go

@@ -20,6 +20,7 @@ limitations under the License.
 package main
 
 import (
+	"fmt"
 	"os"
 
 	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
@@ -36,7 +37,12 @@ func main() {
 	hk.AddServer(NewKubeAPIServer())
 	hk.AddServer(NewKubeControllerManager())
 	hk.AddServer(NewScheduler())
-	hk.AddServer(NewKubelet())
+	if kubelet, err := NewKubelet(); err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	} else {
+		hk.AddServer(kubelet)
+	}
 	hk.AddServer(NewKubeProxy())
 	hk.AddServer(NewKubeAggregator())
 

cmd/kubelet/BUILD

@@ -23,10 +23,14 @@ go_library(
     deps = [
         "//cmd/kubelet/app:go_default_library",
         "//cmd/kubelet/app/options:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
         "//pkg/client/metrics/prometheus:go_default_library",
+        "//pkg/features:go_default_library",
+        "//pkg/kubelet/kubeletconfig:go_default_library",
         "//pkg/version/prometheus:go_default_library",
         "//pkg/version/verflag:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
     ],

cmd/kubelet/app/BUILD

@@ -52,6 +52,7 @@ go_library(
         "//pkg/kubelet/dockershim/remote:go_default_library",
         "//pkg/kubelet/eviction:go_default_library",
         "//pkg/kubelet/eviction/api:go_default_library",
+        "//pkg/kubelet/kubeletconfig:go_default_library",
         "//pkg/kubelet/network:go_default_library",
         "//pkg/kubelet/network/cni:go_default_library",
         "//pkg/kubelet/network/kubenet:go_default_library",
@@ -100,8 +101,6 @@ go_library(
         "//vendor/golang.org/x/exp/inotify:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",

cmd/kubelet/app/options/BUILD

@@ -19,6 +19,8 @@ go_library(
         "//pkg/apis/componentconfig:go_default_library",
         "//pkg/apis/componentconfig/install:go_default_library",
         "//pkg/apis/componentconfig/v1alpha1:go_default_library",
+        "//pkg/apis/componentconfig/validation:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/util/taints:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

cmd/kubelet/app/options/options.go

@@ -18,6 +18,7 @@ limitations under the License.
 package options
 
 import (
+	"fmt"
 	_ "net/http/pprof"
 	"strings"
 
@@ -26,9 +27,10 @@ import (
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
-	// Need to make sure the componentconfig api is installed so defaulting funcs work
-	_ "k8s.io/kubernetes/pkg/apis/componentconfig/install"
+	_ "k8s.io/kubernetes/pkg/apis/componentconfig/install" // Need to make sure the componentconfig api is installed so defaulting funcs work
 	"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
+	componentconfigvalidation "k8s.io/kubernetes/pkg/apis/componentconfig/validation"
+	"k8s.io/kubernetes/pkg/features"
 	utiltaints "k8s.io/kubernetes/pkg/util/taints"
 
 	"github.com/spf13/pflag"
@@ -79,6 +81,72 @@ type KubeletFlags struct {
 
 	// Container-runtime-specific options.
 	ContainerRuntimeOptions
+
+	// certDirectory is the directory where the TLS certs are located (by
+	// default /var/run/kubernetes). If tlsCertFile and tlsPrivateKeyFile
+	// are provided, this flag will be ignored.
+	CertDirectory string
+
+	// cloudProvider is the provider for cloud services.
+	// +optional
+	CloudProvider string
+
+	// cloudConfigFile is the path to the cloud provider configuration file.
+	// +optional
+	CloudConfigFile string
+
+	// rootDirectory is the directory path to place kubelet files (volume
+	// mounts,etc).
+	RootDirectory string
+
+	// The Kubelet will use this directory for checkpointing downloaded configurations and tracking configuration health.
+	// The Kubelet will create this directory if it does not already exist.
+	// The path may be absolute or relative; relative paths are under the Kubelet's current working directory.
+	// Providing this flag enables dynamic kubelet configuration.
+	// To use this flag, the DynamicKubeletConfig feature gate must be enabled.
+	DynamicConfigDir flag.StringFlag
+
+	// The Kubelet will look in this directory for an init configuration.
+	// The path may be absolute or relative; relative paths are under the Kubelet's current working directory.
+	// Omit this flag to use the combination of built-in default configuration values and flags.
+	// To use this flag, the DynamicKubeletConfig feature gate must be enabled.
+	InitConfigDir flag.StringFlag
+}
+
+// NewKubeletFlags will create a new KubeletFlags with default values
+func NewKubeletFlags() *KubeletFlags {
+	return &KubeletFlags{
+		// TODO(#41161:v1.10.0): Remove the default kubeconfig path and --require-kubeconfig.
+		RequireKubeConfig:       false,
+		KubeConfig:              flag.NewStringFlag("/var/lib/kubelet/kubeconfig"),
+		ContainerRuntimeOptions: *NewContainerRuntimeOptions(),
+		CertDirectory:           "/var/run/kubernetes",
+		CloudProvider:           v1alpha1.AutoDetectCloudProvider,
+		RootDirectory:           v1alpha1.DefaultRootDir,
+	}
+}
+
+func ValidateKubeletFlags(f *KubeletFlags) error {
+	// ensure that nobody sets DynamicConfigDir if the dynamic config feature gate is turned off
+	if f.DynamicConfigDir.Provided() && !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
+		return fmt.Errorf("the DynamicKubeletConfig feature gate must be enabled in order to use the --dynamic-config-dir flag")
+	}
+	// ensure that nobody sets InitConfigDir if the dynamic config feature gate is turned off
+	if f.InitConfigDir.Provided() && !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
+		return fmt.Errorf("the DynamicKubeletConfig feature gate must be enabled in order to use the --init-config-dir flag")
+	}
+	return nil
+}
+
+// NewKubeletConfiguration will create a new KubeletConfiguration with default values
+func NewKubeletConfiguration() (*componentconfig.KubeletConfiguration, error) {
+	versioned := &v1alpha1.KubeletConfiguration{}
+	api.Scheme.Default(versioned)
+	config := &componentconfig.KubeletConfiguration{}
+	if err := api.Scheme.Convert(versioned, config, nil); err != nil {
+		return nil, err
+	}
+	return config, nil
 }
 
 // KubeletServer encapsulates all of the parameters necessary for starting up
@@ -89,29 +157,33 @@ type KubeletServer struct {
 }
 
 // NewKubeletServer will create a new KubeletServer with default values.
-func NewKubeletServer() *KubeletServer {
-	versioned := &v1alpha1.KubeletConfiguration{}
-	api.Scheme.Default(versioned)
-	config := componentconfig.KubeletConfiguration{}
-	api.Scheme.Convert(versioned, &config, nil)
-	return &KubeletServer{
-		KubeletFlags: KubeletFlags{
-			// TODO(#41161:v1.10.0): Remove the default kubeconfig path and --require-kubeconfig.
-			RequireKubeConfig:       false,
-			KubeConfig:              flag.NewStringFlag("/var/lib/kubelet/kubeconfig"),
-			ContainerRuntimeOptions: *NewContainerRuntimeOptions(),
-		},
-		KubeletConfiguration: config,
+func NewKubeletServer() (*KubeletServer, error) {
+	config, err := NewKubeletConfiguration()
+	if err != nil {
+		return nil, err
 	}
+	return &KubeletServer{
+		KubeletFlags:         *NewKubeletFlags(),
+		KubeletConfiguration: *config,
+	}, nil
 }
 
-type kubeletConfiguration componentconfig.KubeletConfiguration
+// validateKubeletServer validates configuration of KubeletServer and returns an error if the input configuration is invalid
+func ValidateKubeletServer(s *KubeletServer) error {
+	// please add any KubeletConfiguration validation to the componentconfigvalidation.ValidateKubeletConfiguration function
+	if err := componentconfigvalidation.ValidateKubeletConfiguration(&s.KubeletConfiguration); err != nil {
+		return err
+	}
+	if err := ValidateKubeletFlags(&s.KubeletFlags); err != nil {
+		return err
+	}
+	return nil
+}
 
 // AddFlags adds flags for a specific KubeletServer to the specified FlagSet
 func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
-	var kc *kubeletConfiguration = (*kubeletConfiguration)(&s.KubeletConfiguration)
 	s.KubeletFlags.AddFlags(fs)
-	kc.addFlags(fs)
+	AddKubeletConfigFlags(fs, &s.KubeletConfiguration)
 }
 
 // AddFlags adds flags for a specific KubeletFlags to the specified FlagSet
@@ -140,10 +212,21 @@ func (f *KubeletFlags) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address of the node. If set, kubelet will use this IP address for the node")
 
 	fs.StringVar(&f.ProviderID, "provider-id", f.ProviderID, "Unique identifier for identifying the node in a machine database, i.e cloudprovider")
+
+	fs.StringVar(&f.CertDirectory, "cert-dir", f.CertDirectory, "The directory where the TLS certs are located. "+
+		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
+
+	fs.StringVar(&f.CloudProvider, "cloud-provider", f.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider.")
+	fs.StringVar(&f.CloudConfigFile, "cloud-config", f.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
+
+	fs.StringVar(&f.RootDirectory, "root-dir", f.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).")
+
+	fs.Var(&f.DynamicConfigDir, "dynamic-config-dir", "The Kubelet will use this directory for checkpointing downloaded configurations and tracking configuration health. The Kubelet will create this directory if it does not already exist. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Providing this flag enables dynamic Kubelet configuration. Presently, you must also enable the DynamicKubeletConfig feature gate to pass this flag.")
+	fs.Var(&f.InitConfigDir, "init-config-dir", "The Kubelet will look in this directory for the init configuration. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this argument to use the built-in default configuration values. Presently, you must also enable the DynamicKubeletConfig feature gate to pass this flag.")
 }
 
-// addFlags adds flags for a specific componentconfig.KubeletConfiguration to the specified FlagSet
-func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
+// AddKubeletConfigFlags adds flags for a specific componentconfig.KubeletConfiguration to the specified FlagSet
+func AddKubeletConfigFlags(fs *pflag.FlagSet, c *componentconfig.KubeletConfiguration) {
 	fs.BoolVar(&c.FailSwapOn, "fail-swap-on", true, "Makes the Kubelet fail to start if swap is enabled on the node. ")
 	fs.BoolVar(&c.FailSwapOn, "experimental-fail-swap-on", true, "DEPRECATED: please use --fail-swap-on instead.")
 	fs.MarkDeprecated("experimental-fail-swap-on", "This flag is deprecated and will be removed in future releases. please use --fail-swap-on instead.")
@@ -186,10 +269,7 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
 		"If --tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate and key "+
 		"are generated for the public address and saved to the directory passed to --cert-dir.")
 	fs.StringVar(&c.TLSPrivateKeyFile, "tls-private-key-file", c.TLSPrivateKeyFile, "File containing x509 private key matching --tls-cert-file.")
-	fs.StringVar(&c.CertDirectory, "cert-dir", c.CertDirectory, "The directory where the TLS certs are located. "+
-		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
 
-	fs.StringVar(&c.RootDirectory, "root-dir", c.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).")
 	fs.StringVar(&c.SeccompProfileRoot, "seccomp-profile-root", c.SeccompProfileRoot, "Directory path for seccomp profiles.")
 	fs.BoolVar(&c.AllowPrivileged, "allow-privileged", c.AllowPrivileged, "If true, allow containers to request privileged mode.")
 	fs.StringSliceVar(&c.HostNetworkSources, "host-network-sources", c.HostNetworkSources, "Comma-separated list of sources from which the Kubelet allows pods to use of host network.")
@@ -227,8 +307,6 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
 	fs.Int32Var(&c.ImageGCLowThresholdPercent, "image-gc-low-threshold", c.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to.")
 	fs.DurationVar(&c.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", c.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes.  To disable volume calculations, set to 0.")
 	fs.StringVar(&c.VolumePluginDir, "volume-plugin-dir", c.VolumePluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins")
-	fs.StringVar(&c.CloudProvider, "cloud-provider", c.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider.")
-	fs.StringVar(&c.CloudConfigFile, "cloud-config", c.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
 	fs.StringVar(&c.FeatureGates, "feature-gates", c.FeatureGates, "A set of key=value pairs that describe feature gates for alpha/experimental features. "+
 		"Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n"))