Merge pull request kubernetes#64210 from luxas/kubeadm_kubelet_extraargs

Automatic merge from submit-queue (batch tested with PRs 64322, 64210, 64458, 64232, 64370). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubeadm: Move .NodeName and .CRISocket to a common sub-struct

**What this PR does / why we need it**:
Regroups some common fields for `kubeadm init` and `kubeadm join` only used for the initial node registration.
Lets the user specify ExtraArgs to the kubelet.
Now also runs the dynamic env file creation for `kubeadm join`.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes kubernetes/kubeadm#847
Follows-up kubernetes#63887
Related to kubernetes/kubeadm#822

**Special notes for your reviewer**: WIP, but please review so we can finalize the direction of the PR

**Release note**:

```release-note
[action required] `.NodeName` and `.CRISocket` in the `MasterConfiguration` and `NodeConfiguration` v1alpha1 API objects are now `.NodeRegistration.Name` and `.NodeRegistration.CRISocket` respectively in the v1alpha2 API. The `.NoTaintMaster` field has been removed in the v1alpha2 API.
```
@kubernetes/sig-cluster-lifecycle-pr-reviews @liztio

cmd/kubeadm/app/apis/kubeadm/fuzzer/BUILD

@@ -16,6 +16,7 @@ go_library(
         "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
         "//pkg/util/pointer:go_default_library",
         "//vendor/github.com/google/gofuzz:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
     ],

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -21,6 +21,7 @@ import (
 
 	fuzz "github.com/google/gofuzz"
 
+	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
@@ -43,7 +44,6 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.APIServerCertSANs = []string{"foo"}
 
 			obj.Token = "foo"
-			obj.CRISocket = "foo"
 			obj.TokenTTL = &metav1.Duration{Duration: 1 * time.Hour}
 			obj.TokenUsages = []string{"foo"}
 			obj.TokenGroups = []string{"foo"}
@@ -59,16 +59,21 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 				MountPath: "foo",
 				Writable:  false,
 			}}
+			// Note: We don't set values here for obj.Etcd.External, as these are mutually exlusive.
+			// And to make sure the fuzzer doesn't set a random value for obj.Etcd.External, we let
+			// kubeadmapi.Etcd implement fuzz.Interface (we handle that ourselves)
 			obj.Etcd.Local = &kubeadm.LocalEtcd{
 				Image:          "foo",
 				DataDir:        "foo",
 				ServerCertSANs: []string{"foo"},
 				PeerCertSANs:   []string{"foo"},
 				ExtraArgs:      map[string]string{"foo": "foo"},
 			}
-			// Note: We don't set values here for obj.Etcd.External, as these are mutually exlusive.
-			// And to make sure the fuzzer doesn't set a random value for obj.Etcd.External, we let
-			// kubeadmapi.Etcd implement fuzz.Interface (we handle that ourselves)
+			obj.NodeRegistration = kubeadm.NodeRegistrationOptions{
+				CRISocket: "foo",
+				Name:      "foo",
+				Taints:    []v1.Taint{},
+			}
 			obj.KubeletConfiguration = kubeadm.KubeletConfiguration{
 				BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{
 					StaticPodPath: "foo",
@@ -139,8 +144,11 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.DiscoveryTimeout = &metav1.Duration{Duration: 1}
 			obj.TLSBootstrapToken = "foo"
 			obj.Token = "foo"
-			obj.CRISocket = "foo"
 			obj.ClusterName = "foo"
+			obj.NodeRegistration = kubeadm.NodeRegistrationOptions{
+				CRISocket: "foo",
+				Name:      "foo",
+			}
 		},
 	}
 }

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -44,13 +44,9 @@ type MasterConfiguration struct {
 	Networking Networking
 	// KubernetesVersion is the target version of the control plane.
 	KubernetesVersion string
-	// NodeName is the name of the node that will host the k8s control plane.
-	// Defaults to the hostname if not provided.
-	NodeName string
-	// NoTaintMaster will, if set, suppress the tainting of the
-	// master node allowing workloads to be run on it (e.g. in
-	// single node configurations).
-	NoTaintMaster bool
+
+	// NodeRegistration holds fields that relate to registering the new master node to the cluster
+	NodeRegistration NodeRegistrationOptions
 
 	// Token is used for establishing bidirectional trust between nodes and masters.
 	// Used for joining nodes in the cluster.
@@ -62,9 +58,6 @@ type MasterConfiguration struct {
 	// Extra groups that this token will authenticate as when used for authentication
 	TokenGroups []string
 
-	// CRISocket is used to retrieve container runtime info.
-	CRISocket string
-
 	// APIServerExtraArgs is a set of extra flags to pass to the API Server or override
 	// default ones in form of <flagname>=<value>.
 	// TODO: This is temporary and ideally we would like to switch all components to
@@ -138,6 +131,28 @@ type API struct {
 	BindPort int32
 }
 
+// NodeRegistrationOptions holds fields that relate to registering a new master or node to the cluster, either via "kubeadm init" or "kubeadm join"
+type NodeRegistrationOptions struct {
+
+	// Name is the `.Metadata.Name` field of the Node API object that will be created in this `kubeadm init` or `kubeadm joiń` operation.
+	// This field is also used in the CommonName field of the kubelet's client certificate to the API server.
+	// Defaults to the hostname of the node if not provided.
+	Name string
+
+	// CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use
+	CRISocket string
+
+	// Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process
+	// it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your master node, set this field to an
+	// empty slice, i.e. `taints: {}` in the YAML file. This field is solely used for Node registration.
+	Taints []v1.Taint
+
+	// ExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file
+	// kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap
+	// Flags have higher higher priority when parsing. These values are local and specific to the node kubeadm is executing on.
+	ExtraArgs map[string]string
+}
+
 // TokenDiscovery contains elements needed for token discovery.
 type TokenDiscovery struct {
 	// ID is the first part of a bootstrap token. Considered public information.
@@ -223,6 +238,9 @@ type ExternalEtcd struct {
 type NodeConfiguration struct {
 	metav1.TypeMeta
 
+	// NodeRegistration holds fields that relate to registering the new master node to the cluster
+	NodeRegistration NodeRegistrationOptions
+
 	// CACertPath is the path to the SSL certificate authority used to
 	// secure comunications between node and master.
 	// Defaults to "/etc/kubernetes/pki/ca.crt".
@@ -239,16 +257,11 @@ type NodeConfiguration struct {
 	DiscoveryTokenAPIServers []string
 	// DiscoveryTimeout modifies the discovery timeout
 	DiscoveryTimeout *metav1.Duration
-	// NodeName is the name of the node to join the cluster. Defaults
-	// to the name of the host.
-	NodeName string
 	// TLSBootstrapToken is a token used for TLS bootstrapping.
 	// Defaults to Token.
 	TLSBootstrapToken string
 	// Token is used for both discovery and TLS bootstrapping.
 	Token string
-	// CRISocket is used to retrieve container runtime info.
-	CRISocket string
 	// The cluster name
 	ClusterName string
 
@@ -332,13 +345,13 @@ type CommonConfiguration interface {
 // GetCRISocket will return the CRISocket that is defined for the MasterConfiguration.
 // This is used internally to deduplicate the kubeadm preflight checks.
 func (cfg *MasterConfiguration) GetCRISocket() string {
-	return cfg.CRISocket
+	return cfg.NodeRegistration.CRISocket
 }
 
 // GetNodeName will return the NodeName that is defined for the MasterConfiguration.
 // This is used internally to deduplicate the kubeadm preflight checks.
 func (cfg *MasterConfiguration) GetNodeName() string {
-	return cfg.NodeName
+	return cfg.NodeRegistration.Name
 }
 
 // GetKubernetesVersion will return the KubernetesVersion that is defined for the MasterConfiguration.
@@ -350,13 +363,13 @@ func (cfg *MasterConfiguration) GetKubernetesVersion() string {
 // GetCRISocket will return the CRISocket that is defined for the NodeConfiguration.
 // This is used internally to deduplicate the kubeadm preflight checks.
 func (cfg *NodeConfiguration) GetCRISocket() string {
-	return cfg.CRISocket
+	return cfg.NodeRegistration.CRISocket
 }
 
 // GetNodeName will return the NodeName that is defined for the NodeConfiguration.
 // This is used internally to deduplicate the kubeadm preflight checks.
 func (cfg *NodeConfiguration) GetNodeName() string {
-	return cfg.NodeName
+	return cfg.NodeRegistration.Name
 }
 
 // GetKubernetesVersion will return an empty string since KubernetesVersion is not a

cmd/kubeadm/app/apis/kubeadm/v1alpha1/conversion.go

@@ -20,15 +20,20 @@ import (
 	"reflect"
 	"strings"
 
+	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/conversion"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
 func addConversionFuncs(scheme *runtime.Scheme) error {
 	// Add non-generated conversion functions
 	err := scheme.AddConversionFuncs(
 		Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration,
+		Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration,
+		Convert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration,
+		Convert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration,
 		Convert_v1alpha1_Etcd_To_kubeadm_Etcd,
 		Convert_kubeadm_Etcd_To_v1alpha1_Etcd,
 	)
@@ -39,19 +44,35 @@ func addConversionFuncs(scheme *runtime.Scheme) error {
 	return nil
 }
 
+// Upgrades below
+
 func Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in *MasterConfiguration, out *kubeadm.MasterConfiguration, s conversion.Scope) error {
 	if err := autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in, out, s); err != nil {
 		return err
 	}
 
 	UpgradeCloudProvider(in, out)
 	UpgradeAuthorizationModes(in, out)
+	UpgradeNodeRegistrationOptionsForMaster(in, out)
 	// We don't support migrating information from the .PrivilegedPods field which was removed in v1alpha2
 	// We don't support migrating information from the .ImagePullPolicy field which was removed in v1alpha2
 
 	return nil
 }
 
+func Convert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration(in *NodeConfiguration, out *kubeadm.NodeConfiguration, s conversion.Scope) error {
+	if err := autoConvert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration(in, out, s); err != nil {
+		return err
+	}
+
+	// .NodeName has moved to .NodeRegistration.Name
+	out.NodeRegistration.Name = in.NodeName
+	// .CRISocket has moved to .NodeRegistration.CRISocket
+	out.NodeRegistration.CRISocket = in.CRISocket
+
+	return nil
+}
+
 func Convert_v1alpha1_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conversion.Scope) error {
 	if err := autoConvert_v1alpha1_Etcd_To_kubeadm_Etcd(in, out, s); err != nil {
 		return err
@@ -123,3 +144,45 @@ func UpgradeAuthorizationModes(in *MasterConfiguration, out *kubeadm.MasterConfi
 		out.APIServerExtraArgs["authorization-mode"] = strings.Join(in.AuthorizationModes, ",")
 	}
 }
+
+func UpgradeNodeRegistrationOptionsForMaster(in *MasterConfiguration, out *kubeadm.MasterConfiguration) {
+	// .NodeName has moved to .NodeRegistration.Name
+	out.NodeRegistration.Name = in.NodeName
+
+	// .CRISocket has moved to .NodeRegistration.CRISocket
+	out.NodeRegistration.CRISocket = in.CRISocket
+
+	// Transfer the information from .NoTaintMaster to the new layout
+	if in.NoTaintMaster {
+		out.NodeRegistration.Taints = []v1.Taint{}
+	} else {
+		out.NodeRegistration.Taints = []v1.Taint{constants.MasterTaint}
+	}
+}
+
+// Downgrades below
+
+func Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in *kubeadm.MasterConfiguration, out *MasterConfiguration, s conversion.Scope) error {
+	if err := autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in, out, s); err != nil {
+		return err
+	}
+
+	// Converting from newer API version to an older API version isn't supported. This is here only for the roundtrip tests meanwhile.
+	out.NodeName = in.NodeRegistration.Name
+	out.CRISocket = in.NodeRegistration.CRISocket
+	out.NoTaintMaster = in.NodeRegistration.Taints != nil && len(in.NodeRegistration.Taints) == 0
+
+	return nil
+}
+
+func Convert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in *kubeadm.NodeConfiguration, out *NodeConfiguration, s conversion.Scope) error {
+	if err := autoConvert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in, out, s); err != nil {
+		return err
+	}
+
+	// Converting from newer API version to an older API version isn't supported. This is here only for the roundtrip tests meanwhile.
+	out.NodeName = in.NodeRegistration.Name
+	out.CRISocket = in.NodeRegistration.CRISocket
+
+	return nil
+}