Refactor the upgrade process to stop using config on upgrade

Signed-off-by: Marko Mudrinić <mudrinic.mare@gmail.com>

pkg/scripts/kubeadm.go

@@ -66,7 +66,7 @@ var (
 	`)
 
 	kubeadmUpgradeScriptTemplate = heredoc.Doc(`
-		echo yes | sudo {{ .KUBEADM_UPGRADE }}{{ if .LEADER }} --config={{ .WORK_DIR }}/cfg/master_{{ .NODE_ID }}.yaml{{ end }}
+		sudo {{ .KUBEADM_UPGRADE }}
 		sudo find /etc/kubernetes/pki/ -name *.crt -exec chmod 600 {} \;
 	`)
 

pkg/scripts/kubeadm_test.go

@@ -280,7 +280,7 @@ func TestKubeadmUpgrade(t *testing.T) {
 			name: "leader",
 			args: args{
 				workdir:    "some",
-				kubeadmCmd: "kubeadm upgrade apply v1.1.1",
+				kubeadmCmd: "kubeadm upgrade apply --yes v1.1.1",
 				leader:     true,
 			},
 		},

pkg/scripts/testdata/TestKubeadmUpgrade-leader.golden

@@ -1,4 +1,4 @@
 set -xeuo pipefail
 export "PATH=$PATH:/sbin:/usr/local/bin:/opt/bin"
-echo yes | sudo kubeadm upgrade apply v1.1.1 --config=some/cfg/master_0.yaml
+sudo kubeadm upgrade apply --yes v1.1.1
 sudo find /etc/kubernetes/pki/ -name *.crt -exec chmod 600 {} \;

pkg/scripts/testdata/TestKubeadmUpgrade-v1beta2.golden

@@ -1,4 +1,4 @@
 set -xeuo pipefail
 export "PATH=$PATH:/sbin:/usr/local/bin:/opt/bin"
-echo yes | sudo kubeadm upgrade node
+sudo kubeadm upgrade node
 sudo find /etc/kubernetes/pki/ -name *.crt -exec chmod 600 {} \;

pkg/tasks/kubeadm_config.go

@@ -17,6 +17,7 @@ limitations under the License.
 package tasks
 
 import (
+	"context"
 	"fmt"
 
 	kubeoneapi "k8c.io/kubeone/pkg/apis/kubeone"
@@ -25,6 +26,11 @@ import (
 	"k8c.io/kubeone/pkg/scripts"
 	"k8c.io/kubeone/pkg/state"
 	"k8c.io/kubeone/pkg/templates/kubeadm"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 func determinePauseImage(s *state.State) error {
@@ -58,10 +64,7 @@ func generateKubeadm(s *state.State) error {
 		return err
 	}
 
-	kubeadmProvider, err := kubeadm.New(s.Cluster.Versions.Kubernetes)
-	if err != nil {
-		return err
-	}
+	kubeadmProvider := kubeadm.New(s.Cluster.Versions.Kubernetes)
 
 	for idx := range s.Cluster.ControlPlane.Hosts {
 		node := s.Cluster.ControlPlane.Hosts[idx]
@@ -70,7 +73,7 @@ func generateKubeadm(s *state.State) error {
 			return err
 		}
 
-		s.Configuration.AddFile(fmt.Sprintf("cfg/master_%d.yaml", node.ID), kubeadmConf)
+		s.Configuration.AddFile(fmt.Sprintf("cfg/master_%d.yaml", node.ID), kubeadmConf.FullConfiguration)
 	}
 
 	for idx := range s.Cluster.StaticWorkers.Hosts {
@@ -80,7 +83,7 @@ func generateKubeadm(s *state.State) error {
 			return err
 		}
 
-		s.Configuration.AddFile(fmt.Sprintf("cfg/worker_%d.yaml", node.ID), kubeadmConf)
+		s.Configuration.AddFile(fmt.Sprintf("cfg/worker_%d.yaml", node.ID), kubeadmConf.JoinConfiguration)
 	}
 
 	return s.RunTaskOnAllNodes(uploadKubeadmToNode, state.RunParallel)
@@ -89,3 +92,62 @@ func generateKubeadm(s *state.State) error {
 func uploadKubeadmToNode(s *state.State, _ *kubeoneapi.HostConfig, conn executor.Interface) error {
 	return s.Configuration.UploadTo(conn, s.WorkDir)
 }
+
+func uploadKubeadmToConfigMaps(s *state.State) error {
+	s.Logger.Info("Updating kubeadm ConfigMaps...")
+
+	leader, err := s.Cluster.Leader()
+	if err != nil {
+		return err
+	}
+
+	kubeadmProvider := kubeadm.New(s.Cluster.Versions.Kubernetes)
+
+	kubeadmConfig, err := kubeadmProvider.Config(s, leader)
+	if err != nil {
+		return err
+	}
+
+	s.Logger.Debug("Updating kube-system/kubeadm-config ConfigMap...")
+	updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		return updateConfigMap(s, "kubeadm-config", metav1.NamespaceSystem, "ClusterConfiguration", kubeadmConfig.ClusterConfiguration)
+	})
+	if updateErr != nil {
+		return fail.Runtime(err, "updating kubeadm ConfigMaps")
+	}
+
+	s.Logger.Debug("Updating kube-system/kubelet-config ConfigMap...")
+	updateErr = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		return updateConfigMap(s, "kubelet-config", metav1.NamespaceSystem, "kubelet", kubeadmConfig.KubeletConfiguration)
+	})
+	if updateErr != nil {
+		return fail.Runtime(err, "updating kubeadm ConfigMaps")
+	}
+
+	s.Logger.Debug("Updating kube-system/kube-proxy ConfigMap...")
+	updateErr = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		return updateConfigMap(s, "kube-proxy", metav1.NamespaceSystem, "config.conf", kubeadmConfig.KubeProxyConfiguration)
+	})
+	if updateErr != nil {
+		return fail.Runtime(err, "updating kubeadm ConfigMaps")
+	}
+
+	return nil
+}
+
+func updateConfigMap(s *state.State, name, namespace, key, value string) error {
+	configMap := corev1.ConfigMap{}
+	objKey := client.ObjectKey{
+		Name:      name,
+		Namespace: namespace,
+	}
+
+	err := s.DynamicClient.Get(context.Background(), objKey, &configMap)
+	if err != nil {
+		return fmt.Errorf("updating %s/%s ConfigMap: %w", objKey.Namespace, objKey.Name, err)
+	}
+
+	configMap.Data[key] = value
+
+	return s.DynamicClient.Update(s.Context, &configMap)
+}

pkg/tasks/kubeadm_upgrade.go

@@ -24,10 +24,7 @@ import (
 )
 
 func upgradeLeaderControlPlane(s *state.State, nodeID int) error {
-	kadm, err := kubeadm.New(s.Cluster.Versions.Kubernetes)
-	if err != nil {
-		return err
-	}
+	kadm := kubeadm.New(s.Cluster.Versions.Kubernetes)
 
 	cmd, err := scripts.KubeadmUpgrade(kadm.UpgradeLeaderCommand(), s.WorkDir, true, nodeID)
 	if err != nil {
@@ -40,10 +37,7 @@ func upgradeLeaderControlPlane(s *state.State, nodeID int) error {
 }
 
 func upgradeFollowerControlPlane(s *state.State, nodeID int) error {
-	kadm, err := kubeadm.New(s.Cluster.Versions.Kubernetes)
-	if err != nil {
-		return err
-	}
+	kadm := kubeadm.New(s.Cluster.Versions.Kubernetes)
 
 	cmd, err := scripts.KubeadmUpgrade(kadm.UpgradeFollowerCommand(), s.WorkDir, false, nodeID)
 	if err != nil {
@@ -56,12 +50,9 @@ func upgradeFollowerControlPlane(s *state.State, nodeID int) error {
 }
 
 func upgradeStaticWorker(s *state.State) error {
-	kadm, err := kubeadm.New(s.Cluster.Versions.Kubernetes)
-	if err != nil {
-		return err
-	}
+	kadm := kubeadm.New(s.Cluster.Versions.Kubernetes)
 
-	_, _, err = s.Runner.Run(`sudo `+kadm.UpgradeStaticWorkerCommand(), nil)
+	_, _, err := s.Runner.Run(`sudo `+kadm.UpgradeStaticWorkerCommand(), nil)
 
 	return fail.SSH(err, "running kubeadm upgrade on static worker")
 }

pkg/tasks/tasks.go

@@ -324,6 +324,7 @@ func WithUpgrade(t Tasks) Tasks {
 		append(kubernetesConfigFiles()...). // this, in the upgrade process where config rails are handled
 		append(Tasks{
 			{Fn: kubeconfig.BuildKubernetesClientset, Operation: "building kubernetes clientset"},
+			{Fn: uploadKubeadmToConfigMaps, Operation: "updating kubeadm configmaps"},
 			{Fn: runPreflightChecks, Operation: "checking preflight safetynet", Retries: 1},
 			{Fn: upgradeLeader, Operation: "upgrading leader control plane"},
 			{Fn: upgradeFollower, Operation: "upgrading follower control plane"},

pkg/templates/kubeadm/kubeadm.go

@@ -25,16 +25,24 @@ const (
 	kubeadmUpgradeNodeCommand = "kubeadm upgrade node"
 )
 
+type Config struct {
+	FullConfiguration      string
+	ClusterConfiguration   string
+	JoinConfiguration      string
+	KubeletConfiguration   string
+	KubeProxyConfiguration string
+}
+
 // Kubedm interface abstract differences between different kubeadm versions
 type Kubedm interface {
-	Config(s *state.State, instance kubeoneapi.HostConfig) (string, error)
-	ConfigWorker(s *state.State, instance kubeoneapi.HostConfig) (string, error)
+	Config(s *state.State, instance kubeoneapi.HostConfig) (*Config, error)
+	ConfigWorker(s *state.State, instance kubeoneapi.HostConfig) (*Config, error)
 	UpgradeLeaderCommand() string
 	UpgradeFollowerCommand() string
 	UpgradeStaticWorkerCommand() string
 }
 
 // New constructor
-func New(ver string) (Kubedm, error) {
-	return &kubeadmv1beta3{version: ver}, nil
+func New(ver string) Kubedm {
+	return &kubeadmv1beta3{version: ver}
 }

pkg/templates/kubeadm/kubeadmv1beta3.go

@@ -20,35 +20,76 @@ import (
 	"fmt"
 
 	kubeoneapi "k8c.io/kubeone/pkg/apis/kubeone"
+	"k8c.io/kubeone/pkg/fail"
 	"k8c.io/kubeone/pkg/state"
 	"k8c.io/kubeone/pkg/templates"
 	"k8c.io/kubeone/pkg/templates/kubeadm/v1beta3"
+
+	"k8s.io/apimachinery/pkg/runtime"
 )
 
 type kubeadmv1beta3 struct {
 	version string
 }
 
-func (*kubeadmv1beta3) Config(s *state.State, instance kubeoneapi.HostConfig) (string, error) {
+func (*kubeadmv1beta3) Config(s *state.State, instance kubeoneapi.HostConfig) (*Config, error) {
 	config, err := v1beta3.NewConfig(s, instance)
 	if err != nil {
-		return "", err
+		return nil, err
+	}
+
+	fullConfig, err := templates.KubernetesToYAML([]runtime.Object{
+		config.InitConfiguration,
+		config.JoinConfiguration,
+		config.ClusterConfiguration,
+		config.KubeletConfiguration,
+		config.KubeProxyConfiguration,
+	})
+	if err != nil {
+		return nil, fail.Runtime(err, "converting kubeadm configuration to yaml")
+	}
+
+	clusterConfig, err := templates.KubernetesToYAML([]runtime.Object{config.ClusterConfiguration})
+	if err != nil {
+		return nil, fail.Runtime(err, "converting kubeadm ClusterConfiguration to yaml")
+	}
+
+	kubeletConfig, err := templates.KubernetesToYAML([]runtime.Object{config.KubeletConfiguration})
+	if err != nil {
+		return nil, fail.Runtime(err, "converting kubeadm KubeletConfiguration to yaml")
+	}
+
+	kubeProxyConfig, err := templates.KubernetesToYAML([]runtime.Object{config.KubeProxyConfiguration})
+	if err != nil {
+		return nil, fail.Runtime(err, "converting kubeadm KubeProxyConfiguration to yaml")
 	}
 
-	return templates.KubernetesToYAML(config)
+	return &Config{
+		FullConfiguration:      fullConfig,
+		ClusterConfiguration:   clusterConfig,
+		KubeletConfiguration:   kubeletConfig,
+		KubeProxyConfiguration: kubeProxyConfig,
+	}, nil
 }
 
-func (*kubeadmv1beta3) ConfigWorker(s *state.State, instance kubeoneapi.HostConfig) (string, error) {
+func (*kubeadmv1beta3) ConfigWorker(s *state.State, instance kubeoneapi.HostConfig) (*Config, error) {
 	config, err := v1beta3.NewConfigWorker(s, instance)
 	if err != nil {
-		return "", err
+		return nil, err
+	}
+
+	joinConfig, err := templates.KubernetesToYAML([]runtime.Object{config.JoinConfiguration})
+	if err != nil {
+		return nil, fail.Runtime(err, "converting kubeadm JoinConfiguration to yaml")
 	}
 
-	return templates.KubernetesToYAML(config)
+	return &Config{
+		JoinConfiguration: joinConfig,
+	}, nil
 }
 
 func (k *kubeadmv1beta3) UpgradeLeaderCommand() string {
-	return fmt.Sprintf("kubeadm upgrade apply %s", k.version)
+	return fmt.Sprintf("kubeadm upgrade apply --yes %s", k.version)
 }
 
 func (*kubeadmv1beta3) UpgradeFollowerCommand() string {