feat: custom image settings for k8s upgrade

Allows to use custom registry/images. Fixes: #8275 Co-authored-by: @g3offrey Signed-off-by: Matthieu STROHL <mstrohl@dive-in-it.com> Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
siderolabs · Feb 15, 2024 · 3fe82ec · 3fe82ec
1 parent fa3b933
commit 3fe82ec
Show file tree

Hide file tree

Showing 8 changed files with 86 additions and 47 deletions.
diff --git a/cmd/talosctl/cmd/talos/upgrade-k8s.go b/cmd/talosctl/cmd/talos/upgrade-k8s.go
@@ -49,6 +49,13 @@ func init() {
 	upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.DryRun, "dry-run", false, "skip the actual upgrade and show the upgrade plan instead")
 	upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.PrePullImages, "pre-pull-images", true, "pre-pull images before upgrade")
 	upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.UpgradeKubelet, "upgrade-kubelet", true, "upgrade kubelet service")
+
+	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.KubeletImage, "kubelet-image", constants.KubeletImage, "kubelet image to use")
+	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.APIServerImage, "apiserver-image", constants.KubernetesAPIServerImage, "kube-apiserver image to use")
+	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.ControllerManagerImage, "controller-manager-image", constants.KubernetesControllerManagerImage, "kube-controller-manager image to use")
+	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.SchedulerImage, "scheduler-image", constants.KubernetesSchedulerImage, "kube-scheduler image to use")
+	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.ProxyImage, "proxy-image", constants.KubeProxyImage, "kube-proxy image to use")
+
 	addCommand(upgradeK8sCmd)
 }
 

diff --git a/hack/release.toml b/hack/release.toml
@@ -93,6 +93,13 @@ config:
         - content: MONITOR ${upsmonHost} 1 remote pass password
           mountPath: /usr/local/etc/nut/upsmon.conf
 ```
+"""
+
+    [notes.k8supgrade]
+        title = "Kubernetes Upgrade"
+        description = """\
+The command `talosctl upgrade-k8s` now supports specifying custom image references for Kubernetes components via `--*-image` flags.
+The default behavior is unchanged, and the flags are optional.
 """
 
 [make_deps]

diff --git a/internal/integration/provision/provision.go b/internal/integration/provision/provision.go
@@ -417,6 +417,12 @@ func (suite *BaseSuite) upgradeKubernetes(fromVersion, toVersion string, skipKub
 		UpgradeKubelet: !skipKubeletUpgrade,
 		PrePullImages:  true,
 
+		KubeletImage:           constants.KubeletImage,
+		APIServerImage:         constants.KubernetesAPIServerImage,
+		ControllerManagerImage: constants.KubernetesControllerManagerImage,
+		SchedulerImage:         constants.KubernetesSchedulerImage,
+		ProxyImage:             constants.KubeProxyImage,
+
 		EncoderOpt: encoder.WithComments(encoder.CommentsAll),
 	}
 

diff --git a/pkg/cluster/kubernetes/kubelet.go b/pkg/cluster/kubernetes/kubelet.go
@@ -22,7 +22,6 @@ import (
 	"github.com/siderolabs/talos/pkg/kubernetes"
 	"github.com/siderolabs/talos/pkg/machinery/client"
 	v1alpha1config "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1"
-	"github.com/siderolabs/talos/pkg/machinery/constants"
 	"github.com/siderolabs/talos/pkg/machinery/resources/k8s"
 	"github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1"
 )
@@ -199,7 +198,7 @@ func upgradeKubeletPatcher(
 			}
 		}
 
-		image := fmt.Sprintf("%s:v%s", constants.KubeletImage, options.Path.ToVersion())
+		image := fmt.Sprintf("%s:v%s", options.KubeletImage, options.Path.ToVersion())
 
 		if oldImage == image {
 			return errUpdateSkipped

diff --git a/pkg/cluster/kubernetes/talos_managed.go b/pkg/cluster/kubernetes/talos_managed.go
@@ -124,9 +124,9 @@ func Upgrade(ctx context.Context, cluster UpgradeProvider, options UpgradeOption
 
 func prePullImages(ctx context.Context, talosClient *client.Client, options UpgradeOptions) error {
 	for _, imageRef := range []string{
-		fmt.Sprintf("%s:v%s", constants.KubernetesAPIServerImage, options.Path.ToVersion()),
-		fmt.Sprintf("%s:v%s", constants.KubernetesControllerManagerImage, options.Path.ToVersion()),
-		fmt.Sprintf("%s:v%s", constants.KubernetesSchedulerImage, options.Path.ToVersion()),
+		fmt.Sprintf("%s:v%s", options.APIServerImage, options.Path.ToVersion()),
+		fmt.Sprintf("%s:v%s", options.ControllerManagerImage, options.Path.ToVersion()),
+		fmt.Sprintf("%s:v%s", options.SchedulerImage, options.Path.ToVersion()),
 	} {
 		for _, node := range options.controlPlaneNodes {
 			options.Log(" > %q: pre-pulling %s", node, imageRef)
@@ -146,7 +146,7 @@ func prePullImages(ctx context.Context, talosClient *client.Client, options Upgr
 		return nil
 	}
 
-	imageRef := fmt.Sprintf("%s:v%s", constants.KubeletImage, options.Path.ToVersion())
+	imageRef := fmt.Sprintf("%s:v%s", options.KubeletImage, options.Path.ToVersion())
 
 	for _, node := range append(append([]string(nil), options.controlPlaneNodes...), options.workerNodes...) {
 		options.Log(" > %q: pre-pulling %s", node, imageRef)
@@ -206,7 +206,7 @@ func patchKubeProxy(options UpgradeOptions) func(config *v1alpha1config.Config)
 			config.ClusterConfig.ProxyConfig = &v1alpha1config.ProxyConfig{}
 		}
 
-		config.ClusterConfig.ProxyConfig.ContainerImage = fmt.Sprintf("%s:v%s", constants.KubeProxyImage, options.Path.ToVersion())
+		config.ClusterConfig.ProxyConfig.ContainerImage = fmt.Sprintf("%s:v%s", options.ProxyImage, options.Path.ToVersion())
 
 		return nil
 	}
@@ -304,7 +304,23 @@ func upgradeStaticPodOnNode(ctx context.Context, cluster UpgradeProvider, option
 
 var errUpdateSkipped = errors.New("update skipped")
 
-//nolint:gocyclo,cyclop
+func staticPodImage(logUpdate func(oldImage string), imageName, containerImage, configImage string, options UpgradeOptions) (string, error) {
+	image := fmt.Sprintf("%s:v%s", imageName, options.Path.ToVersion())
+
+	if containerImage == image || configImage == image {
+		return "", errUpdateSkipped
+	}
+
+	logUpdate(containerImage)
+
+	if options.DryRun {
+		return "", errUpdateSkipped
+	}
+
+	return image, nil
+}
+
+//nolint:gocyclo
 func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResource resource.Resource) func(config *v1alpha1config.Config) error {
 	return func(config *v1alpha1config.Config) error {
 		if config.ClusterConfig == nil {
@@ -349,16 +365,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou
 				config.ClusterConfig.APIServerConfig = &v1alpha1config.APIServerConfig{}
 			}
 
-			image := fmt.Sprintf("%s:v%s", constants.KubernetesAPIServerImage, options.Path.ToVersion())
-
-			if config.ClusterConfig.APIServerConfig.ContainerImage == image || configImage == image {
-				return errUpdateSkipped
-			}
-
-			logUpdate(config.ClusterConfig.APIServerConfig.ContainerImage)
-
-			if options.DryRun {
-				return errUpdateSkipped
+			image, err := staticPodImage(logUpdate,
+				options.APIServerImage,
+				config.ClusterConfig.APIServerConfig.ContainerImage,
+				configImage,
+				options)
+			if err != nil {
+				return err
 			}
 
 			config.ClusterConfig.APIServerConfig.ContainerImage = image
@@ -367,16 +380,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou
 				config.ClusterConfig.ControllerManagerConfig = &v1alpha1config.ControllerManagerConfig{}
 			}
 
-			image := fmt.Sprintf("%s:v%s", constants.KubernetesControllerManagerImage, options.Path.ToVersion())
-
-			if config.ClusterConfig.ControllerManagerConfig.ContainerImage == image || configImage == image {
-				return errUpdateSkipped
-			}
-
-			logUpdate(config.ClusterConfig.ControllerManagerConfig.ContainerImage)
-
-			if options.DryRun {
-				return errUpdateSkipped
+			image, err := staticPodImage(logUpdate,
+				options.ControllerManagerImage,
+				config.ClusterConfig.ControllerManagerConfig.ContainerImage,
+				configImage,
+				options)
+			if err != nil {
+				return err
 			}
 
 			config.ClusterConfig.ControllerManagerConfig.ContainerImage = image
@@ -385,16 +395,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou
 				config.ClusterConfig.SchedulerConfig = &v1alpha1config.SchedulerConfig{}
 			}
 
-			image := fmt.Sprintf("%s:v%s", constants.KubernetesSchedulerImage, options.Path.ToVersion())
-
-			if config.ClusterConfig.SchedulerConfig.ContainerImage == image || configImage == image {
-				return errUpdateSkipped
-			}
-
-			logUpdate(config.ClusterConfig.SchedulerConfig.ContainerImage)
-
-			if options.DryRun {
-				return errUpdateSkipped
+			image, err := staticPodImage(logUpdate,
+				options.SchedulerImage,
+				config.ClusterConfig.SchedulerConfig.ContainerImage,
+				configImage,
+				options)
+			if err != nil {
+				return err
 			}
 
 			config.ClusterConfig.SchedulerConfig.ContainerImage = image

diff --git a/pkg/cluster/kubernetes/upgrade.go b/pkg/cluster/kubernetes/upgrade.go
@@ -32,6 +32,12 @@ type UpgradeOptions struct {
 	DryRun               bool
 	EncoderOpt           encoder.Option
 
+	KubeletImage           string
+	APIServerImage         string
+	ControllerManagerImage string
+	SchedulerImage         string
+	ProxyImage             string
+
 	controlPlaneNodes []string
 	workerNodes       []string
 }

diff --git a/website/content/v1.7/kubernetes-guides/upgrading-kubernetes.md b/website/content/v1.7/kubernetes-guides/upgrading-kubernetes.md
@@ -106,6 +106,8 @@ This command runs in several phases:
 
 If the command fails for any reason, it can be safely restarted to continue the upgrade process from the moment of the failure.
 
+> Note: When using custom/overridden Kubernetes component images, use flags `--*-image` to override the default image names.
+
 ## Manual Kubernetes Upgrade
 
 Kubernetes can be upgraded manually by following the steps outlined below.

diff --git a/website/content/v1.7/reference/cli.md b/website/content/v1.7/reference/cli.md
@@ -2887,15 +2887,20 @@ talosctl upgrade-k8s [flags]
 ### Options
 
 ```
-      --dry-run           skip the actual upgrade and show the upgrade plan instead
-      --endpoint string   the cluster control plane endpoint
-      --from string       the Kubernetes control plane version to upgrade from
-  -h, --help              help for upgrade-k8s
-      --pre-pull-images   pre-pull images before upgrade (default true)
-      --to string         the Kubernetes control plane version to upgrade to (default "1.29.1")
-      --upgrade-kubelet   upgrade kubelet service (default true)
-      --with-docs         patch all machine configs adding the documentation for each field (default true)
-      --with-examples     patch all machine configs with the commented examples (default true)
+      --apiserver-image string            kube-apiserver image to use (default "registry.k8s.io/kube-apiserver")
+      --controller-manager-image string   kube-controller-manager image to use (default "registry.k8s.io/kube-controller-manager")
+      --dry-run                           skip the actual upgrade and show the upgrade plan instead
+      --endpoint string                   the cluster control plane endpoint
+      --from string                       the Kubernetes control plane version to upgrade from
+  -h, --help                              help for upgrade-k8s
+      --kubelet-image string              kubelet image to use (default "ghcr.io/siderolabs/kubelet")
+      --pre-pull-images                   pre-pull images before upgrade (default true)
+      --proxy-image string                kube-proxy image to use (default "registry.k8s.io/kube-proxy")
+      --scheduler-image string            kube-scheduler image to use (default "registry.k8s.io/kube-scheduler")
+      --to string                         the Kubernetes control plane version to upgrade to (default "1.29.1")
+      --upgrade-kubelet                   upgrade kubelet service (default true)
+      --with-docs                         patch all machine configs adding the documentation for each field (default true)
+      --with-examples                     patch all machine configs with the commented examples (default true)
 ```
 
 ### Options inherited from parent commands