From 3fe82ec461995b680ecf060af75b47cd175a6342 Mon Sep 17 00:00:00 2001 From: Matthieu S Date: Wed, 7 Feb 2024 14:19:43 +0100 Subject: [PATCH] feat: custom image settings for k8s upgrade Allows to use custom registry/images. Fixes: #8275 Co-authored-by: @g3offrey Signed-off-by: Matthieu STROHL Signed-off-by: Andrey Smirnov --- cmd/talosctl/cmd/talos/upgrade-k8s.go | 7 ++ hack/release.toml | 7 ++ internal/integration/provision/provision.go | 6 ++ pkg/cluster/kubernetes/kubelet.go | 3 +- pkg/cluster/kubernetes/talos_managed.go | 79 ++++++++++--------- pkg/cluster/kubernetes/upgrade.go | 6 ++ .../kubernetes-guides/upgrading-kubernetes.md | 2 + website/content/v1.7/reference/cli.md | 23 +++--- 8 files changed, 86 insertions(+), 47 deletions(-) diff --git a/cmd/talosctl/cmd/talos/upgrade-k8s.go b/cmd/talosctl/cmd/talos/upgrade-k8s.go index 6262b6a026..7316df7d42 100644 --- a/cmd/talosctl/cmd/talos/upgrade-k8s.go +++ b/cmd/talosctl/cmd/talos/upgrade-k8s.go @@ -49,6 +49,13 @@ func init() { upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.DryRun, "dry-run", false, "skip the actual upgrade and show the upgrade plan instead") upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.PrePullImages, "pre-pull-images", true, "pre-pull images before upgrade") upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.UpgradeKubelet, "upgrade-kubelet", true, "upgrade kubelet service") + + upgradeK8sCmd.Flags().StringVar(&upgradeOptions.KubeletImage, "kubelet-image", constants.KubeletImage, "kubelet image to use") + upgradeK8sCmd.Flags().StringVar(&upgradeOptions.APIServerImage, "apiserver-image", constants.KubernetesAPIServerImage, "kube-apiserver image to use") + upgradeK8sCmd.Flags().StringVar(&upgradeOptions.ControllerManagerImage, "controller-manager-image", constants.KubernetesControllerManagerImage, "kube-controller-manager image to use") + upgradeK8sCmd.Flags().StringVar(&upgradeOptions.SchedulerImage, "scheduler-image", constants.KubernetesSchedulerImage, "kube-scheduler image to use") + upgradeK8sCmd.Flags().StringVar(&upgradeOptions.ProxyImage, "proxy-image", constants.KubeProxyImage, "kube-proxy image to use") + addCommand(upgradeK8sCmd) } diff --git a/hack/release.toml b/hack/release.toml index 500aae6184..c75575b171 100644 --- a/hack/release.toml +++ b/hack/release.toml @@ -93,6 +93,13 @@ config: - content: MONITOR ${upsmonHost} 1 remote pass password mountPath: /usr/local/etc/nut/upsmon.conf ``` +""" + + [notes.k8supgrade] + title = "Kubernetes Upgrade" + description = """\ +The command `talosctl upgrade-k8s` now supports specifying custom image references for Kubernetes components via `--*-image` flags. +The default behavior is unchanged, and the flags are optional. """ [make_deps] diff --git a/internal/integration/provision/provision.go b/internal/integration/provision/provision.go index a589aa98a5..f211699653 100644 --- a/internal/integration/provision/provision.go +++ b/internal/integration/provision/provision.go @@ -417,6 +417,12 @@ func (suite *BaseSuite) upgradeKubernetes(fromVersion, toVersion string, skipKub UpgradeKubelet: !skipKubeletUpgrade, PrePullImages: true, + KubeletImage: constants.KubeletImage, + APIServerImage: constants.KubernetesAPIServerImage, + ControllerManagerImage: constants.KubernetesControllerManagerImage, + SchedulerImage: constants.KubernetesSchedulerImage, + ProxyImage: constants.KubeProxyImage, + EncoderOpt: encoder.WithComments(encoder.CommentsAll), } diff --git a/pkg/cluster/kubernetes/kubelet.go b/pkg/cluster/kubernetes/kubelet.go index 3c25b81f9a..55c3cf1184 100644 --- a/pkg/cluster/kubernetes/kubelet.go +++ b/pkg/cluster/kubernetes/kubelet.go @@ -22,7 +22,6 @@ import ( "github.com/siderolabs/talos/pkg/kubernetes" "github.com/siderolabs/talos/pkg/machinery/client" v1alpha1config "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" - "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/machinery/resources/k8s" "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" ) @@ -199,7 +198,7 @@ func upgradeKubeletPatcher( } } - image := fmt.Sprintf("%s:v%s", constants.KubeletImage, options.Path.ToVersion()) + image := fmt.Sprintf("%s:v%s", options.KubeletImage, options.Path.ToVersion()) if oldImage == image { return errUpdateSkipped diff --git a/pkg/cluster/kubernetes/talos_managed.go b/pkg/cluster/kubernetes/talos_managed.go index d89e7f4a0e..e525d16319 100644 --- a/pkg/cluster/kubernetes/talos_managed.go +++ b/pkg/cluster/kubernetes/talos_managed.go @@ -124,9 +124,9 @@ func Upgrade(ctx context.Context, cluster UpgradeProvider, options UpgradeOption func prePullImages(ctx context.Context, talosClient *client.Client, options UpgradeOptions) error { for _, imageRef := range []string{ - fmt.Sprintf("%s:v%s", constants.KubernetesAPIServerImage, options.Path.ToVersion()), - fmt.Sprintf("%s:v%s", constants.KubernetesControllerManagerImage, options.Path.ToVersion()), - fmt.Sprintf("%s:v%s", constants.KubernetesSchedulerImage, options.Path.ToVersion()), + fmt.Sprintf("%s:v%s", options.APIServerImage, options.Path.ToVersion()), + fmt.Sprintf("%s:v%s", options.ControllerManagerImage, options.Path.ToVersion()), + fmt.Sprintf("%s:v%s", options.SchedulerImage, options.Path.ToVersion()), } { for _, node := range options.controlPlaneNodes { options.Log(" > %q: pre-pulling %s", node, imageRef) @@ -146,7 +146,7 @@ func prePullImages(ctx context.Context, talosClient *client.Client, options Upgr return nil } - imageRef := fmt.Sprintf("%s:v%s", constants.KubeletImage, options.Path.ToVersion()) + imageRef := fmt.Sprintf("%s:v%s", options.KubeletImage, options.Path.ToVersion()) for _, node := range append(append([]string(nil), options.controlPlaneNodes...), options.workerNodes...) { options.Log(" > %q: pre-pulling %s", node, imageRef) @@ -206,7 +206,7 @@ func patchKubeProxy(options UpgradeOptions) func(config *v1alpha1config.Config) config.ClusterConfig.ProxyConfig = &v1alpha1config.ProxyConfig{} } - config.ClusterConfig.ProxyConfig.ContainerImage = fmt.Sprintf("%s:v%s", constants.KubeProxyImage, options.Path.ToVersion()) + config.ClusterConfig.ProxyConfig.ContainerImage = fmt.Sprintf("%s:v%s", options.ProxyImage, options.Path.ToVersion()) return nil } @@ -304,7 +304,23 @@ func upgradeStaticPodOnNode(ctx context.Context, cluster UpgradeProvider, option var errUpdateSkipped = errors.New("update skipped") -//nolint:gocyclo,cyclop +func staticPodImage(logUpdate func(oldImage string), imageName, containerImage, configImage string, options UpgradeOptions) (string, error) { + image := fmt.Sprintf("%s:v%s", imageName, options.Path.ToVersion()) + + if containerImage == image || configImage == image { + return "", errUpdateSkipped + } + + logUpdate(containerImage) + + if options.DryRun { + return "", errUpdateSkipped + } + + return image, nil +} + +//nolint:gocyclo func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResource resource.Resource) func(config *v1alpha1config.Config) error { return func(config *v1alpha1config.Config) error { if config.ClusterConfig == nil { @@ -349,16 +365,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou config.ClusterConfig.APIServerConfig = &v1alpha1config.APIServerConfig{} } - image := fmt.Sprintf("%s:v%s", constants.KubernetesAPIServerImage, options.Path.ToVersion()) - - if config.ClusterConfig.APIServerConfig.ContainerImage == image || configImage == image { - return errUpdateSkipped - } - - logUpdate(config.ClusterConfig.APIServerConfig.ContainerImage) - - if options.DryRun { - return errUpdateSkipped + image, err := staticPodImage(logUpdate, + options.APIServerImage, + config.ClusterConfig.APIServerConfig.ContainerImage, + configImage, + options) + if err != nil { + return err } config.ClusterConfig.APIServerConfig.ContainerImage = image @@ -367,16 +380,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou config.ClusterConfig.ControllerManagerConfig = &v1alpha1config.ControllerManagerConfig{} } - image := fmt.Sprintf("%s:v%s", constants.KubernetesControllerManagerImage, options.Path.ToVersion()) - - if config.ClusterConfig.ControllerManagerConfig.ContainerImage == image || configImage == image { - return errUpdateSkipped - } - - logUpdate(config.ClusterConfig.ControllerManagerConfig.ContainerImage) - - if options.DryRun { - return errUpdateSkipped + image, err := staticPodImage(logUpdate, + options.ControllerManagerImage, + config.ClusterConfig.ControllerManagerConfig.ContainerImage, + configImage, + options) + if err != nil { + return err } config.ClusterConfig.ControllerManagerConfig.ContainerImage = image @@ -385,16 +395,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou config.ClusterConfig.SchedulerConfig = &v1alpha1config.SchedulerConfig{} } - image := fmt.Sprintf("%s:v%s", constants.KubernetesSchedulerImage, options.Path.ToVersion()) - - if config.ClusterConfig.SchedulerConfig.ContainerImage == image || configImage == image { - return errUpdateSkipped - } - - logUpdate(config.ClusterConfig.SchedulerConfig.ContainerImage) - - if options.DryRun { - return errUpdateSkipped + image, err := staticPodImage(logUpdate, + options.SchedulerImage, + config.ClusterConfig.SchedulerConfig.ContainerImage, + configImage, + options) + if err != nil { + return err } config.ClusterConfig.SchedulerConfig.ContainerImage = image diff --git a/pkg/cluster/kubernetes/upgrade.go b/pkg/cluster/kubernetes/upgrade.go index 959fa5ea22..980d2904f5 100644 --- a/pkg/cluster/kubernetes/upgrade.go +++ b/pkg/cluster/kubernetes/upgrade.go @@ -32,6 +32,12 @@ type UpgradeOptions struct { DryRun bool EncoderOpt encoder.Option + KubeletImage string + APIServerImage string + ControllerManagerImage string + SchedulerImage string + ProxyImage string + controlPlaneNodes []string workerNodes []string } diff --git a/website/content/v1.7/kubernetes-guides/upgrading-kubernetes.md b/website/content/v1.7/kubernetes-guides/upgrading-kubernetes.md index a5e83ce19a..7c60620655 100644 --- a/website/content/v1.7/kubernetes-guides/upgrading-kubernetes.md +++ b/website/content/v1.7/kubernetes-guides/upgrading-kubernetes.md @@ -106,6 +106,8 @@ This command runs in several phases: If the command fails for any reason, it can be safely restarted to continue the upgrade process from the moment of the failure. +> Note: When using custom/overridden Kubernetes component images, use flags `--*-image` to override the default image names. + ## Manual Kubernetes Upgrade Kubernetes can be upgraded manually by following the steps outlined below. diff --git a/website/content/v1.7/reference/cli.md b/website/content/v1.7/reference/cli.md index 766fa9e434..6e8d483cfe 100644 --- a/website/content/v1.7/reference/cli.md +++ b/website/content/v1.7/reference/cli.md @@ -2887,15 +2887,20 @@ talosctl upgrade-k8s [flags] ### Options ``` - --dry-run skip the actual upgrade and show the upgrade plan instead - --endpoint string the cluster control plane endpoint - --from string the Kubernetes control plane version to upgrade from - -h, --help help for upgrade-k8s - --pre-pull-images pre-pull images before upgrade (default true) - --to string the Kubernetes control plane version to upgrade to (default "1.29.1") - --upgrade-kubelet upgrade kubelet service (default true) - --with-docs patch all machine configs adding the documentation for each field (default true) - --with-examples patch all machine configs with the commented examples (default true) + --apiserver-image string kube-apiserver image to use (default "registry.k8s.io/kube-apiserver") + --controller-manager-image string kube-controller-manager image to use (default "registry.k8s.io/kube-controller-manager") + --dry-run skip the actual upgrade and show the upgrade plan instead + --endpoint string the cluster control plane endpoint + --from string the Kubernetes control plane version to upgrade from + -h, --help help for upgrade-k8s + --kubelet-image string kubelet image to use (default "ghcr.io/siderolabs/kubelet") + --pre-pull-images pre-pull images before upgrade (default true) + --proxy-image string kube-proxy image to use (default "registry.k8s.io/kube-proxy") + --scheduler-image string kube-scheduler image to use (default "registry.k8s.io/kube-scheduler") + --to string the Kubernetes control plane version to upgrade to (default "1.29.1") + --upgrade-kubelet upgrade kubelet service (default true) + --with-docs patch all machine configs adding the documentation for each field (default true) + --with-examples patch all machine configs with the commented examples (default true) ``` ### Options inherited from parent commands