From af1b50a8319f6347d98a48a322b874a358d08812 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Sat, 29 May 2021 00:49:42 +0300 Subject: [PATCH 01/11] Hold containerd version Signed-off-by: Artiom Diomin --- pkg/clientutil/crds.go | 2 +- pkg/scripts/render.go | 64 +++++++++++-------- .../TestKubeadmAmazonLinux-force.golden | 10 ++- ...beadmAmazonLinux-overwrite_registry.golden | 8 ++- ...onLinux-overwrite_registry_insecure.golden | 8 ++- .../TestKubeadmAmazonLinux-proxy.golden | 8 ++- .../TestKubeadmAmazonLinux-simple.golden | 8 ++- .../TestKubeadmAmazonLinux-v1.16.1.golden | 8 ++- .../testdata/TestKubeadmCentOS-force.golden | 11 +++- ...estKubeadmCentOS-overwrite_registry.golden | 9 ++- ...mCentOS-overwrite_registry_insecure.golden | 9 ++- .../testdata/TestKubeadmCentOS-proxy.golden | 9 ++- .../testdata/TestKubeadmCentOS-simple.golden | 9 ++- .../testdata/TestKubeadmCentOS-v1.16.1.golden | 9 ++- .../TestKubeadmCentOS-with_containerd.golden | 2 - ...h_containerd_with_insecure_registry.golden | 2 - ...estKubeadmDebian-overwrite_registry.golden | 8 ++- ...mDebian-overwrite_registry_insecure.golden | 8 ++- .../testdata/TestKubeadmDebian-simple.golden | 8 ++- ...TestUpgradeKubeadmAndCNIAmazonLinux.golden | 10 ++- .../TestUpgradeKubeadmAndCNICentOS.golden | 11 +++- .../TestUpgradeKubeadmAndCNIDebian.golden | 10 ++- ...UpgradeKubeletAndKubectlAmazonLinux.golden | 10 ++- .../TestUpgradeKubeletAndKubectlCentOS.golden | 11 +++- .../TestUpgradeKubeletAndKubectlDebian.golden | 10 ++- 25 files changed, 182 insertions(+), 80 deletions(-) diff --git a/pkg/clientutil/crds.go b/pkg/clientutil/crds.go index 63ae8f9c8..40aacd2e5 100644 --- a/pkg/clientutil/crds.go +++ b/pkg/clientutil/crds.go @@ -19,7 +19,7 @@ package clientutil import ( "context" - apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" dynclient "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/pkg/scripts/render.go b/pkg/scripts/render.go index 6d15c0476..915256436 100644 --- a/pkg/scripts/render.go +++ b/pkg/scripts/render.go @@ -39,7 +39,7 @@ var ( {{ end }} {{- if or .FORCE .UPGRADE }} - sudo apt-mark unhold docker-ce docker-ce-cli + sudo apt-mark unhold docker-ce docker-ce-cli containerd.io || true {{- end }} {{- $DOCKER_VERSION_TO_INSTALL := "%s" }} @@ -58,19 +58,24 @@ var ( {{- if .FORCE }} --allow-downgrades \ {{- end }} - docker-ce=5:{{ $DOCKER_VERSION_TO_INSTALL }} docker-ce-cli=5:{{ $DOCKER_VERSION_TO_INSTALL }} - sudo apt-mark hold docker-ce docker-ce-cli + docker-ce=5:{{ $DOCKER_VERSION_TO_INSTALL }} \ + docker-ce-cli=5:{{ $DOCKER_VERSION_TO_INSTALL }} \ + containerd.io=%s + sudo apt-mark hold docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload + sudo systemctl enable --now containerd sudo systemctl enable --now docker `, defaultDockerVersion, defaultLegacyDockerVersion, latestDockerVersion, + defaultContainerdVersion, ), "yum-docker-ce-amzn": heredoc.Docf(` {{- if or .FORCE .UPGRADE }} - sudo yum versionlock delete docker cri-tools || true + sudo yum versionlock delete docker cri-tools containerd {{- end }} {{- $CRICTL_VERSION_TO_INSTALL := "%s" }} @@ -83,20 +88,25 @@ var ( {{ $DOCKER_VERSION_TO_INSTALL = "%s" }} {{- end }} - sudo yum install -y docker-{{ $DOCKER_VERSION_TO_INSTALL }} cri-tools-{{ $CRICTL_VERSION_TO_INSTALL }} - sudo yum versionlock add docker cri-tools + sudo yum install -y \ + docker-{{ $DOCKER_VERSION_TO_INSTALL }} \ + containerd.io-%s \ + cri-tools-{{ $CRICTL_VERSION_TO_INSTALL }} + sudo yum versionlock add docker cri-tools containerd cat </dev/null -sudo yum versionlock delete docker-ce docker-ce-cli || true +sudo yum versionlock delete docker-ce docker-ce-cli containerd.io + +sudo yum install -y \ + docker-ce-19.03.* \ + docker-ce-cli-19.03.* \ + containerd.io-1.4.* +sudo yum versionlock add docker-ce docker-ce-cli containerd.io -sudo yum install -y docker-ce-19.03.* docker-ce-cli-19.03.* -sudo yum versionlock add docker-ce docker-ce-cli sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden index 3e8db7849..4d4c8b3b9 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden @@ -89,9 +89,14 @@ sudo yum install -y yum-utils sudo yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo sudo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true >/dev/null -sudo yum install -y docker-ce-19.03.* docker-ce-cli-19.03.* -sudo yum versionlock add docker-ce docker-ce-cli +sudo yum install -y \ + docker-ce-19.03.* \ + docker-ce-cli-19.03.* \ + containerd.io-1.4.* +sudo yum versionlock add docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden index 9eb05c567..5642cfca8 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden @@ -92,9 +92,14 @@ sudo yum install -y yum-utils sudo yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo sudo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true >/dev/null -sudo yum install -y docker-ce-19.03.* docker-ce-cli-19.03.* -sudo yum versionlock add docker-ce docker-ce-cli +sudo yum install -y \ + docker-ce-19.03.* \ + docker-ce-cli-19.03.* \ + containerd.io-1.4.* +sudo yum versionlock add docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden b/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden index fd2070ab3..495831cda 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden @@ -89,9 +89,14 @@ sudo yum install -y yum-utils sudo yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo sudo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true >/dev/null -sudo yum install -y docker-ce-19.03.* docker-ce-cli-19.03.* -sudo yum versionlock add docker-ce docker-ce-cli +sudo yum install -y \ + docker-ce-19.03.* \ + docker-ce-cli-19.03.* \ + containerd.io-1.4.* +sudo yum versionlock add docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden b/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden index 3e8db7849..4d4c8b3b9 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden @@ -89,9 +89,14 @@ sudo yum install -y yum-utils sudo yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo sudo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true >/dev/null -sudo yum install -y docker-ce-19.03.* docker-ce-cli-19.03.* -sudo yum versionlock add docker-ce docker-ce-cli +sudo yum install -y \ + docker-ce-19.03.* \ + docker-ce-cli-19.03.* \ + containerd.io-1.4.* +sudo yum versionlock add docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden b/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden index ec215e4b6..6032b9938 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden @@ -94,9 +94,14 @@ sudo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true >/ sudo sed -i 's/\$releasever/7/g' /etc/yum.repos.d/docker-ce.repo -sudo yum install -y docker-ce-18.09.* docker-ce-cli-18.09.* -sudo yum versionlock add docker-ce docker-ce-cli +sudo yum install -y \ + docker-ce-18.09.* \ + docker-ce-cli-18.09.* \ + containerd.io-1.4.* +sudo yum versionlock add docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden index bc67a77b6..e858abbc6 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden @@ -77,8 +77,6 @@ sudo yum-config-manager --add-repo=https://download.docker.com/linux/centos/dock sudo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true -sudo yum install -y yum-plugin-versionlock - sudo yum install -y containerd.io-1.4.* diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden index 956e690d6..b17c92cbb 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden @@ -77,8 +77,6 @@ sudo yum-config-manager --add-repo=https://download.docker.com/linux/centos/dock sudo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true -sudo yum install -y yum-plugin-versionlock - sudo yum install -y containerd.io-1.4.* diff --git a/pkg/scripts/testdata/TestKubeadmDebian-overwrite_registry.golden b/pkg/scripts/testdata/TestKubeadmDebian-overwrite_registry.golden index d1506afc7..06ef50d10 100644 --- a/pkg/scripts/testdata/TestKubeadmDebian-overwrite_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmDebian-overwrite_registry.golden @@ -90,9 +90,13 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get install \ --option "Dpkg::Options::=--force-confold" \ --no-install-recommends \ -y \ - docker-ce=5:19.03.* docker-ce-cli=5:19.03.* -sudo apt-mark hold docker-ce docker-ce-cli + docker-ce=5:19.03.* \ + docker-ce-cli=5:19.03.* \ + containerd.io=1.4.* +sudo apt-mark hold docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestKubeadmDebian-overwrite_registry_insecure.golden b/pkg/scripts/testdata/TestKubeadmDebian-overwrite_registry_insecure.golden index 7608cc97b..837fdfcbb 100644 --- a/pkg/scripts/testdata/TestKubeadmDebian-overwrite_registry_insecure.golden +++ b/pkg/scripts/testdata/TestKubeadmDebian-overwrite_registry_insecure.golden @@ -93,9 +93,13 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get install \ --option "Dpkg::Options::=--force-confold" \ --no-install-recommends \ -y \ - docker-ce=5:19.03.* docker-ce-cli=5:19.03.* -sudo apt-mark hold docker-ce docker-ce-cli + docker-ce=5:19.03.* \ + docker-ce-cli=5:19.03.* \ + containerd.io=1.4.* +sudo apt-mark hold docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestKubeadmDebian-simple.golden b/pkg/scripts/testdata/TestKubeadmDebian-simple.golden index d1506afc7..06ef50d10 100644 --- a/pkg/scripts/testdata/TestKubeadmDebian-simple.golden +++ b/pkg/scripts/testdata/TestKubeadmDebian-simple.golden @@ -90,9 +90,13 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get install \ --option "Dpkg::Options::=--force-confold" \ --no-install-recommends \ -y \ - docker-ce=5:19.03.* docker-ce-cli=5:19.03.* -sudo apt-mark hold docker-ce docker-ce-cli + docker-ce=5:19.03.* \ + docker-ce-cli=5:19.03.* \ + containerd.io=1.4.* +sudo apt-mark hold docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden index 697a43110..ee5274652 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden @@ -84,16 +84,20 @@ cat </dev/null -sudo yum versionlock delete docker-ce docker-ce-cli || true +sudo yum versionlock delete docker-ce docker-ce-cli containerd.io + +sudo yum install -y \ + docker-ce-19.03.* \ + docker-ce-cli-19.03.* \ + containerd.io-1.4.* +sudo yum versionlock add docker-ce docker-ce-cli containerd.io -sudo yum install -y docker-ce-19.03.* docker-ce-cli-19.03.* -sudo yum versionlock add docker-ce docker-ce-cli sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIDebian.golden b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIDebian.golden index 8a0d15963..f2b89eed0 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIDebian.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIDebian.golden @@ -86,15 +86,19 @@ echo "deb https://download.docker.com/linux/ubuntu bionic stable" | sudo tee /etc/apt/sources.list.d/docker.list sudo apt-get update -sudo apt-mark unhold docker-ce docker-ce-cli +sudo apt-mark unhold docker-ce docker-ce-cli containerd.io || true sudo DEBIAN_FRONTEND=noninteractive apt-get install \ --option "Dpkg::Options::=--force-confold" \ --no-install-recommends \ -y \ - docker-ce=5:19.03.* docker-ce-cli=5:19.03.* -sudo apt-mark hold docker-ce docker-ce-cli + docker-ce=5:19.03.* \ + docker-ce-cli=5:19.03.* \ + containerd.io=1.4.* +sudo apt-mark hold docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden index 7ab616940..fdaee5014 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden @@ -84,16 +84,20 @@ cat </dev/null -sudo yum versionlock delete docker-ce docker-ce-cli || true +sudo yum versionlock delete docker-ce docker-ce-cli containerd.io + +sudo yum install -y \ + docker-ce-19.03.* \ + docker-ce-cli-19.03.* \ + containerd.io-1.4.* +sudo yum versionlock add docker-ce docker-ce-cli containerd.io -sudo yum install -y docker-ce-19.03.* docker-ce-cli-19.03.* -sudo yum versionlock add docker-ce docker-ce-cli sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker diff --git a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlDebian.golden b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlDebian.golden index c9fa5ccde..3bee44aed 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlDebian.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlDebian.golden @@ -86,15 +86,19 @@ echo "deb https://download.docker.com/linux/ubuntu bionic stable" | sudo tee /etc/apt/sources.list.d/docker.list sudo apt-get update -sudo apt-mark unhold docker-ce docker-ce-cli +sudo apt-mark unhold docker-ce docker-ce-cli containerd.io || true sudo DEBIAN_FRONTEND=noninteractive apt-get install \ --option "Dpkg::Options::=--force-confold" \ --no-install-recommends \ -y \ - docker-ce=5:19.03.* docker-ce-cli=5:19.03.* -sudo apt-mark hold docker-ce docker-ce-cli + docker-ce=5:19.03.* \ + docker-ce-cli=5:19.03.* \ + containerd.io=1.4.* +sudo apt-mark hold docker-ce docker-ce-cli containerd.io + sudo systemctl daemon-reload +sudo systemctl enable --now containerd sudo systemctl enable --now docker From d9854319acf91ff2e7517e45f3daf5c4546721b1 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 31 May 2021 13:36:05 +0300 Subject: [PATCH 02/11] rename config subcommand functions to uniform Signed-off-by: Artiom Diomin --- pkg/cmd/config-images.go | 2 +- pkg/cmd/config.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/cmd/config-images.go b/pkg/cmd/config-images.go index bbb13ccdf..0203628f1 100644 --- a/pkg/cmd/config-images.go +++ b/pkg/cmd/config-images.go @@ -36,7 +36,7 @@ type listImagesOpts struct { Filter string `longflag:"filter"` } -func imagesCmd(rootFlags *pflag.FlagSet) *cobra.Command { +func configImagesCmd(rootFlags *pflag.FlagSet) *cobra.Command { cmd := &cobra.Command{ Use: "images", Short: "images manipulations", diff --git a/pkg/cmd/config.go b/pkg/cmd/config.go index f9cf8078f..17a99c0df 100644 --- a/pkg/cmd/config.go +++ b/pkg/cmd/config.go @@ -88,16 +88,16 @@ func configCmd(rootFlags *pflag.FlagSet) *cobra.Command { Short: "Commands for working with the KubeOneCluster configuration manifests", } - cmd.AddCommand(printCmd()) - cmd.AddCommand(migrateCmd(rootFlags)) - cmd.AddCommand(machinedeploymentsCmd(rootFlags)) - cmd.AddCommand(imagesCmd(rootFlags)) + cmd.AddCommand(configPrintCmd()) + cmd.AddCommand(configMigrateCmd(rootFlags)) + cmd.AddCommand(configMachinedeploymentsCmd(rootFlags)) + cmd.AddCommand(configImagesCmd(rootFlags)) return cmd } -// printCmd setups the print command -func printCmd() *cobra.Command { +// configPrintCmd setups the print command +func configPrintCmd() *cobra.Command { opts := &printOpts{} cmd := &cobra.Command{ Use: "print", @@ -177,8 +177,8 @@ func printCmd() *cobra.Command { return cmd } -// migrateCmd setups the migrate command -func migrateCmd(rootFlags *pflag.FlagSet) *cobra.Command { +// configMigrateCmd setups the migrate command +func configMigrateCmd(rootFlags *pflag.FlagSet) *cobra.Command { cmd := &cobra.Command{ Use: "migrate", Short: "Migrate the v1alpha1 KubeOneCluster manifest to the v1beta1 version", @@ -203,8 +203,8 @@ The new manifest is printed on the standard output. return cmd } -// machinedeploymentsCmd setups the machinedeployments command -func machinedeploymentsCmd(rootFlags *pflag.FlagSet) *cobra.Command { +// configMachinedeploymentsCmd setups the machinedeployments command +func configMachinedeploymentsCmd(rootFlags *pflag.FlagSet) *cobra.Command { cmd := &cobra.Command{ Use: "machinedeployments", Short: "Print the manifest for creating MachineDeployments", From c3b892ab19a2f8cae6a4e2fc792b4e348746d595 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 31 May 2021 14:27:29 +0300 Subject: [PATCH 03/11] kubeone migrate to-containerd Signed-off-by: Artiom Diomin --- pkg/cmd/migrate.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++ pkg/cmd/root.go | 1 + 2 files changed, 49 insertions(+) create mode 100644 pkg/cmd/migrate.go diff --git a/pkg/cmd/migrate.go b/pkg/cmd/migrate.go new file mode 100644 index 000000000..9ee27593b --- /dev/null +++ b/pkg/cmd/migrate.go @@ -0,0 +1,48 @@ +/* +Copyright 2019 The KubeOne Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "github.com/MakeNowJust/heredoc/v2" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +func migrateCmd(fs *pflag.FlagSet) *cobra.Command { + cmd := &cobra.Command{ + Use: "migrate", + Short: "Commands for running different migrations", + } + + cmd.AddCommand(migrateToContainerdCmd(fs)) + return cmd +} + +func migrateToContainerdCmd(fs *pflag.FlagSet) *cobra.Command { + return &cobra.Command{ + Use: "to-containerd", + Short: "Migrate live cluster from docker to containerd", + Long: heredoc.Doc(` + + Following the dockershim deprecation https://kubernetes.io/blog/2020/12/02/dockershim-faq/ + this command helps to migrate Container Runtime to ContainerD. + `), + RunE: func(_ *cobra.Command, args []string) error { + return nil + }, + } +} diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index f389653ea..b004bb8f4 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -117,6 +117,7 @@ func newRoot() *cobra.Command { versionCmd(), statusCmd(fs), proxyCmd(fs), + migrateCmd(fs), completionCmd(rootCmd), documentCmd(rootCmd), ) From f9e0f21c74f5181552778171f7dd80b03be2b1a2 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 31 May 2021 18:11:47 +0300 Subject: [PATCH 04/11] WithContainerDMigration task Signed-off-by: Artiom Diomin --- pkg/cmd/migrate.go | 21 +++++++++++++++++++-- pkg/tasks/probes.go | 11 +++++++++-- pkg/tasks/tasks.go | 7 +++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/pkg/cmd/migrate.go b/pkg/cmd/migrate.go index 9ee27593b..a67eea79b 100644 --- a/pkg/cmd/migrate.go +++ b/pkg/cmd/migrate.go @@ -18,8 +18,11 @@ package cmd import ( "github.com/MakeNowJust/heredoc/v2" + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" + + "k8c.io/kubeone/pkg/tasks" ) func migrateCmd(fs *pflag.FlagSet) *cobra.Command { @@ -41,8 +44,22 @@ func migrateToContainerdCmd(fs *pflag.FlagSet) *cobra.Command { Following the dockershim deprecation https://kubernetes.io/blog/2020/12/02/dockershim-faq/ this command helps to migrate Container Runtime to ContainerD. `), - RunE: func(_ *cobra.Command, args []string) error { - return nil + RunE: func(_ *cobra.Command, _ []string) error { + gopts, err := persistentGlobalOptions(fs) + if err != nil { + return errors.Wrap(err, "unable to get global flags") + } + + return runMigrateToContainerd(gopts) }, } } + +func runMigrateToContainerd(opts *globalOptions) error { + s, err := opts.BuildState() + if err != nil { + return errors.Wrap(err, "failed to initialize State") + } + + return errors.Wrap(tasks.WithContainerDMigration(nil).Run(s), "failed to get cluster status") +} diff --git a/pkg/tasks/probes.go b/pkg/tasks/probes.go index 13c58f0b3..77a3ff293 100644 --- a/pkg/tasks/probes.go +++ b/pkg/tasks/probes.go @@ -54,7 +54,8 @@ func safeguard(s *state.State) error { return err } - configuredClusterContainerRuntime := s.Cluster.ContainerRuntime.String() + cr := s.Cluster.ContainerRuntime + configuredClusterContainerRuntime := cr.String() for _, node := range nodes.Items { if !s.Cluster.IsManagedNode(node.Name) { @@ -65,11 +66,17 @@ func safeguard(s *state.State) error { nodesContainerRuntime := strings.Split(node.Status.NodeInfo.ContainerRuntimeVersion, ":")[0] if nodesContainerRuntime != configuredClusterContainerRuntime { + errMsg := "Migration is not supported yet" + if cr.Containerd != nil { + errMsg = "Use `kubeone migrate to-containerd`" + } + return errors.Errorf( - "Container runtime on node %q is %q, but %q is configured. Migration is not supported yet.", + "Container runtime on node %q is %q, but %q is configured. %s.", node.Name, nodesContainerRuntime, configuredClusterContainerRuntime, + errMsg, ) } } diff --git a/pkg/tasks/tasks.go b/pkg/tasks/tasks.go index 1d6cafdaf..0a431ae39 100644 --- a/pkg/tasks/tasks.go +++ b/pkg/tasks/tasks.go @@ -292,6 +292,13 @@ func WithReset(t Tasks) Tasks { }...) } +func WithContainerDMigration(t Tasks) Tasks { + return WithHostnameOS(t). + append(Tasks{ + {Fn: kubeconfig.BuildKubernetesClientset, ErrMsg: "failed to build kubernetes clientset"}, + }...) +} + func WithClusterStatus(t Tasks) Tasks { return WithHostnameOS(t). append(Tasks{ From aeb35a8d29275ff22f0f7771f4d1f535a8d9e9cf Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Tue, 1 Jun 2021 23:39:08 +0300 Subject: [PATCH 05/11] Upgrade machine-controller to v1.30.0 Signed-off-by: Artiom Diomin --- pkg/templates/images/images.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/templates/images/images.go b/pkg/templates/images/images.go index b1da64af9..c414095e1 100644 --- a/pkg/templates/images/images.go +++ b/pkg/templates/images/images.go @@ -53,7 +53,7 @@ func baseResources() map[Resource]string { CalicoNode: "docker.io/calico/node:v3.16.5", DNSNodeCache: "k8s.gcr.io/k8s-dns-node-cache:1.15.13", Flannel: "quay.io/coreos/flannel:v0.13.0", - MachineController: "docker.io/kubermatic/machine-controller:v1.29.1", + MachineController: "docker.io/kubermatic/machine-controller:v1.30.0", MetricsServer: "k8s.gcr.io/metrics-server:v0.3.6", } } From 8bbe032f2281d744f43801d1479f9862c4e2ae6f Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Tue, 1 Jun 2021 23:39:39 +0300 Subject: [PATCH 06/11] Run containerd migrations tasks Signed-off-by: Artiom Diomin --- pkg/scripts/os.go | 28 +++++++++++++++ pkg/scripts/render.go | 78 +++++++++++++---------------------------- pkg/tasks/containerd.go | 51 +++++++++++++++++++++++++++ pkg/tasks/tasks.go | 15 ++++++++ 4 files changed, 118 insertions(+), 54 deletions(-) create mode 100644 pkg/tasks/containerd.go diff --git a/pkg/scripts/os.go b/pkg/scripts/os.go index 155d8d0ec..41cd3a9ee 100644 --- a/pkg/scripts/os.go +++ b/pkg/scripts/os.go @@ -16,7 +16,35 @@ limitations under the License. package scripts +import ( + "github.com/MakeNowJust/heredoc/v2" +) + const ( defaultKubernetesCNIVersion = "0.8.7" defaultCriToolsVersion = "1.21.0" ) + +var migrateToContainerdScriptTemplate = heredoc.Doc(` + sudo systemctl stop kubelet + sudo docker ps -q | xargs sudo docker stop + sudo docker ps -qa | xargs sudo docker rm + sudo systemctl disable --now docker + + {{ template "containerd-config" . }} + + source /var/lib/kubelet/kubeadm-flags.env + KUBELET_EXTRA_ARGS="$KUBELET_EXTRA_ARGS --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock" + + cat < Date: Thu, 3 Jun 2021 18:42:25 +0300 Subject: [PATCH 07/11] Parse and edit kubeadm-flags.env Signed-off-by: Artiom Diomin --- pkg/scripts/os.go | 16 +-- pkg/scripts/os_test.go | 31 +++++ ...igrateToContainerd-insecureRegistry.golden | 44 +++++++ .../TestMigrateToContainerd-simple.golden | 42 +++++++ pkg/tasks/containerd.go | 114 +++++++++++++++++- pkg/tasks/containerd_test.go | 99 +++++++++++++++ pkg/tasks/tasks.go | 10 +- 7 files changed, 346 insertions(+), 10 deletions(-) create mode 100644 pkg/scripts/testdata/TestMigrateToContainerd-insecureRegistry.golden create mode 100644 pkg/scripts/testdata/TestMigrateToContainerd-simple.golden create mode 100644 pkg/tasks/containerd_test.go diff --git a/pkg/scripts/os.go b/pkg/scripts/os.go index 41cd3a9ee..6bc416434 100644 --- a/pkg/scripts/os.go +++ b/pkg/scripts/os.go @@ -27,18 +27,18 @@ const ( var migrateToContainerdScriptTemplate = heredoc.Doc(` sudo systemctl stop kubelet - sudo docker ps -q | xargs sudo docker stop - sudo docker ps -qa | xargs sudo docker rm + sudo docker ps -q | xargs sudo docker stop || true + sudo docker ps -qa | xargs sudo docker rm || true sudo systemctl disable --now docker {{ template "containerd-config" . }} - source /var/lib/kubelet/kubeadm-flags.env - KUBELET_EXTRA_ARGS="$KUBELET_EXTRA_ARGS --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock" - - cat < Date: Fri, 4 Jun 2021 20:02:22 +0300 Subject: [PATCH 08/11] Patch kubeadm cri-socket in Node annotations Signed-off-by: Artiom Diomin --- go.mod | 1 + go.sum | 6 ++- pkg/scripts/os.go | 1 - ...igrateToContainerd-insecureRegistry.golden | 5 +-- .../TestMigrateToContainerd-simple.golden | 5 +-- pkg/tasks/containerd.go | 37 ++++++++++++++++++- pkg/tasks/tasks.go | 3 +- 7 files changed, 46 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 0696c4a45..182041202 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/aws/aws-sdk-go v1.36.2 github.com/docker/distribution v2.7.1+incompatible github.com/dominodatalab/os-release v0.0.0-20190522011736-bcdb4a3e3c2f + github.com/gogo/protobuf v1.3.2 // indirect github.com/google/go-cmp v0.5.4 github.com/imdario/mergo v0.3.11 github.com/koron-go/prefixw v0.0.0-20181013140428-271b207a7572 diff --git a/go.sum b/go.sum index 3823b3e9f..4a087b4df 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,9 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -512,6 +513,7 @@ github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-2019 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1113,6 +1115,7 @@ golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -1122,6 +1125,7 @@ golang.org/x/tools v0.0.0-20201105220310-78b158585360/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/scripts/os.go b/pkg/scripts/os.go index 6bc416434..5ccd45ce7 100644 --- a/pkg/scripts/os.go +++ b/pkg/scripts/os.go @@ -29,7 +29,6 @@ var migrateToContainerdScriptTemplate = heredoc.Doc(` sudo systemctl stop kubelet sudo docker ps -q | xargs sudo docker stop || true sudo docker ps -qa | xargs sudo docker rm || true - sudo systemctl disable --now docker {{ template "containerd-config" . }} diff --git a/pkg/scripts/testdata/TestMigrateToContainerd-insecureRegistry.golden b/pkg/scripts/testdata/TestMigrateToContainerd-insecureRegistry.golden index 75442baae..1c2a2ad54 100644 --- a/pkg/scripts/testdata/TestMigrateToContainerd-insecureRegistry.golden +++ b/pkg/scripts/testdata/TestMigrateToContainerd-insecureRegistry.golden @@ -1,9 +1,8 @@ set -xeu pipefail export "PATH=$PATH:/sbin:/usr/local/bin:/opt/bin" sudo systemctl stop kubelet -sudo docker ps -q | xargs sudo docker stop -sudo docker ps -qa | xargs sudo docker rm -sudo systemctl disable --now docker +sudo docker ps -q | xargs sudo docker stop || true +sudo docker ps -qa | xargs sudo docker rm || true cat < Date: Sat, 5 Jun 2021 17:04:07 +0300 Subject: [PATCH 09/11] Fix linter issues Signed-off-by: Artiom Diomin --- pkg/tasks/containerd.go | 5 +++-- pkg/tasks/containerd_test.go | 1 + pkg/templates/canal/canal.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/tasks/containerd.go b/pkg/tasks/containerd.go index 7a1cc18de..8965d70b5 100644 --- a/pkg/tasks/containerd.go +++ b/pkg/tasks/containerd.go @@ -63,6 +63,7 @@ func patchCRISocketAnnotation(s *state.State) error { } for _, node := range nodes.Items { + node := node if socketPath, found := node.Annotations[kubeadmCRISocket]; found { if socketPath != "/var/run/dockershim.sock" { continue @@ -153,7 +154,7 @@ func unmarshalKubeletFlags(buf []byte) (map[string]string, error) { // throw away KUBELET_KUBEADM_ARGS= s1 := strings.SplitN(string(buf), "=", 2) if len(s1) != 2 { - return nil, errors.New("can't parse: wrong split lenght") + return nil, errors.New("can't parse: wrong split length") } envValue := strings.Trim(s1[1], `"`) @@ -163,7 +164,7 @@ func unmarshalKubeletFlags(buf []byte) (map[string]string, error) { for _, flg := range flagsvalues { fl := strings.Split(flg, "=") if len(fl) != 2 { - return nil, errors.New("wrong split lenght") + return nil, errors.New("wrong split length") } kubeletflagsMap[fl[0]] = fl[1] } diff --git a/pkg/tasks/containerd_test.go b/pkg/tasks/containerd_test.go index 03ac6c5c4..c2687b9f5 100644 --- a/pkg/tasks/containerd_test.go +++ b/pkg/tasks/containerd_test.go @@ -42,6 +42,7 @@ func Test_marshalKubeletFlags(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { if got := marshalKubeletFlags(tt.args.kubeletflags); !reflect.DeepEqual(got, tt.want) { t.Errorf("marshalKubeletFlags() = \n%s, want\n%s", got, tt.want) diff --git a/pkg/templates/canal/canal.go b/pkg/templates/canal/canal.go index 1be5658f3..e953fa780 100644 --- a/pkg/templates/canal/canal.go +++ b/pkg/templates/canal/canal.go @@ -137,7 +137,7 @@ func Deploy(s *state.State) error { flannelImage := s.Images.Get(images.Flannel) crds := canalCRDs() - k8sobjects := append(crds, + k8sobjects := append(crds, //nolint:gocritic // RBAC calicoKubeControllersClusterRole(), calicoNodeClusterRole(), From 3de0154617dde3b9cc4ae52d126af41b993458c6 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 7 Jun 2021 16:31:40 +0300 Subject: [PATCH 10/11] Add a link to docs on how to rolling restart machinedeployments Signed-off-by: Artiom Diomin --- pkg/tasks/tasks.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/tasks/tasks.go b/pkg/tasks/tasks.go index 448237080..0be03d5a9 100644 --- a/pkg/tasks/tasks.go +++ b/pkg/tasks/tasks.go @@ -312,7 +312,8 @@ func WithContainerDMigration(t Tasks) Tasks { return err } - s.Logger.Warn("roll-over your machineDeployments to get containerd") + s.Logger.Warn("Now please rolling restart your machineDeployments to get containerd") + s.Logger.Warn("see more at: https://docs.kubermatic.com/kubeone/v1.3/tutorials/rollout_machinedeployment/") return nil }, ErrMsg: "failed to ensure machine-controller", From ac2acb99528d864143461ca0e2b4ac5551cc7589 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 7 Jun 2021 19:23:46 +0300 Subject: [PATCH 11/11] Enable flatcar containerd migration Signed-off-by: Artiom Diomin --- pkg/scripts/os.go | 7 ++- pkg/scripts/os_test.go | 21 ++++--- .../TestMigrateToContainerd-flatcat.golden | 7 +++ pkg/tasks/containerd.go | 55 +++++++++++++++---- pkg/tasks/tasks.go | 2 +- 5 files changed, 70 insertions(+), 22 deletions(-) create mode 100644 pkg/scripts/testdata/TestMigrateToContainerd-flatcat.golden diff --git a/pkg/scripts/os.go b/pkg/scripts/os.go index 5ccd45ce7..d1a6c208f 100644 --- a/pkg/scripts/os.go +++ b/pkg/scripts/os.go @@ -30,7 +30,9 @@ var migrateToContainerdScriptTemplate = heredoc.Doc(` sudo docker ps -q | xargs sudo docker stop || true sudo docker ps -qa | xargs sudo docker rm || true + {{ if .GENERATE_CONTAINERD_CONFIG -}} {{ template "containerd-config" . }} + {{- end }} {{- /* /var/lib/kubelet/kubeadm-flags.env should be modified by the caller of @@ -42,8 +44,9 @@ var migrateToContainerdScriptTemplate = heredoc.Doc(` sudo systemctl restart kubelet `) -func MigrateToContainerd(insecureRegistry string) (string, error) { +func MigrateToContainerd(insecureRegistry string, generateContainerdConfig bool) (string, error) { return Render(migrateToContainerdScriptTemplate, Data{ - "INSECURE_REGISTRY": insecureRegistry, + "INSECURE_REGISTRY": insecureRegistry, + "GENERATE_CONTAINERD_CONFIG": generateContainerdConfig, }) } diff --git a/pkg/scripts/os_test.go b/pkg/scripts/os_test.go index b73f4376b..c8f24013e 100644 --- a/pkg/scripts/os_test.go +++ b/pkg/scripts/os_test.go @@ -165,23 +165,30 @@ func TestMigrateToContainerd(t *testing.T) { t.Parallel() tests := []struct { - name string - insecureRegistry string - err error + name string + insecureRegistry string + generateContainerdConfig bool + err error }{ { - name: "simple", + name: "simple", + generateContainerdConfig: true, + }, + { + name: "flatcat", + generateContainerdConfig: false, }, { - name: "insecureRegistry", - insecureRegistry: "some.registry", + name: "insecureRegistry", + insecureRegistry: "some.registry", + generateContainerdConfig: true, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := MigrateToContainerd(tt.insecureRegistry) + got, err := MigrateToContainerd(tt.insecureRegistry, tt.generateContainerdConfig) if err != tt.err { t.Errorf("MigrateToContainerd() error = %v, wantErr %v", err, tt.err) return diff --git a/pkg/scripts/testdata/TestMigrateToContainerd-flatcat.golden b/pkg/scripts/testdata/TestMigrateToContainerd-flatcat.golden new file mode 100644 index 000000000..35073299b --- /dev/null +++ b/pkg/scripts/testdata/TestMigrateToContainerd-flatcat.golden @@ -0,0 +1,7 @@ +set -xeu pipefail +export "PATH=$PATH:/sbin:/usr/local/bin:/opt/bin" +sudo systemctl stop kubelet +sudo docker ps -q | xargs sudo docker stop || true +sudo docker ps -qa | xargs sudo docker rm || true + +sudo systemctl restart kubelet diff --git a/pkg/tasks/containerd.go b/pkg/tasks/containerd.go index 8965d70b5..f8b2cbbc4 100644 --- a/pkg/tasks/containerd.go +++ b/pkg/tasks/containerd.go @@ -32,6 +32,7 @@ import ( "k8c.io/kubeone/pkg/state" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -117,22 +118,20 @@ func migrateToContainerdTask(s *state.State, node *kubeone.HostConfig, conn ssh. return errors.New("file is not writable") } - err = fw.Truncate(0) - if err != nil { + if err = fw.Truncate(0); err != nil { return err } - _, err = fw.Seek(0, io.SeekStart) - if err != nil { + if _, err = fw.Seek(0, io.SeekStart); err != nil { return err } - _, err = io.Copy(fw, bytes.NewBuffer(buf)) - if err != nil { + if _, err = io.Copy(fw, bytes.NewBuffer(buf)); err != nil { return err } - migrateScript, err := scripts.MigrateToContainerd(s.Cluster.RegistryConfiguration.InsecureRegistryAddress()) + generateContainerdConfig := node.OperatingSystem != kubeone.OperatingSystemNameFlatcar + migrateScript, err := scripts.MigrateToContainerd(s.Cluster.RegistryConfiguration.InsecureRegistryAddress(), generateContainerdConfig) if err != nil { return err } @@ -142,12 +141,44 @@ func migrateToContainerdTask(s *state.State, node *kubeone.HostConfig, conn ssh. return err } - // TODO(kron4eg): replace with better waiting polling - sleepTime := 30 * time.Second - s.Logger.Infof("Waiting %s to ensure main control plane components are up...", sleepTime) - time.Sleep(sleepTime) + s.Logger.Infof("Waiting all pods on %q to became Ready...", node.Hostname) + err = wait.Poll(10*time.Second, 10*time.Minute, func() (bool, error) { + var podsList corev1.PodList - return nil + if perr := s.DynamicClient.List(s.Context, &podsList); perr != nil { + return false, err + } + + for _, pod := range podsList.Items { + if pod.Spec.NodeName != node.Hostname { + continue + } + + if pod.Status.Phase != corev1.PodRunning { + s.Logger.Debugf("Pod %s/%s is not running", pod.Namespace, pod.Name) + return false, nil + } + + for _, podcond := range pod.Status.Conditions { + if podcond.Type == corev1.PodReady && podcond.Status != corev1.ConditionTrue { + s.Logger.Debugf("Pod %s/%s is not ready", pod.Namespace, pod.Name) + return false, nil + } + } + + for _, condstatus := range pod.Status.ContainerStatuses { + if !condstatus.Ready { + s.Logger.Debugf("Container %s in pod %s/%s is not ready", condstatus.Name, pod.Namespace, pod.Name) + return false, nil + } + } + } + + s.Logger.Debugf("All pods on %s Node are ready", node.Hostname) + return true, nil + }) + + return err } func unmarshalKubeletFlags(buf []byte) (map[string]string, error) { diff --git a/pkg/tasks/tasks.go b/pkg/tasks/tasks.go index 0be03d5a9..ed26a2957 100644 --- a/pkg/tasks/tasks.go +++ b/pkg/tasks/tasks.go @@ -313,7 +313,7 @@ func WithContainerDMigration(t Tasks) Tasks { } s.Logger.Warn("Now please rolling restart your machineDeployments to get containerd") - s.Logger.Warn("see more at: https://docs.kubermatic.com/kubeone/v1.3/tutorials/rollout_machinedeployment/") + s.Logger.Warn("see more at: https://docs.kubermatic.com/kubeone/v1.3/cheat_sheets/rollout_machinedeployment/") return nil }, ErrMsg: "failed to ensure machine-controller",