From 81afbfb8a2d0e72c773bd7352c6c62feb30925ec Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Tue, 26 Jan 2021 15:28:15 +0200 Subject: [PATCH 01/21] Update machine-controller to v1.24.3 (#1227) * Update machine-controller to v1.24.3 And enable -node-controller-runtime Signed-off-by: Artiom Diomin * Use ubuntu-18.04 for kubernetes 1.16 e2e tests Signed-off-by: Artiom Diomin --- .prow.yaml | 4 ++++ pkg/templates/machinecontroller/deployment.go | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.prow.yaml b/.prow.yaml index 8982af152..5ba67c69a 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -147,6 +147,8 @@ presubmits: value: "1.16.15" - name: TEST_SET value: "conformance" + - name: TF_VAR_ami + value: ami-08985edfecbbbcf52 resources: requests: cpu: 1 @@ -833,6 +835,8 @@ presubmits: value: "1.17.12" - name: TEST_SET value: "upgrades" + - name: TF_VAR_ami + value: ami-08985edfecbbbcf52 - name: pull-kubeone-e2e-aws-upgrade-containerd-1.16-1.17 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" diff --git a/pkg/templates/machinecontroller/deployment.go b/pkg/templates/machinecontroller/deployment.go index ecd06b8a8..a20533126 100644 --- a/pkg/templates/machinecontroller/deployment.go +++ b/pkg/templates/machinecontroller/deployment.go @@ -48,7 +48,7 @@ const ( MachineControllerAppLabelValue = "machine-controller" MachineControllerImageRegistry = "docker.io" MachineControllerImage = "/kubermatic/machine-controller:" - MachineControllerTag = "v1.23.1" + MachineControllerTag = "v1.24.3" ) func CRDs() []runtime.Object { @@ -760,6 +760,13 @@ func machineControllerDeployment(cluster *kubeoneapi.KubeOneCluster, credentials "-node-csr-approver", } + switch { + case cluster.ContainerRuntime.Containerd != nil: + fallthrough + case cluster.ContainerRuntime.Docker != nil: + args = append(args, "-node-container-runtime", cluster.ContainerRuntime.String()) + } + if cluster.Proxy.HTTP != "" { args = append(args, "-node-http-proxy", cluster.Proxy.HTTP) } From 2568525f7e6847f715ea992257153be35954679e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Wed, 27 Jan 2021 12:17:15 +0100 Subject: [PATCH 02/21] Add containerRuntime API to the full config (#1229) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/cmd/config.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/cmd/config.go b/pkg/cmd/config.go index bf84e8109..b81a29788 100644 --- a/pkg/cmd/config.go +++ b/pkg/cmd/config.go @@ -565,6 +565,27 @@ cloudProvider: # Path to file that will be uploaded and used as custom '--cloud-config' file. cloudConfig: "{{ .CloudProviderCloudCfg }}" +# Controls which container runtime will be installed on instances. +# By default: +# * Docker will be installed for Kubernetes clusters up to 1.20 +# * containerd will be installed for Kubernetes clusters 1.21+ +# Currently, it's not possible to migrate existing clusters from one to another +# container runtime, however, migration from Docker to containerd is planned +# for one of the upcoming KubeOne releases. +# Only one container runtime can be present at the time. +# +# Note: Kubernetes has announced deprecation of Docker (dockershim) support. +# It's expected that the Docker support will be removed in Kubernetes 1.22. +# It's highly advised to use containerd for all newly created clusters. +containerRuntime: + # Installs containerd container runtime. + # Default for 1.21+ Kubernetes clusters. + # containerd: {} + # Installs Docker container runtime. + # Default for Kubernetes clusters up to 1.20. + # This option will be removed once Kubernetes 1.21 reaches EOL. + # docker: {} + features: # Enable the PodNodeSelector admission plugin in API server. # More info: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podnodeselector From a039926b9838fcbd79424ea0d48c0bb299ee7278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Wed, 27 Jan 2021 12:31:16 +0100 Subject: [PATCH 03/21] Add the changelog for the v1.2.0-beta.0 release (#1230) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- CHANGELOG.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 115e861f2..ffca720dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,65 @@ # Changelog +# [v1.2.0-beta.0](https://github.com/kubermatic/kubeone/releases/tag/v1.2.0-beta.0) - 2021-01-27 + +## Attention Needed + +* Kubernetes has announced deprecation of the Docker (dockershim) support in + the Kubernetes 1.20 release. It's expected that Docker support will be + removed in Kubernetes 1.22 + * All newly created clusters running Kubernetes 1.21+ will be provisioned + with containerd instead of Docker + * Automated migration from Docker to containerd is currently not available, + but is planned for one of the upcoming KubeOne releases + * We highly recommend using containerd instead of Docker for all newly + created clusters. You can opt-in to use containerd instead of Docker by + adding `containerRuntime` configuration to your KubeOne configuration + manifest: + ```yaml + containerRuntime: + containerd: {} + ``` + For the configuration file reference, run `kubeone config print --full`. + + +## Known Issues + +* Provisioning Kubernetes 1.20 clusters results with one of the control plane + nodes being unhealthy/broken for the first 5-10 minutes after provisioning + the cluster. This causes KubeOne to fail to create MachineDeployment objects + because the `machine-controller-webhook` service can't be found. Also, one of + the NodeLocalDNS pods might get stuck in the crash loop. + * KubeOne currently still doesn't support Kubernetes 1.20. We do **not** + recommend provisioning 1.20 clusters or upgrading existing clusters to + Kubernetes 1.20 + * We're currently investigating the issue. You can follow the progress + in the issue [#1222](https://github.com/kubermatic/kubeone/issues/1222) + +## Added + +* Add support for containerd container runtime ([#1180](https://github.com/kubermatic/kubeone/pull/1180), [#1188](https://github.com/kubermatic/kubeone/pull/1188), [#1190](https://github.com/kubermatic/kubeone/pull/1190), [#1205](https://github.com/kubermatic/kubeone/pull/1205), [#1227](https://github.com/kubermatic/kubeone/pull/1227), [#1229](https://github.com/kubermatic/kubeone/pull/1229)) + * Kubernetes has announced deprecation of the Docker (dockershim) support in + the Kubernetes 1.20 release. It's expected that Docker support will be + removed in Kubernetes 1.22 + * All newly created clusters running Kubernetes 1.21+ will default to + containerd instead of Docker + * Automated migration from Docker to containerd is currently not available, + but is planned for one of the upcoming KubeOne releases + +## Changed + +### Bug Fixes + +* Fix wrong legacy Docker version on RPM systems ([#1191](https://github.com/kubermatic/kubeone/pull/1191)) + +### Terraform Configs + +* Replace GoBetween load-balancer in vSphere Terraform example by keepalived ([#1217](https://github.com/kubermatic/kubeone/pull/1217)) + +### Addons + +* Fix DNS resolution issues for the Backups addon ([#1179](https://github.com/kubermatic/kubeone/pull/1179)) + # [v1.2.0-alpha.0](https://github.com/kubermatic/kubeone/releases/tag/v1.2.0-alpha.0) - 2020-11-27 ## Added From 46c90bdf01d14e85775d777a7b775a0fbfd84501 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Fri, 29 Jan 2021 15:06:25 +0200 Subject: [PATCH 04/21] Remove CoreOS (#1232) Signed-off-by: Artiom Diomin --- examples/terraform/azure/variables.tf | 1 - examples/terraform/digitalocean/variables.tf | 1 - examples/terraform/gce/variables.tf | 1 - examples/terraform/hetzner/variables.tf | 1 - examples/terraform/openstack/variables.tf | 1 - examples/terraform/packet/variables.tf | 1 - examples/terraform/vsphere/variables.tf | 1 - pkg/apis/kubeone/types.go | 3 +-- pkg/apis/kubeone/v1beta1/types.go | 3 +-- pkg/scripts/os.go | 24 +++++++++---------- pkg/scripts/os_test.go | 24 +++++++++---------- ...golden => TestKubeadmFlatcar-force.golden} | 0 ...tKubeadmFlatcar-overwrite_registry.golden} | 0 ...latcar-overwrite_registry_insecure.golden} | 0 ...olden => TestKubeadmFlatcar-simple.golden} | 0 ...olden => TestRemoveBinariesFlatcar.golden} | 0 ...=> TestUpgradeKubeadmAndCNIFlatcar.golden} | 0 ...estUpgradeKubeletAndKubectlFlatcar.golden} | 0 pkg/tasks/kubernetes_binaries.go | 14 +++++------ pkg/tasks/prerequisites.go | 7 +++--- pkg/tasks/probes.go | 6 ++--- pkg/tasks/reset.go | 7 +++--- test/e2e/os.go | 10 ++++---- 23 files changed, 46 insertions(+), 59 deletions(-) rename pkg/scripts/testdata/{TestKubeadmCoreOS-force.golden => TestKubeadmFlatcar-force.golden} (100%) rename pkg/scripts/testdata/{TestKubeadmCoreOS-overwrite_registry.golden => TestKubeadmFlatcar-overwrite_registry.golden} (100%) rename pkg/scripts/testdata/{TestKubeadmCoreOS-overwrite_registry_insecure.golden => TestKubeadmFlatcar-overwrite_registry_insecure.golden} (100%) rename pkg/scripts/testdata/{TestKubeadmCoreOS-simple.golden => TestKubeadmFlatcar-simple.golden} (100%) rename pkg/scripts/testdata/{TestRemoveBinariesCoreOS.golden => TestRemoveBinariesFlatcar.golden} (100%) rename pkg/scripts/testdata/{TestUpgradeKubeadmAndCNICoreOS.golden => TestUpgradeKubeadmAndCNIFlatcar.golden} (100%) rename pkg/scripts/testdata/{TestUpgradeKubeletAndKubectlCoreOS.golden => TestUpgradeKubeletAndKubectlFlatcar.golden} (100%) diff --git a/examples/terraform/azure/variables.tf b/examples/terraform/azure/variables.tf index ee3b63694..6a685de56 100644 --- a/examples/terraform/azure/variables.tf +++ b/examples/terraform/azure/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/digitalocean/variables.tf b/examples/terraform/digitalocean/variables.tf index a321fa5d8..8b7318130 100644 --- a/examples/terraform/digitalocean/variables.tf +++ b/examples/terraform/digitalocean/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/gce/variables.tf b/examples/terraform/gce/variables.tf index db4dddcb7..b6ebfd1b1 100644 --- a/examples/terraform/gce/variables.tf +++ b/examples/terraform/gce/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/hetzner/variables.tf b/examples/terraform/hetzner/variables.tf index 261d3904b..c8bf98477 100644 --- a/examples/terraform/hetzner/variables.tf +++ b/examples/terraform/hetzner/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/openstack/variables.tf b/examples/terraform/openstack/variables.tf index 4b7c31f43..2a3f0d5a9 100644 --- a/examples/terraform/openstack/variables.tf +++ b/examples/terraform/openstack/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/packet/variables.tf b/examples/terraform/packet/variables.tf index 7a89f9c32..18b575466 100644 --- a/examples/terraform/packet/variables.tf +++ b/examples/terraform/packet/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/vsphere/variables.tf b/examples/terraform/vsphere/variables.tf index 6f3f0724c..696a63314 100644 --- a/examples/terraform/vsphere/variables.tf +++ b/examples/terraform/vsphere/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/pkg/apis/kubeone/types.go b/pkg/apis/kubeone/types.go index b1d558ef7..126c88255 100644 --- a/pkg/apis/kubeone/types.go +++ b/pkg/apis/kubeone/types.go @@ -78,12 +78,11 @@ type ContainerRuntimeContainerd struct{} // OperatingSystemName defines the operating system used on instances type OperatingSystemName string -var ( +const ( OperatingSystemNameUbuntu OperatingSystemName = "ubuntu" OperatingSystemNameCentOS OperatingSystemName = "centos" OperatingSystemNameRHEL OperatingSystemName = "rhel" OperatingSystemNameAmazon OperatingSystemName = "amzn" - OperatingSystemNameCoreOS OperatingSystemName = "coreos" OperatingSystemNameFlatcar OperatingSystemName = "flatcar" OperatingSystemNameUnknown OperatingSystemName = "" ) diff --git a/pkg/apis/kubeone/v1beta1/types.go b/pkg/apis/kubeone/v1beta1/types.go index 87efb9187..21647635b 100644 --- a/pkg/apis/kubeone/v1beta1/types.go +++ b/pkg/apis/kubeone/v1beta1/types.go @@ -78,12 +78,11 @@ type ContainerRuntimeContainerd struct{} // OperatingSystemName defines the operating system used on instances type OperatingSystemName string -var ( +const ( OperatingSystemNameUbuntu OperatingSystemName = "ubuntu" OperatingSystemNameCentOS OperatingSystemName = "centos" OperatingSystemNameRHEL OperatingSystemName = "rhel" OperatingSystemNameAmazon OperatingSystemName = "amzn" - OperatingSystemNameCoreOS OperatingSystemName = "coreos" OperatingSystemNameFlatcar OperatingSystemName = "flatcar" OperatingSystemNameUnknown OperatingSystemName = "" ) diff --git a/pkg/scripts/os.go b/pkg/scripts/os.go index c513d2e8f..d0629bc1f 100644 --- a/pkg/scripts/os.go +++ b/pkg/scripts/os.go @@ -307,7 +307,7 @@ sudo systemctl restart kubelet {{- end }} ` - kubeadmCoreOSTemplate = ` + kubeadmFlatcarTemplate = ` source /etc/kubeone/proxy-env {{ template "detect-host-cpu-architecture" }} @@ -400,7 +400,7 @@ sudo rm -f /etc/systemd/system/kubelet.service /etc/systemd/system/kubelet.servi sudo systemctl daemon-reload ` - removeBinariesCoreOSScriptTemplate = ` + removeBinariesFlatcarScriptTemplate = ` # Stop kubelet sudo systemctl stop kubelet || true # Remove CNI and binaries @@ -411,7 +411,7 @@ sudo rm -f /etc/systemd/system/kubelet.service /etc/systemd/system/kubelet.servi sudo systemctl daemon-reload ` - upgradeKubeadmAndCNICoreOSScriptTemplate = ` + upgradeKubeadmAndCNIFlatcarScriptTemplate = ` {{ template "detect-host-cpu-architecture" }} source /etc/kubeone/proxy-env @@ -433,7 +433,7 @@ sudo mv /var/tmp/kube-binaries/kubeadm . sudo chmod +x kubeadm ` - upgradeKubeletAndKubectlCoreOSScriptTemplate = ` + upgradeKubeletAndKubectlFlatcarScriptTemplate = ` source /etc/kubeone/proxy-env {{ template "detect-host-cpu-architecture" }} @@ -547,8 +547,8 @@ func KubeadmAmazonLinux(cluster *kubeone.KubeOneCluster, force bool) (string, er }) } -func KubeadmCoreOS(cluster *kubeone.KubeOneCluster) (string, error) { - return Render(kubeadmCoreOSTemplate, Data{ +func KubeadmFlatcar(cluster *kubeone.KubeOneCluster) (string, error) { + return Render(kubeadmFlatcarTemplate, Data{ "KUBERNETES_VERSION": cluster.Versions.Kubernetes, "KUBERNETES_CNI_VERSION": defaultKubernetesCNIVersion, "INSECURE_REGISTRY": cluster.RegistryConfiguration.InsecureRegistryAddress(), @@ -567,8 +567,8 @@ func RemoveBinariesAmazonLinux() (string, error) { return Render(removeBinariesAmazonLinuxScriptTemplate, Data{}) } -func RemoveBinariesCoreOS() (string, error) { - return Render(removeBinariesCoreOSScriptTemplate, nil) +func RemoveBinariesFlatcar() (string, error) { + return Render(removeBinariesFlatcarScriptTemplate, nil) } func UpgradeKubeadmAndCNIDebian(cluster *kubeone.KubeOneCluster) (string, error) { @@ -626,8 +626,8 @@ func UpgradeKubeadmAndCNIAmazonLinux(cluster *kubeone.KubeOneCluster) (string, e }) } -func UpgradeKubeadmAndCNICoreOS(k8sVersion string) (string, error) { - return Render(upgradeKubeadmAndCNICoreOSScriptTemplate, Data{ +func UpgradeKubeadmAndCNIFlatcar(k8sVersion string) (string, error) { + return Render(upgradeKubeadmAndCNIFlatcarScriptTemplate, Data{ "KUBERNETES_VERSION": k8sVersion, "KUBERNETES_CNI_VERSION": defaultKubernetesCNIVersion, }) @@ -691,8 +691,8 @@ func UpgradeKubeletAndKubectlAmazonLinux(cluster *kubeone.KubeOneCluster) (strin }) } -func UpgradeKubeletAndKubectlCoreOS(k8sVersion string) (string, error) { - return Render(upgradeKubeletAndKubectlCoreOSScriptTemplate, Data{ +func UpgradeKubeletAndKubectlFlatcar(k8sVersion string) (string, error) { + return Render(upgradeKubeletAndKubectlFlatcarScriptTemplate, Data{ "KUBERNETES_VERSION": k8sVersion, }) } diff --git a/pkg/scripts/os_test.go b/pkg/scripts/os_test.go index 007e1e910..9dc17028a 100644 --- a/pkg/scripts/os_test.go +++ b/pkg/scripts/os_test.go @@ -337,7 +337,7 @@ func TestKubeadmAmazonLinux(t *testing.T) { } } -func TestKubeadmCoreOS(t *testing.T) { +func TestKubeadmFlatcar(t *testing.T) { t.Parallel() type args struct { @@ -377,9 +377,9 @@ func TestKubeadmCoreOS(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := KubeadmCoreOS(&tt.args.cluster) + got, err := KubeadmFlatcar(&tt.args.cluster) if err != tt.err { - t.Errorf("KubeadmCoreOS() error = %v, wantErr %v", err, tt.err) + t.Errorf("KubeadmFlatcar() error = %v, wantErr %v", err, tt.err) return } @@ -424,12 +424,12 @@ func TestRemoveBinariesAmazonLinux(t *testing.T) { testhelper.DiffOutput(t, testhelper.FSGoldenName(t), got, *updateFlag) } -func TestRemoveBinariesCoreOS(t *testing.T) { +func TestRemoveBinariesFlatcar(t *testing.T) { t.Parallel() - got, err := RemoveBinariesCoreOS() + got, err := RemoveBinariesFlatcar() if err != nil { - t.Errorf("RemoveBinariesCoreOS() error = %v", err) + t.Errorf("RemoveBinariesFlatcar() error = %v", err) return } @@ -475,12 +475,12 @@ func TestUpgradeKubeadmAndCNIAmazonLinux(t *testing.T) { testhelper.DiffOutput(t, testhelper.FSGoldenName(t), got, *updateFlag) } -func TestUpgradeKubeadmAndCNICoreOS(t *testing.T) { +func TestUpgradeKubeadmAndCNIFlatcar(t *testing.T) { t.Parallel() - got, err := UpgradeKubeadmAndCNICoreOS("v1.17.4") + got, err := UpgradeKubeadmAndCNIFlatcar("v1.17.4") if err != nil { - t.Errorf("UpgradeKubeadmAndCNICoreOS() error = %v", err) + t.Errorf("UpgradeKubeadmAndCNIFlatcar() error = %v", err) return } @@ -526,12 +526,12 @@ func TestUpgradeKubeletAndKubectlAmazonLinux(t *testing.T) { testhelper.DiffOutput(t, testhelper.FSGoldenName(t), got, *updateFlag) } -func TestUpgradeKubeletAndKubectlCoreOS(t *testing.T) { +func TestUpgradeKubeletAndKubectlFlatcar(t *testing.T) { t.Parallel() - got, err := UpgradeKubeletAndKubectlCoreOS("v1.17.4") + got, err := UpgradeKubeletAndKubectlFlatcar("v1.17.4") if err != nil { - t.Errorf("UpgradeKubeletAndKubectlCoreOS() error = %v", err) + t.Errorf("UpgradeKubeletAndKubectlFlatcar() error = %v", err) return } diff --git a/pkg/scripts/testdata/TestKubeadmCoreOS-force.golden b/pkg/scripts/testdata/TestKubeadmFlatcar-force.golden similarity index 100% rename from pkg/scripts/testdata/TestKubeadmCoreOS-force.golden rename to pkg/scripts/testdata/TestKubeadmFlatcar-force.golden diff --git a/pkg/scripts/testdata/TestKubeadmCoreOS-overwrite_registry.golden b/pkg/scripts/testdata/TestKubeadmFlatcar-overwrite_registry.golden similarity index 100% rename from pkg/scripts/testdata/TestKubeadmCoreOS-overwrite_registry.golden rename to pkg/scripts/testdata/TestKubeadmFlatcar-overwrite_registry.golden diff --git a/pkg/scripts/testdata/TestKubeadmCoreOS-overwrite_registry_insecure.golden b/pkg/scripts/testdata/TestKubeadmFlatcar-overwrite_registry_insecure.golden similarity index 100% rename from pkg/scripts/testdata/TestKubeadmCoreOS-overwrite_registry_insecure.golden rename to pkg/scripts/testdata/TestKubeadmFlatcar-overwrite_registry_insecure.golden diff --git a/pkg/scripts/testdata/TestKubeadmCoreOS-simple.golden b/pkg/scripts/testdata/TestKubeadmFlatcar-simple.golden similarity index 100% rename from pkg/scripts/testdata/TestKubeadmCoreOS-simple.golden rename to pkg/scripts/testdata/TestKubeadmFlatcar-simple.golden diff --git a/pkg/scripts/testdata/TestRemoveBinariesCoreOS.golden b/pkg/scripts/testdata/TestRemoveBinariesFlatcar.golden similarity index 100% rename from pkg/scripts/testdata/TestRemoveBinariesCoreOS.golden rename to pkg/scripts/testdata/TestRemoveBinariesFlatcar.golden diff --git a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNICoreOS.golden b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIFlatcar.golden similarity index 100% rename from pkg/scripts/testdata/TestUpgradeKubeadmAndCNICoreOS.golden rename to pkg/scripts/testdata/TestUpgradeKubeadmAndCNIFlatcar.golden diff --git a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCoreOS.golden b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlFlatcar.golden similarity index 100% rename from pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCoreOS.golden rename to pkg/scripts/testdata/TestUpgradeKubeletAndKubectlFlatcar.golden diff --git a/pkg/tasks/kubernetes_binaries.go b/pkg/tasks/kubernetes_binaries.go index 448436f94..ce04788b8 100644 --- a/pkg/tasks/kubernetes_binaries.go +++ b/pkg/tasks/kubernetes_binaries.go @@ -27,8 +27,7 @@ import ( func upgradeKubeletAndKubectlBinaries(s *state.State, node kubeoneapi.HostConfig) error { return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ kubeoneapi.OperatingSystemNameUbuntu: upgradeKubeletAndKubectlBinariesDebian, - kubeoneapi.OperatingSystemNameCoreOS: upgradeKubeletAndKubectlBinariesCoreOS, - kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeletAndKubectlBinariesCoreOS, + kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeletAndKubectlBinariesFlatcar, kubeoneapi.OperatingSystemNameCentOS: upgradeKubeletAndKubectlBinariesCentOS, kubeoneapi.OperatingSystemNameAmazon: upgradeKubeletAndKubectlBinariesAmazonLinux, kubeoneapi.OperatingSystemNameRHEL: upgradeKubeletAndKubectlBinariesCentOS, @@ -38,8 +37,7 @@ func upgradeKubeletAndKubectlBinaries(s *state.State, node kubeoneapi.HostConfig func upgradeKubeadmAndCNIBinaries(s *state.State, node kubeoneapi.HostConfig) error { return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ kubeoneapi.OperatingSystemNameUbuntu: upgradeKubeadmAndCNIBinariesDebian, - kubeoneapi.OperatingSystemNameCoreOS: upgradeKubeadmAndCNIBinariesCoreOS, - kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeadmAndCNIBinariesCoreOS, + kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeadmAndCNIBinariesFlatcar, kubeoneapi.OperatingSystemNameCentOS: upgradeKubeadmAndCNIBinariesCentOS, kubeoneapi.OperatingSystemNameAmazon: upgradeKubeadmAndCNIBinariesAmazonLinux, kubeoneapi.OperatingSystemNameRHEL: upgradeKubeadmAndCNIBinariesCentOS, @@ -57,8 +55,8 @@ func upgradeKubeletAndKubectlBinariesDebian(s *state.State) error { return errors.WithStack(err) } -func upgradeKubeletAndKubectlBinariesCoreOS(s *state.State) error { - cmd, err := scripts.UpgradeKubeletAndKubectlCoreOS(s.Cluster.Versions.Kubernetes) +func upgradeKubeletAndKubectlBinariesFlatcar(s *state.State) error { + cmd, err := scripts.UpgradeKubeletAndKubectlFlatcar(s.Cluster.Versions.Kubernetes) if err != nil { return err } @@ -123,8 +121,8 @@ func upgradeKubeadmAndCNIBinariesAmazonLinux(s *state.State) error { return errors.WithStack(err) } -func upgradeKubeadmAndCNIBinariesCoreOS(s *state.State) error { - cmd, err := scripts.UpgradeKubeadmAndCNICoreOS(s.Cluster.Versions.Kubernetes) +func upgradeKubeadmAndCNIBinariesFlatcar(s *state.State) error { + cmd, err := scripts.UpgradeKubeadmAndCNIFlatcar(s.Cluster.Versions.Kubernetes) if err != nil { return err } diff --git a/pkg/tasks/prerequisites.go b/pkg/tasks/prerequisites.go index a6763b580..1658ed43d 100644 --- a/pkg/tasks/prerequisites.go +++ b/pkg/tasks/prerequisites.go @@ -86,9 +86,8 @@ func createEnvironmentFile(s *state.State) error { func installKubeadm(s *state.State, node kubeoneapi.HostConfig) error { return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ kubeoneapi.OperatingSystemNameUbuntu: installKubeadmDebian, - kubeoneapi.OperatingSystemNameCoreOS: installKubeadmCoreOS, kubeoneapi.OperatingSystemNameAmazon: installKubeadmAmazonLinux, - kubeoneapi.OperatingSystemNameFlatcar: installKubeadmCoreOS, + kubeoneapi.OperatingSystemNameFlatcar: installKubeadmFlatcar, kubeoneapi.OperatingSystemNameCentOS: installKubeadmCentOS, kubeoneapi.OperatingSystemNameRHEL: installKubeadmCentOS, }) @@ -127,8 +126,8 @@ func installKubeadmAmazonLinux(s *state.State) error { return errors.WithStack(err) } -func installKubeadmCoreOS(s *state.State) error { - cmd, err := scripts.KubeadmCoreOS(s.Cluster) +func installKubeadmFlatcar(s *state.State) error { + cmd, err := scripts.KubeadmFlatcar(s.Cluster) if err != nil { return err } diff --git a/pkg/tasks/probes.go b/pkg/tasks/probes.go index 3a1661586..ebe96a192 100644 --- a/pkg/tasks/probes.go +++ b/pkg/tasks/probes.go @@ -110,8 +110,7 @@ func runProbes(s *state.State) error { } for _, host := range s.Cluster.ControlPlane.Hosts { - switch host.OperatingSystem { - case kubeoneapi.OperatingSystemNameFlatcar, kubeoneapi.OperatingSystemNameCoreOS: + if host.OperatingSystem == kubeoneapi.OperatingSystemNameFlatcar { s.Cluster.ContainerRuntime.Docker = &kubeoneapi.ContainerRuntimeDocker{} } } @@ -189,8 +188,7 @@ func investigateHost(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Conne containerRuntimeOpts := []systemdUnitInfoOpt{withComponentVersion(versionCmdGenerator)} - switch h.Config.OperatingSystem { - case kubeoneapi.OperatingSystemNameCoreOS, kubeoneapi.OperatingSystemNameFlatcar: + if h.Config.OperatingSystem == kubeoneapi.OperatingSystemNameFlatcar { // Flatcar is special containerRuntimeOpts = []systemdUnitInfoOpt{withFlatcarContainerRuntimeVersion} } diff --git a/pkg/tasks/reset.go b/pkg/tasks/reset.go index 0b807ef19..01be2a6b6 100644 --- a/pkg/tasks/reset.go +++ b/pkg/tasks/reset.go @@ -130,8 +130,7 @@ func removeBinaries(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connec return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ kubeoneapi.OperatingSystemNameUbuntu: removeBinariesDebian, - kubeoneapi.OperatingSystemNameCoreOS: removeBinariesCoreOS, - kubeoneapi.OperatingSystemNameFlatcar: removeBinariesCoreOS, + kubeoneapi.OperatingSystemNameFlatcar: removeBinariesFlatcar, kubeoneapi.OperatingSystemNameCentOS: removeBinariesCentOS, kubeoneapi.OperatingSystemNameRHEL: removeBinariesCentOS, kubeoneapi.OperatingSystemNameAmazon: removeBinariesAmazonLinux, @@ -168,8 +167,8 @@ func removeBinariesAmazonLinux(s *state.State) error { return errors.WithStack(err) } -func removeBinariesCoreOS(s *state.State) error { - cmd, err := scripts.RemoveBinariesCoreOS() +func removeBinariesFlatcar(s *state.State) error { + cmd, err := scripts.RemoveBinariesFlatcar() if err != nil { return err } diff --git a/test/e2e/os.go b/test/e2e/os.go index 5e757043e..d4b73101f 100644 --- a/test/e2e/os.go +++ b/test/e2e/os.go @@ -29,7 +29,6 @@ const ( OperatingSystemUbuntu OperatingSystem = "ubuntu" OperatingSystemCentOS7 OperatingSystem = "centos7" OperatingSystemCentOS8 OperatingSystem = "centos" - OperatingSystemCoreOS OperatingSystem = "coreos" OperatingSystemFlatcar OperatingSystem = "flatcar" OperatingSystemDefault OperatingSystem = "" ) @@ -40,8 +39,11 @@ const ( func ValidateOperatingSystem(osName string) error { switch OperatingSystem(osName) { - case OperatingSystemUbuntu, OperatingSystemCoreOS, OperatingSystemFlatcar, - OperatingSystemCentOS7, OperatingSystemCentOS8, OperatingSystemDefault: + case OperatingSystemUbuntu, + OperatingSystemFlatcar, + OperatingSystemCentOS7, + OperatingSystemCentOS8, + OperatingSystemDefault: return nil } return errors.New("failed to validate operating system") @@ -79,7 +81,7 @@ func sshUsername(osName OperatingSystem) (string, error) { return "ubuntu", nil case OperatingSystemCentOS7, OperatingSystemCentOS8: return "centos", nil - case OperatingSystemCoreOS, OperatingSystemFlatcar: + case OperatingSystemFlatcar: return "core", nil } From ca631e418c6e279d6ead7ec1def8abc0adbea382 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Fri, 29 Jan 2021 16:09:23 +0200 Subject: [PATCH 05/21] Drop mounting flexvolume plugins into the openstack CCM (#1234) Signed-off-by: Artiom Diomin --- pkg/templates/externalccm/openstack.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/pkg/templates/externalccm/openstack.go b/pkg/templates/externalccm/openstack.go index 88741a67f..1f862bde0 100644 --- a/pkg/templates/externalccm/openstack.go +++ b/pkg/templates/externalccm/openstack.go @@ -164,8 +164,7 @@ func osDaemonSet(image string) *appsv1.DaemonSet { runAsUser int64 = 1001 hostPathDirectoryOrCreate = corev1.HostPathDirectoryOrCreate - caCertsPath = "/etc/ssl/certs" - flexvolumeDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" + caCertsPath = "/etc/ssl/certs" ) return &appsv1.DaemonSet{ @@ -235,10 +234,6 @@ func osDaemonSet(image string) *appsv1.DaemonSet { MountPath: "/etc/config", ReadOnly: true, }, - { - Name: "flexvolume-dir", - MountPath: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec", - }, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -249,15 +244,6 @@ func osDaemonSet(image string) *appsv1.DaemonSet { }, HostNetwork: true, Volumes: []corev1.Volume{ - { - Name: "flexvolume-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: flexvolumeDir, - Type: &hostPathDirectoryOrCreate, - }, - }, - }, { Name: "k8s-certs", VolumeSource: corev1.VolumeSource{ From ec0ab69403bfce55c10d9d083e4ab900a2843941 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Fri, 29 Jan 2021 17:43:22 +0200 Subject: [PATCH 06/21] Add debian support (#1233) Signed-off-by: Artiom Diomin --- pkg/apis/kubeone/types.go | 1 + pkg/apis/kubeone/v1beta1/types.go | 1 + pkg/tasks/kubernetes_binaries.go | 14 ++++++++------ pkg/tasks/prerequisites.go | 5 +++-- pkg/tasks/reset.go | 7 ++++--- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/pkg/apis/kubeone/types.go b/pkg/apis/kubeone/types.go index 126c88255..8a452c64c 100644 --- a/pkg/apis/kubeone/types.go +++ b/pkg/apis/kubeone/types.go @@ -80,6 +80,7 @@ type OperatingSystemName string const ( OperatingSystemNameUbuntu OperatingSystemName = "ubuntu" + OperatingSystemNameDebian OperatingSystemName = "debian" OperatingSystemNameCentOS OperatingSystemName = "centos" OperatingSystemNameRHEL OperatingSystemName = "rhel" OperatingSystemNameAmazon OperatingSystemName = "amzn" diff --git a/pkg/apis/kubeone/v1beta1/types.go b/pkg/apis/kubeone/v1beta1/types.go index 21647635b..1d26dd71b 100644 --- a/pkg/apis/kubeone/v1beta1/types.go +++ b/pkg/apis/kubeone/v1beta1/types.go @@ -80,6 +80,7 @@ type OperatingSystemName string const ( OperatingSystemNameUbuntu OperatingSystemName = "ubuntu" + OperatingSystemNameDebian OperatingSystemName = "debian" OperatingSystemNameCentOS OperatingSystemName = "centos" OperatingSystemNameRHEL OperatingSystemName = "rhel" OperatingSystemNameAmazon OperatingSystemName = "amzn" diff --git a/pkg/tasks/kubernetes_binaries.go b/pkg/tasks/kubernetes_binaries.go index ce04788b8..cb7b714bd 100644 --- a/pkg/tasks/kubernetes_binaries.go +++ b/pkg/tasks/kubernetes_binaries.go @@ -26,21 +26,23 @@ import ( func upgradeKubeletAndKubectlBinaries(s *state.State, node kubeoneapi.HostConfig) error { return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ - kubeoneapi.OperatingSystemNameUbuntu: upgradeKubeletAndKubectlBinariesDebian, - kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeletAndKubectlBinariesFlatcar, - kubeoneapi.OperatingSystemNameCentOS: upgradeKubeletAndKubectlBinariesCentOS, kubeoneapi.OperatingSystemNameAmazon: upgradeKubeletAndKubectlBinariesAmazonLinux, + kubeoneapi.OperatingSystemNameCentOS: upgradeKubeletAndKubectlBinariesCentOS, + kubeoneapi.OperatingSystemNameDebian: upgradeKubeletAndKubectlBinariesDebian, + kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeletAndKubectlBinariesFlatcar, kubeoneapi.OperatingSystemNameRHEL: upgradeKubeletAndKubectlBinariesCentOS, + kubeoneapi.OperatingSystemNameUbuntu: upgradeKubeletAndKubectlBinariesDebian, }) } func upgradeKubeadmAndCNIBinaries(s *state.State, node kubeoneapi.HostConfig) error { return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ - kubeoneapi.OperatingSystemNameUbuntu: upgradeKubeadmAndCNIBinariesDebian, - kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeadmAndCNIBinariesFlatcar, - kubeoneapi.OperatingSystemNameCentOS: upgradeKubeadmAndCNIBinariesCentOS, kubeoneapi.OperatingSystemNameAmazon: upgradeKubeadmAndCNIBinariesAmazonLinux, + kubeoneapi.OperatingSystemNameCentOS: upgradeKubeadmAndCNIBinariesCentOS, + kubeoneapi.OperatingSystemNameDebian: upgradeKubeadmAndCNIBinariesDebian, + kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeadmAndCNIBinariesFlatcar, kubeoneapi.OperatingSystemNameRHEL: upgradeKubeadmAndCNIBinariesCentOS, + kubeoneapi.OperatingSystemNameUbuntu: upgradeKubeadmAndCNIBinariesDebian, }) } diff --git a/pkg/tasks/prerequisites.go b/pkg/tasks/prerequisites.go index 1658ed43d..ff0c2d342 100644 --- a/pkg/tasks/prerequisites.go +++ b/pkg/tasks/prerequisites.go @@ -85,11 +85,12 @@ func createEnvironmentFile(s *state.State) error { func installKubeadm(s *state.State, node kubeoneapi.HostConfig) error { return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ - kubeoneapi.OperatingSystemNameUbuntu: installKubeadmDebian, kubeoneapi.OperatingSystemNameAmazon: installKubeadmAmazonLinux, - kubeoneapi.OperatingSystemNameFlatcar: installKubeadmFlatcar, kubeoneapi.OperatingSystemNameCentOS: installKubeadmCentOS, + kubeoneapi.OperatingSystemNameDebian: installKubeadmDebian, + kubeoneapi.OperatingSystemNameFlatcar: installKubeadmFlatcar, kubeoneapi.OperatingSystemNameRHEL: installKubeadmCentOS, + kubeoneapi.OperatingSystemNameUbuntu: installKubeadmDebian, }) } diff --git a/pkg/tasks/reset.go b/pkg/tasks/reset.go index 01be2a6b6..2685d9016 100644 --- a/pkg/tasks/reset.go +++ b/pkg/tasks/reset.go @@ -129,11 +129,12 @@ func removeBinaries(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connec } return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ - kubeoneapi.OperatingSystemNameUbuntu: removeBinariesDebian, - kubeoneapi.OperatingSystemNameFlatcar: removeBinariesFlatcar, + kubeoneapi.OperatingSystemNameAmazon: removeBinariesAmazonLinux, kubeoneapi.OperatingSystemNameCentOS: removeBinariesCentOS, + kubeoneapi.OperatingSystemNameDebian: removeBinariesDebian, + kubeoneapi.OperatingSystemNameFlatcar: removeBinariesFlatcar, kubeoneapi.OperatingSystemNameRHEL: removeBinariesCentOS, - kubeoneapi.OperatingSystemNameAmazon: removeBinariesAmazonLinux, + kubeoneapi.OperatingSystemNameUbuntu: removeBinariesDebian, }) } From 69c006718c2f3bccf7fb9dd22bd71112e30a7fc5 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Tue, 2 Feb 2021 15:38:37 +0200 Subject: [PATCH 07/21] Fix confusing default in OpenIDConnect (#1235) Signed-off-by: Artiom Diomin --- pkg/apis/kubeone/v1alpha1/defaults.go | 82 +++++++++--------- pkg/apis/kubeone/v1beta1/defaults.go | 118 +++++++++++++------------- 2 files changed, 100 insertions(+), 100 deletions(-) diff --git a/pkg/apis/kubeone/v1alpha1/defaults.go b/pkg/apis/kubeone/v1alpha1/defaults.go index 1cd78a941..ed98e37fc 100644 --- a/pkg/apis/kubeone/v1alpha1/defaults.go +++ b/pkg/apis/kubeone/v1alpha1/defaults.go @@ -90,9 +90,7 @@ func SetDefaults_APIEndpoints(obj *KubeOneCluster) { } obj.APIEndpoint.Host = obj.Hosts[0].PublicAddress } - if obj.APIEndpoint.Port == 0 { - obj.APIEndpoint.Port = 6443 - } + obj.APIEndpoint.Port = defaulti(obj.APIEndpoint.Port, 6443) } func SetDefaults_Versions(obj *KubeOneCluster) { @@ -101,18 +99,10 @@ func SetDefaults_Versions(obj *KubeOneCluster) { } func SetDefaults_ClusterNetwork(obj *KubeOneCluster) { - if len(obj.ClusterNetwork.PodSubnet) == 0 { - obj.ClusterNetwork.PodSubnet = DefaultPodSubnet - } - if len(obj.ClusterNetwork.ServiceSubnet) == 0 { - obj.ClusterNetwork.ServiceSubnet = DefaultServiceSubnet - } - if len(obj.ClusterNetwork.ServiceDomainName) == 0 { - obj.ClusterNetwork.ServiceDomainName = DefaultServiceDNS - } - if len(obj.ClusterNetwork.NodePortRange) == 0 { - obj.ClusterNetwork.NodePortRange = DefaultNodePortRange - } + obj.ClusterNetwork.PodSubnet = defaults(obj.ClusterNetwork.PodSubnet, DefaultPodSubnet) + obj.ClusterNetwork.ServiceSubnet = defaults(obj.ClusterNetwork.ServiceSubnet, DefaultServiceSubnet) + obj.ClusterNetwork.ServiceDomainName = defaults(obj.ClusterNetwork.ServiceDomainName, DefaultServiceDNS) + obj.ClusterNetwork.NodePortRange = defaults(obj.ClusterNetwork.NodePortRange, DefaultNodePortRange) if obj.ClusterNetwork.CNI == nil { obj.ClusterNetwork.CNI = &CNI{ Provider: CNIProviderCanal, @@ -165,29 +155,31 @@ func SetDefaults_Features(obj *KubeOneCluster) { if obj.Features.StaticAuditLog != nil && obj.Features.StaticAuditLog.Enable { defaultStaticAuditLogConfig(&obj.Features.StaticAuditLog.Config) } + if obj.Features.OpenIDConnect != nil && obj.Features.OpenIDConnect.Enable { + defaultOpenIDConnect(&obj.Features.OpenIDConnect.Config) + } +} + +func defaultOpenIDConnect(config *OpenIDConnectConfig) { + config.ClientID = defaults(config.ClientID, "kubernetes") + config.UsernameClaim = defaults(config.UsernameClaim, "sub") + config.UsernamePrefix = defaults(config.UsernamePrefix, "oidc:") + config.GroupsClaim = defaults(config.GroupsClaim, "groups") + config.GroupsPrefix = defaults(config.GroupsPrefix, "oidc:") + config.SigningAlgs = defaults(config.SigningAlgs, "RS256") } func SetDefaults_Addons(obj *KubeOneCluster) { if obj.Addons != nil && obj.Addons.Enable { - if len(obj.Addons.Path) == 0 { - obj.Addons.Path = "./addons" - } + obj.Addons.Path = defaults(obj.Addons.Path, "./addons") } } func defaultStaticAuditLogConfig(obj *StaticAuditLogConfig) { - if obj.LogPath == "" { - obj.LogPath = "/var/log/kubernetes/audit.log" - } - if obj.LogMaxAge == 0 { - obj.LogMaxAge = 30 - } - if obj.LogMaxBackup == 0 { - obj.LogMaxBackup = 3 - } - if obj.LogMaxSize == 0 { - obj.LogMaxSize = 100 - } + obj.LogPath = defaults(obj.LogPath, "/var/log/kubernetes/audit.log") + obj.LogMaxAge = defaulti(obj.LogMaxAge, 30) + obj.LogMaxBackup = defaulti(obj.LogMaxBackup, 3) + obj.LogMaxSize = defaulti(obj.LogMaxSize, 100) } func defaultHostConfig(obj *HostConfig) { @@ -197,19 +189,25 @@ func defaultHostConfig(obj *HostConfig) { if len(obj.PrivateAddress) == 0 && len(obj.PublicAddress) > 0 { obj.PrivateAddress = obj.PublicAddress } - if len(obj.SSHPrivateKeyFile) == 0 && len(obj.SSHAgentSocket) == 0 { - obj.SSHAgentSocket = "env:SSH_AUTH_SOCK" - } - if obj.SSHUsername == "" { - obj.SSHUsername = "root" - } - if obj.SSHPort == 0 { - obj.SSHPort = 22 + if obj.SSHPrivateKeyFile == "" { + obj.SSHAgentSocket = defaults(obj.SSHAgentSocket, "env:SSH_AUTH_SOCK") } - if obj.BastionPort == 0 { - obj.BastionPort = 22 + obj.SSHUsername = defaults(obj.SSHUsername, "root") + obj.SSHPort = defaulti(obj.SSHPort, 22) + obj.BastionPort = defaulti(obj.BastionPort, 22) + obj.BastionUser = defaults(obj.BastionUser, obj.SSHUsername) +} + +func defaults(input string, defaultValue string) string { + if input != "" { + return input } - if obj.BastionUser == "" { - obj.BastionUser = obj.SSHUsername + return defaultValue +} + +func defaulti(input, defaultValue int) int { + if input != 0 { + return input } + return defaultValue } diff --git a/pkg/apis/kubeone/v1beta1/defaults.go b/pkg/apis/kubeone/v1beta1/defaults.go index 1a1c709ce..062d2e83e 100644 --- a/pkg/apis/kubeone/v1beta1/defaults.go +++ b/pkg/apis/kubeone/v1beta1/defaults.go @@ -109,9 +109,7 @@ func SetDefaults_APIEndpoints(obj *KubeOneCluster) { } obj.APIEndpoint.Host = obj.ControlPlane.Hosts[0].PublicAddress } - if obj.APIEndpoint.Port == 0 { - obj.APIEndpoint.Port = 6443 - } + obj.APIEndpoint.Port = defaulti(obj.APIEndpoint.Port, 6443) } func SetDefaults_Versions(obj *KubeOneCluster) { @@ -139,29 +137,21 @@ func SetDefaults_ContainerRuntime(obj *KubeOneCluster) { } func SetDefaults_ClusterNetwork(obj *KubeOneCluster) { - if len(obj.ClusterNetwork.PodSubnet) == 0 { - obj.ClusterNetwork.PodSubnet = DefaultPodSubnet - } - if len(obj.ClusterNetwork.ServiceSubnet) == 0 { - obj.ClusterNetwork.ServiceSubnet = DefaultServiceSubnet - } - if len(obj.ClusterNetwork.ServiceDomainName) == 0 { - obj.ClusterNetwork.ServiceDomainName = DefaultServiceDNS - } - if len(obj.ClusterNetwork.NodePortRange) == 0 { - obj.ClusterNetwork.NodePortRange = DefaultNodePortRange - } + obj.ClusterNetwork.PodSubnet = defaults(obj.ClusterNetwork.PodSubnet, DefaultPodSubnet) + obj.ClusterNetwork.ServiceSubnet = defaults(obj.ClusterNetwork.ServiceSubnet, DefaultServiceSubnet) + obj.ClusterNetwork.ServiceDomainName = defaults(obj.ClusterNetwork.ServiceDomainName, DefaultServiceDNS) + obj.ClusterNetwork.NodePortRange = defaults(obj.ClusterNetwork.NodePortRange, DefaultNodePortRange) defaultCanal := &CanalSpec{MTU: DefaultCanalMTU} switch { case obj.CloudProvider.AWS != nil: - defaultCanal.MTU = 8951 // 9001 AWS Jumbo Frame - 50 VXLAN bytes + defaultCanal.MTU = defaulti(defaultCanal.MTU, 8951) // 9001 AWS Jumbo Frame - 50 VXLAN bytes case obj.CloudProvider.GCE != nil: - defaultCanal.MTU = 1410 // GCE specific 1460 bytes - 50 VXLAN bytes + defaultCanal.MTU = defaulti(defaultCanal.MTU, 1410) // GCE specific 1460 bytes - 50 VXLAN bytes case obj.CloudProvider.Hetzner != nil: - defaultCanal.MTU = 1400 // Hetzner specific 1450 bytes - 50 VXLAN bytes + defaultCanal.MTU = defaulti(defaultCanal.MTU, 1400) // Hetzner specific 1450 bytes - 50 VXLAN bytes case obj.CloudProvider.Openstack != nil: - defaultCanal.MTU = 1400 // Openstack specific 1450 bytes - 50 VXLAN bytes + defaultCanal.MTU = defaulti(defaultCanal.MTU, 1400) // Openstack specific 1450 bytes - 50 VXLAN bytes } if obj.ClusterNetwork.CNI == nil { @@ -213,18 +203,22 @@ func SetDefaults_AssetConfiguration(obj *KubeOneCluster) { return } - if obj.AssetConfiguration.Kubernetes.ImageRepository == "" { - obj.AssetConfiguration.Kubernetes.ImageRepository = obj.RegistryConfiguration.OverwriteRegistry - } - if obj.AssetConfiguration.CoreDNS.ImageRepository == "" { - obj.AssetConfiguration.CoreDNS.ImageRepository = obj.RegistryConfiguration.OverwriteRegistry - } - if obj.AssetConfiguration.Etcd.ImageRepository == "" { - obj.AssetConfiguration.Etcd.ImageRepository = obj.RegistryConfiguration.OverwriteRegistry - } - if obj.AssetConfiguration.MetricsServer.ImageRepository == "" { - obj.AssetConfiguration.MetricsServer.ImageRepository = obj.RegistryConfiguration.OverwriteRegistry - } + obj.AssetConfiguration.Kubernetes.ImageRepository = defaults( + obj.AssetConfiguration.Kubernetes.ImageRepository, + obj.RegistryConfiguration.OverwriteRegistry, + ) + obj.AssetConfiguration.CoreDNS.ImageRepository = defaults( + obj.AssetConfiguration.CoreDNS.ImageRepository, + obj.RegistryConfiguration.OverwriteRegistry, + ) + obj.AssetConfiguration.Etcd.ImageRepository = defaults( + obj.AssetConfiguration.Etcd.ImageRepository, + obj.RegistryConfiguration.OverwriteRegistry, + ) + obj.AssetConfiguration.MetricsServer.ImageRepository = defaults( + obj.AssetConfiguration.MetricsServer.ImageRepository, + obj.RegistryConfiguration.OverwriteRegistry, + ) } func SetDefaults_Features(obj *KubeOneCluster) { @@ -236,29 +230,31 @@ func SetDefaults_Features(obj *KubeOneCluster) { if obj.Features.StaticAuditLog != nil && obj.Features.StaticAuditLog.Enable { defaultStaticAuditLogConfig(&obj.Features.StaticAuditLog.Config) } + if obj.Features.OpenIDConnect != nil && obj.Features.OpenIDConnect.Enable { + defaultOpenIDConnect(&obj.Features.OpenIDConnect.Config) + } +} + +func defaultOpenIDConnect(config *OpenIDConnectConfig) { + config.ClientID = defaults(config.ClientID, "kubernetes") + config.UsernameClaim = defaults(config.UsernameClaim, "sub") + config.UsernamePrefix = defaults(config.UsernamePrefix, "oidc:") + config.GroupsClaim = defaults(config.GroupsClaim, "groups") + config.GroupsPrefix = defaults(config.GroupsPrefix, "oidc:") + config.SigningAlgs = defaults(config.SigningAlgs, "RS256") } func SetDefaults_Addons(obj *KubeOneCluster) { if obj.Addons != nil && obj.Addons.Enable { - if len(obj.Addons.Path) == 0 { - obj.Addons.Path = "./addons" - } + obj.Addons.Path = defaults(obj.Addons.Path, "./addons") } } func defaultStaticAuditLogConfig(obj *StaticAuditLogConfig) { - if obj.LogPath == "" { - obj.LogPath = "/var/log/kubernetes/audit.log" - } - if obj.LogMaxAge == 0 { - obj.LogMaxAge = 30 - } - if obj.LogMaxBackup == 0 { - obj.LogMaxBackup = 3 - } - if obj.LogMaxSize == 0 { - obj.LogMaxSize = 100 - } + obj.LogPath = defaults(obj.LogPath, "/var/log/kubernetes/audit.log") + obj.LogMaxAge = defaulti(obj.LogMaxAge, 30) + obj.LogMaxBackup = defaulti(obj.LogMaxBackup, 3) + obj.LogMaxSize = defaulti(obj.LogMaxSize, 100) } func defaultHostConfig(obj *HostConfig) { @@ -268,19 +264,25 @@ func defaultHostConfig(obj *HostConfig) { if len(obj.PrivateAddress) == 0 && len(obj.PublicAddress) > 0 { obj.PrivateAddress = obj.PublicAddress } - if len(obj.SSHPrivateKeyFile) == 0 && len(obj.SSHAgentSocket) == 0 { - obj.SSHAgentSocket = "env:SSH_AUTH_SOCK" - } - if obj.SSHUsername == "" { - obj.SSHUsername = "root" + if obj.SSHPrivateKeyFile == "" { + obj.SSHAgentSocket = defaults(obj.SSHAgentSocket, "env:SSH_AUTH_SOCK") } - if obj.SSHPort == 0 { - obj.SSHPort = 22 - } - if obj.BastionPort == 0 { - obj.BastionPort = 22 + obj.SSHUsername = defaults(obj.SSHUsername, "root") + obj.SSHPort = defaulti(obj.SSHPort, 22) + obj.BastionPort = defaulti(obj.BastionPort, 22) + obj.BastionUser = defaults(obj.BastionUser, obj.SSHUsername) +} + +func defaults(input, defaultValue string) string { + if input != "" { + return input } - if obj.BastionUser == "" { - obj.BastionUser = obj.SSHUsername + return defaultValue +} + +func defaulti(input, defaultValue int) int { + if input != 0 { + return input } + return defaultValue } From 76643e0ab715ce601d5d6630e924a894fe3623a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Tue, 2 Feb 2021 15:48:42 +0100 Subject: [PATCH 08/21] Disallow and deprecate the PodPresets feature (#1236) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Disallow the PodPresets feature for 1.20 clusters Signed-off-by: Marko Mudrinić * Deprecate the PodPresets feature Signed-off-by: Marko Mudrinić --- pkg/apis/kubeone/types.go | 4 ++ pkg/apis/kubeone/v1alpha1/types.go | 6 +- pkg/apis/kubeone/v1beta1/types.go | 4 ++ pkg/apis/kubeone/validation/validation.go | 11 +++- .../kubeone/validation/validation_test.go | 64 ++++++++++++++++++- pkg/cmd/config.go | 5 ++ 6 files changed, 88 insertions(+), 6 deletions(-) diff --git a/pkg/apis/kubeone/types.go b/pkg/apis/kubeone/types.go index 8a452c64c..a5e4fb0db 100644 --- a/pkg/apis/kubeone/types.go +++ b/pkg/apis/kubeone/types.go @@ -351,6 +351,7 @@ type Features struct { // PodNodeSelector PodNodeSelector *PodNodeSelector `json:"podNodeSelector,omitempty"` // PodPresets + // Deprecated: will be removed once Kubernetes 1.19 reaches EOL PodPresets *PodPresets `json:"podPresets,omitempty"` // PodSecurityPolicy PodSecurityPolicy *PodSecurityPolicy `json:"podSecurityPolicy,omitempty"` @@ -473,6 +474,9 @@ type PodNodeSelectorConfig struct { } // PodPresets feature flag +// The PodPresets feature has been removed in Kubernetes 1.20. +// This feature is deprecated and will be removed from the API once +// Kubernetes 1.19 reaches EOL. type PodPresets struct { // Enable Enable bool `json:"enable,omitempty"` diff --git a/pkg/apis/kubeone/v1alpha1/types.go b/pkg/apis/kubeone/v1alpha1/types.go index 81cc621c7..334749e54 100644 --- a/pkg/apis/kubeone/v1alpha1/types.go +++ b/pkg/apis/kubeone/v1alpha1/types.go @@ -215,7 +215,8 @@ type MachineControllerConfig struct { // Features controls what features will be enabled on the cluster type Features struct { - PodNodeSelector *PodNodeSelector `json:"podNodeSelector"` + PodNodeSelector *PodNodeSelector `json:"podNodeSelector"` + // Deprecated: will be removed once Kubernetes 1.19 reaches EOL PodPresets *PodPresets `json:"podPresets"` PodSecurityPolicy *PodSecurityPolicy `json:"podSecurityPolicy"` StaticAuditLog *StaticAuditLog `json:"staticAuditLog"` @@ -247,6 +248,9 @@ type PodNodeSelectorConfig struct { } // PodPresets feature flag +// The PodPresets feature has been removed in Kubernetes 1.20. +// This feature is deprecated and will be removed from the API once +// Kubernetes 1.19 reaches EOL. type PodPresets struct { Enable bool `json:"enable"` } diff --git a/pkg/apis/kubeone/v1beta1/types.go b/pkg/apis/kubeone/v1beta1/types.go index 1d26dd71b..a975f323a 100644 --- a/pkg/apis/kubeone/v1beta1/types.go +++ b/pkg/apis/kubeone/v1beta1/types.go @@ -351,6 +351,7 @@ type Features struct { // PodNodeSelector PodNodeSelector *PodNodeSelector `json:"podNodeSelector,omitempty"` // PodPresets + // Deprecated: will be removed once Kubernetes 1.19 reaches EOL PodPresets *PodPresets `json:"podPresets,omitempty"` // PodSecurityPolicy PodSecurityPolicy *PodSecurityPolicy `json:"podSecurityPolicy,omitempty"` @@ -473,6 +474,9 @@ type PodNodeSelectorConfig struct { } // PodPresets feature flag +// The PodPresets feature has been removed in Kubernetes 1.20. +// This feature is deprecated and will be removed from the API once +// Kubernetes 1.19 reaches EOL. type PodPresets struct { // Enable Enable bool `json:"enable,omitempty"` diff --git a/pkg/apis/kubeone/validation/validation.go b/pkg/apis/kubeone/validation/validation.go index eb52bc05f..92727592a 100644 --- a/pkg/apis/kubeone/validation/validation.go +++ b/pkg/apis/kubeone/validation/validation.go @@ -50,7 +50,7 @@ func ValidateKubeOneCluster(c kubeone.KubeOneCluster) field.ErrorList { "machine-controller deployment is disabled, but the configuration still contains dynamic workers")) } - allErrs = append(allErrs, ValidateFeatures(c.Features, field.NewPath("features"))...) + allErrs = append(allErrs, ValidateFeatures(c.Features, c.Versions, field.NewPath("features"))...) allErrs = append(allErrs, ValidateAddons(c.Addons, field.NewPath("addons"))...) allErrs = append(allErrs, ValidateRegistryConfiguration(c.RegistryConfiguration, field.NewPath("registryConfiguration"))...) @@ -294,7 +294,7 @@ func ValidateDynamicWorkerConfig(workerset []kubeone.DynamicWorkerConfig, fldPat } // ValidateFeatures validates the Features structure -func ValidateFeatures(f kubeone.Features, fldPath *field.Path) field.ErrorList { +func ValidateFeatures(f kubeone.Features, versions kubeone.VersionConfig, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if f.PodNodeSelector != nil && f.PodNodeSelector.Enable { @@ -306,6 +306,13 @@ func ValidateFeatures(f kubeone.Features, fldPath *field.Path) field.ErrorList { if f.OpenIDConnect != nil && f.OpenIDConnect.Enable { allErrs = append(allErrs, ValidateOIDCConfig(f.OpenIDConnect.Config, fldPath.Child("openidConnect"))...) } + if f.PodPresets != nil && f.PodPresets.Enable { + kubeVer, _ := semver.NewVersion(versions.Kubernetes) + gteKube120Condition, _ := semver.NewConstraint(">= 1.20") + if gteKube120Condition.Check(kubeVer) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("podPresets"), "podPresets feature is removed in kubernetes 1.20+ and must be disabled")) + } + } return allErrs } diff --git a/pkg/apis/kubeone/validation/validation_test.go b/pkg/apis/kubeone/validation/validation_test.go index 46ba0ba5b..54bfc9d11 100644 --- a/pkg/apis/kubeone/validation/validation_test.go +++ b/pkg/apis/kubeone/validation/validation_test.go @@ -1003,6 +1003,7 @@ func TestValidateFeatures(t *testing.T) { tests := []struct { name string features kubeone.Features + versions kubeone.VersionConfig expectedError bool }{ { @@ -1015,6 +1016,9 @@ func TestValidateFeatures(t *testing.T) { Enable: true, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: false, }, { @@ -1024,11 +1028,17 @@ func TestValidateFeatures(t *testing.T) { Enable: false, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: false, }, { - name: "no feature configured", - features: kubeone.Features{}, + name: "no feature configured", + features: kubeone.Features{}, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: false, }, { @@ -1043,6 +1053,9 @@ func TestValidateFeatures(t *testing.T) { }, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: false, }, { @@ -1053,6 +1066,9 @@ func TestValidateFeatures(t *testing.T) { Config: kubeone.StaticAuditLogConfig{}, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: true, }, { @@ -1063,6 +1079,9 @@ func TestValidateFeatures(t *testing.T) { Config: kubeone.OpenIDConnectConfig{}, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: true, }, { @@ -1073,13 +1092,52 @@ func TestValidateFeatures(t *testing.T) { Config: kubeone.PodNodeSelectorConfig{}, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, + expectedError: true, + }, + { + name: "podPresets enabled on 1.19 cluster", + features: kubeone.Features{ + PodPresets: &kubeone.PodPresets{ + Enable: true, + }, + }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.19.7", + }, + expectedError: false, + }, + { + name: "podPresets enabled on 1.20 cluster", + features: kubeone.Features{ + PodPresets: &kubeone.PodPresets{ + Enable: true, + }, + }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, + expectedError: true, + }, + { + name: "podPresets enabled on 1.21 cluster", + features: kubeone.Features{ + PodPresets: &kubeone.PodPresets{ + Enable: true, + }, + }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.21.0", + }, expectedError: true, }, } for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { - errs := ValidateFeatures(tc.features, nil) + errs := ValidateFeatures(tc.features, tc.versions, nil) if (len(errs) == 0) == tc.expectedError { t.Errorf("test case failed: expected %v, but got %v", tc.expectedError, (len(errs) != 0)) } diff --git a/pkg/cmd/config.go b/pkg/cmd/config.go index b81a29788..91c407928 100644 --- a/pkg/cmd/config.go +++ b/pkg/cmd/config.go @@ -603,6 +603,11 @@ features: podSecurityPolicy: enable: {{ .EnablePodSecurityPolicy }} # Enables PodPresets admission plugin in API server. + # The PodPresets feature has been removed in Kubernetes 1.20. + # This feature is deprecated and will be removed from the API once + # Kubernetes 1.19 reaches EOL. + # Provisioning a Kubernetes 1.20 cluster or upgrading an existing cluster to + # the Kubernetes 1.20 requires this feature to be disabled. podPresets: enable: {{ .EnablePodPresets }} # Enables and configures audit log backend. From 47bdd5b3a091204cf8ca2e0c0be9404f02682e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Tue, 2 Feb 2021 17:00:48 +0100 Subject: [PATCH 09/21] Update KubeOne CI jobs (#1237) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove 1.16 and 1.17 conformance and upgrade tests * Update Golang image to 1.15.7 * Remove 1.16 and 1.17 conformance tests * Remove 1.16 to 1.17 upgrade tests 1.17 to 1.18 upgrade tests were intentionally left in place because we still support upgrading to a supported Kubernetes version. Signed-off-by: Marko Mudrinić * Update containerd E2E tests to a newer K8s version Signed-off-by: Marko Mudrinić * Update Kubernetes versions used in tests Signed-off-by: Marko Mudrinić --- .prow.yaml | 582 ++++++----------------------------------------------- 1 file changed, 61 insertions(+), 521 deletions(-) diff --git a/.prow.yaml b/.prow.yaml index 5ba67c69a..632a8b05e 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -25,7 +25,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.15.0 + - image: golang:1.15.7 command: - make args: @@ -67,7 +67,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.15.0 + - image: golang:1.15.7 command: - make args: @@ -88,7 +88,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.15.0 + - image: golang:1.15.7 command: - make args: @@ -122,38 +122,11 @@ presubmits: memory: 2Gi ######################################################### - # E2E/Conformance tests (AWS, 1.16-1.19) + # E2E/Conformance tests (AWS, 1.18-1.19) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-aws-conformance-1.16 - run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-aws: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "aws" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - - name: TF_VAR_ami - value: ami-08985edfecbbbcf52 - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-aws-containerd-conformance-1.16 + - name: pull-kubeone-e2e-aws-containerd-conformance-1.18 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -174,39 +147,13 @@ presubmits: - name: CONTAINER_RUNTIME value: containerd - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - name: pull-kubeone-e2e-aws-conformance-1.17 - run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-aws: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "aws" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - name: pull-kubeone-e2e-aws-conformance-1.18 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true @@ -226,7 +173,7 @@ presubmits: - name: PROVIDER value: "aws" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: @@ -252,7 +199,7 @@ presubmits: - name: PROVIDER value: "aws" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "conformance" resources: @@ -260,61 +207,10 @@ presubmits: cpu: 1 ######################################################### - # E2E/Conformance tests (DigitalOcean, 1.16-1.19) + # E2E/Conformance tests (DigitalOcean, 1.18-1.19) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-digitalocean-conformance-1.16 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-digitalocean: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "digitalocean" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-digitalocean-conformance-1.17 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-digitalocean: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "digitalocean" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - name: pull-kubeone-e2e-digitalocean-conformance-1.18 always_run: false decorate: true @@ -334,7 +230,7 @@ presubmits: - name: PROVIDER value: "digitalocean" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: @@ -360,7 +256,7 @@ presubmits: - name: PROVIDER value: "digitalocean" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "conformance" resources: @@ -368,61 +264,10 @@ presubmits: cpu: 1 ######################################################### - # E2E/Conformance tests (Hetzner, 1.16-1.19) + # E2E/Conformance tests (Hetzner, 1.18-1.19) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-hetzner-conformance-1.16 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-hetzner: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "hetzner" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-hetzner-conformance-1.17 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-hetzner: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "hetzner" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - name: pull-kubeone-e2e-hetzner-conformance-1.18 always_run: false decorate: true @@ -442,7 +287,7 @@ presubmits: - name: PROVIDER value: "hetzner" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: @@ -468,7 +313,7 @@ presubmits: - name: PROVIDER value: "hetzner" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "conformance" resources: @@ -476,65 +321,10 @@ presubmits: cpu: 1 ######################################################### - # E2E/Conformance tests (GCE, 1.16-1.19) + # E2E/Conformance tests (GCE, 1.18-1.19) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-gce-conformance-1.16 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-gce: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "gce" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - - name: TF_VAR_project - value: "kubeone-terraform-test" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-gce-conformance-1.17 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-gce: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "gce" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "conformance" - - name: TF_VAR_project - value: "kubeone-terraform-test" - resources: - requests: - cpu: 1 - - name: pull-kubeone-e2e-gce-conformance-1.18 always_run: false decorate: true @@ -554,7 +344,7 @@ presubmits: - name: PROVIDER value: "gce" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "conformance" - name: TF_VAR_project @@ -582,7 +372,7 @@ presubmits: - name: PROVIDER value: "gce" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "conformance" - name: TF_VAR_project @@ -593,60 +383,9 @@ presubmits: ######################################################### # E2E/Conformance tests (Packet, 1.16-1.19) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-packet-conformance-1.16 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-packet: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "packet" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-packet-conformance-1.17 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-packet: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "packet" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - name: pull-kubeone-e2e-packet-conformance-1.18 always_run: false decorate: true @@ -666,7 +405,7 @@ presubmits: - name: PROVIDER value: "packet" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: @@ -692,7 +431,7 @@ presubmits: - name: PROVIDER value: "packet" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "conformance" resources: @@ -700,61 +439,10 @@ presubmits: cpu: 1 ######################################################### - # E2E/Conformance tests (OpenStack, 1.16-1.19) + # E2E/Conformance tests (OpenStack, 1.18-1.19) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-openstack-conformance-1.16 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-openstack: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "openstack" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-openstack-conformance-1.17 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-openstack: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "openstack" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - name: pull-kubeone-e2e-openstack-conformance-1.18 always_run: false decorate: true @@ -774,7 +462,7 @@ presubmits: - name: PROVIDER value: "openstack" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: @@ -800,7 +488,7 @@ presubmits: - name: PROVIDER value: "openstack" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "conformance" resources: @@ -809,36 +497,10 @@ presubmits: ######################################################### # E2E/Upgrade tests (AWS) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-aws-upgrade-1.16-1.17 - run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-aws: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "aws" - - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "upgrades" - - name: TF_VAR_ami - value: ami-08985edfecbbbcf52 - - - name: pull-kubeone-e2e-aws-upgrade-containerd-1.16-1.17 + - name: pull-kubeone-e2e-aws-upgrade-containerd-1.17-1.18 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -859,9 +521,9 @@ presubmits: - name: CONTAINER_RUNTIME value: containerd - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "upgrades" @@ -884,9 +546,9 @@ presubmits: - name: PROVIDER value: "aws" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "upgrades" @@ -909,41 +571,17 @@ presubmits: - name: PROVIDER value: "aws" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "upgrades" ######################################################### # E2E/Upgrade tests (DigitalOcean) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-digitalocean-upgrade-1.16-1.17 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-digitalocean: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "digitalocean" - - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "upgrades" - - name: pull-kubeone-e2e-digitalocean-upgrade-1.17-1.18 always_run: false decorate: true @@ -963,9 +601,9 @@ presubmits: - name: PROVIDER value: "digitalocean" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "upgrades" @@ -988,41 +626,17 @@ presubmits: - name: PROVIDER value: "digitalocean" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "upgrades" ######################################################### # E2E/Upgrade tests (Hetzner) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-hetzner-upgrade-1.16-1.17 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-hetzner: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "hetzner" - - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "upgrades" - - name: pull-kubeone-e2e-hetzner-upgrade-1.17-1.18 always_run: false decorate: true @@ -1042,9 +656,9 @@ presubmits: - name: PROVIDER value: "hetzner" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "upgrades" @@ -1067,43 +681,17 @@ presubmits: - name: PROVIDER value: "hetzner" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "upgrades" ######################################################### # E2E/Upgrade tests (GCE) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-gce-upgrade-1.16-1.17 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-gce: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "gce" - - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "upgrades" - - name: TF_VAR_project - value: "kubeone-terraform-test" - - name: pull-kubeone-e2e-gce-upgrade-1.17-1.18 always_run: false decorate: true @@ -1123,9 +711,9 @@ presubmits: - name: PROVIDER value: "gce" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "upgrades" - name: TF_VAR_project @@ -1150,9 +738,9 @@ presubmits: - name: PROVIDER value: "gce" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "upgrades" - name: TF_VAR_project @@ -1160,33 +748,9 @@ presubmits: ######################################################### # E2E/Upgrade tests (Packet) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-packet-upgrade-1.16-1.17 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-packet: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "packet" - - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "upgrades" - - name: pull-kubeone-e2e-packet-upgrade-1.17-1.18 always_run: false decorate: true @@ -1206,9 +770,9 @@ presubmits: - name: PROVIDER value: "packet" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "upgrades" @@ -1231,41 +795,17 @@ presubmits: - name: PROVIDER value: "packet" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "upgrades" ######################################################### # E2E/Upgrade tests (OpenStack) + # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - - name: pull-kubeone-e2e-openstack-upgrade-1.16-1.17 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-openstack: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "openstack" - - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" - - name: TEST_SET - value: "upgrades" - - name: pull-kubeone-e2e-openstack-upgrade-1.17-1.18 always_run: false decorate: true @@ -1285,9 +825,9 @@ presubmits: - name: PROVIDER value: "openstack" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_SET value: "upgrades" @@ -1310,9 +850,9 @@ presubmits: - name: PROVIDER value: "openstack" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.19.7" - name: TEST_SET value: "upgrades" From 5d6ecfae7a4e308cb7a4072f2b58fe06570252c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Wed, 3 Feb 2021 12:17:37 +0100 Subject: [PATCH 10/21] Update the kubeone-e2e image (#1239) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update Kubernetes versions to 1.20.2, 1.19.7, and 1.18.15 * Update Go version from 1.15.2 to 1.15.7 * Update Terraform version from 0.12.29 to 0.12.30 Signed-off-by: Marko Mudrinić --- hack/images/kubeone-e2e/Dockerfile | 8 ++++---- hack/images/kubeone-e2e/install-kube-tests-binaries.sh | 7 +++---- hack/images/kubeone-e2e/release.sh | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/hack/images/kubeone-e2e/Dockerfile b/hack/images/kubeone-e2e/Dockerfile index b24100021..d025d511d 100644 --- a/hack/images/kubeone-e2e/Dockerfile +++ b/hack/images/kubeone-e2e/Dockerfile @@ -14,13 +14,13 @@ # building image -FROM golang:1.15.2 as builder +FROM golang:1.15.7 as builder RUN apt-get update && apt-get install -y \ unzip \ upx-ucl -ENV TERRAFORM_VERSION "0.12.29" +ENV TERRAFORM_VERSION "0.12.30" RUN curl -fL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip | funzip >/usr/local/bin/terraform RUN chmod +x /usr/local/bin/terraform @@ -37,11 +37,11 @@ RUN /opt/install-kube-tests-binaries.sh # resulting image -FROM golang:1.15.2 +FROM golang:1.15.7 ARG version -LABEL "io.kubeone"="Loodse GmbH" +LABEL "io.kubeone"="Kubermatic GmbH" LABEL version=${version} LABEL description="Set of kubernetes binaries to conduct kubeone E2E tests" LABEL maintainer="https://github.com/kubermatic/kubeone/blob/master/OWNERS" diff --git a/hack/images/kubeone-e2e/install-kube-tests-binaries.sh b/hack/images/kubeone-e2e/install-kube-tests-binaries.sh index 7599ad943..d7ea8f86d 100755 --- a/hack/images/kubeone-e2e/install-kube-tests-binaries.sh +++ b/hack/images/kubeone-e2e/install-kube-tests-binaries.sh @@ -17,10 +17,9 @@ set -euox pipefail declare -A full_versions -full_versions["1.16"]="v1.16.15" -full_versions["1.17"]="v1.17.12" -full_versions["1.18"]="v1.18.9" -full_versions["1.19"]="v1.19.2" +full_versions["1.18"]="v1.18.15" +full_versions["1.19"]="v1.19.7" +full_versions["1.20"]="v1.20.2" root_dir=${KUBETESTS_ROOT:-"/opt/kube-test"} tmp_root=${TMP_ROOT:-"/tmp/get-kube"} diff --git a/hack/images/kubeone-e2e/release.sh b/hack/images/kubeone-e2e/release.sh index 2929cab06..3e2008978 100755 --- a/hack/images/kubeone-e2e/release.sh +++ b/hack/images/kubeone-e2e/release.sh @@ -16,7 +16,7 @@ set -euox pipefail -TAG=v0.1.12 +TAG=v0.1.13 docker build --build-arg version=${TAG} --pull -t kubermatic/kubeone-e2e:${TAG} . docker push kubermatic/kubeone-e2e:${TAG} From 90c288ce7acb0852a2d11b96669f725629e0db1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Wed, 3 Feb 2021 13:36:44 +0100 Subject: [PATCH 11/21] Update machine-controller to v1.25.0 (#1238) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/templates/machinecontroller/deployment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/templates/machinecontroller/deployment.go b/pkg/templates/machinecontroller/deployment.go index a20533126..480f0a8ea 100644 --- a/pkg/templates/machinecontroller/deployment.go +++ b/pkg/templates/machinecontroller/deployment.go @@ -48,7 +48,7 @@ const ( MachineControllerAppLabelValue = "machine-controller" MachineControllerImageRegistry = "docker.io" MachineControllerImage = "/kubermatic/machine-controller:" - MachineControllerTag = "v1.24.3" + MachineControllerTag = "v1.25.0" ) func CRDs() []runtime.Object { From 222201dcb6b05fb79d33b5ecb13076e879bfe5d1 Mon Sep 17 00:00:00 2001 From: Christoph Kleineweber Date: Fri, 5 Feb 2021 12:15:36 +0100 Subject: [PATCH 12/21] Add rsync on CentOS and Amazon Linux (#1240) Rsync was installed on Ubuntu, but not CentOS and Amazon Linux --- pkg/scripts/os.go | 6 ++++-- pkg/scripts/testdata/TestKubeadmAmazonLinux-force.golden | 3 ++- .../TestKubeadmAmazonLinux-overwrite_registry.golden | 3 ++- ...estKubeadmAmazonLinux-overwrite_registry_insecure.golden | 3 ++- pkg/scripts/testdata/TestKubeadmAmazonLinux-proxy.golden | 3 ++- pkg/scripts/testdata/TestKubeadmAmazonLinux-simple.golden | 3 ++- pkg/scripts/testdata/TestKubeadmAmazonLinux-v1.16.1.golden | 3 ++- .../testdata/TestKubeadmAmazonLinux-with_containerd.golden | 3 ++- ...mazonLinux-with_containerd_with_insecure_registry.golden | 3 ++- pkg/scripts/testdata/TestKubeadmCentOS-force.golden | 3 ++- .../testdata/TestKubeadmCentOS-overwrite_registry.golden | 3 ++- .../TestKubeadmCentOS-overwrite_registry_insecure.golden | 3 ++- pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden | 3 ++- pkg/scripts/testdata/TestKubeadmCentOS-simple.golden | 3 ++- pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden | 3 ++- .../testdata/TestKubeadmCentOS-with_containerd.golden | 3 ++- ...eadmCentOS-with_containerd_with_insecure_registry.golden | 3 ++- .../testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden | 3 ++- pkg/scripts/testdata/TestUpgradeKubeadmAndCNICentOS.golden | 3 ++- .../testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden | 3 ++- .../testdata/TestUpgradeKubeletAndKubectlCentOS.golden | 3 ++- 21 files changed, 44 insertions(+), 22 deletions(-) diff --git a/pkg/scripts/os.go b/pkg/scripts/os.go index d0629bc1f..e315412c7 100644 --- a/pkg/scripts/os.go +++ b/pkg/scripts/os.go @@ -147,7 +147,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync {{ if .INSTALL_DOCKER }} {{ template "docker-daemon-config" . }} @@ -222,7 +223,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync {{ if .INSTALL_DOCKER }} {{ template "docker-daemon-config" . }} diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-force.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-force.golden index 40162fb5e..3a57dc7cc 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-force.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-force.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry.golden index b3caf8861..b3f705990 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry_insecure.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry_insecure.golden index 1b55d985d..edd5faec0 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry_insecure.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry_insecure.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-proxy.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-proxy.golden index 30a09704f..27bd65792 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-proxy.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-proxy.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-simple.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-simple.golden index b3caf8861..b3f705990 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-simple.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-simple.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-v1.16.1.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-v1.16.1.golden index 1667f44c2..8e3f68ee5 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-v1.16.1.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-v1.16.1.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd.golden index 650e7501d..63a28000a 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd_with_insecure_registry.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd_with_insecure_registry.golden index d384a7823..61809b95d 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd_with_insecure_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd_with_insecure_registry.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-force.golden b/pkg/scripts/testdata/TestKubeadmCentOS-force.golden index d089bd893..ab932341e 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-force.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-force.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden index 8402d13d2..2773395c2 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden index a2388a430..44f743ff0 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden b/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden index 9998c09e6..67cdd15de 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden b/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden index 8402d13d2..2773395c2 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden b/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden index 02cbf4f9c..de24698a6 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden index e10337a55..d6f1109d1 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden index cc81fa5e5..f5a9ac81f 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden index d99c3c7d9..36e9732be 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNICentOS.golden b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNICentOS.golden index d768d7037..c52d7bc9c 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNICentOS.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNICentOS.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden index 570b7ce24..d12c127b4 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCentOS.golden b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCentOS.golden index fff2d10b9..83c52ddc9 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCentOS.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCentOS.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync From e279aec40b8fffb21c0ba6acc27051da1a4bbe3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Mon, 8 Feb 2021 17:50:41 +0100 Subject: [PATCH 13/21] Restart unhealthy API servers when provisioning/upgrading clusters (#1243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/scripts/node.go | 15 +++++++++++++++ pkg/tasks/nodes.go | 12 ++++++++++++ pkg/tasks/tasks.go | 2 ++ 3 files changed, 29 insertions(+) diff --git a/pkg/scripts/node.go b/pkg/scripts/node.go index 195a902ec..6f13606ef 100644 --- a/pkg/scripts/node.go +++ b/pkg/scripts/node.go @@ -29,6 +29,17 @@ var ( sudo KUBECONFIG=/etc/kubernetes/admin.conf \ kubectl drain {{ .NODE_NAME }} --ignore-daemonsets --delete-local-data `) + + restartKubeAPIServerTemplate = heredoc.Doc(` + apiserver_id=$(sudo crictl ps --name=kube-apiserver -q) + [ -z "$apiserver_id" ] && exit 1 + + sudo crictl logs "$apiserver_id" > /tmp/kube-apiserver.log 2>&1 + if sudo grep -q "etcdserver: no leader" /tmp/kube-apiserver.log; then + sudo crictl rm "$apiserver_id" + sleep 10 + fi + `) ) func DrainNode(nodeName string) (string, error) { @@ -40,3 +51,7 @@ func DrainNode(nodeName string) (string, error) { func Hostname() string { return hostnameScript } + +func RestartKubeAPIServer() string { + return restartKubeAPIServerTemplate +} diff --git a/pkg/tasks/nodes.go b/pkg/tasks/nodes.go index a827dcc3e..043f74d05 100644 --- a/pkg/tasks/nodes.go +++ b/pkg/tasks/nodes.go @@ -56,3 +56,15 @@ func uncordonNode(s *state.State, host kubeoneapi.HostConfig) error { return errors.WithStack(updateErr) } + +func restartKubeAPIServer(s *state.State) error { + s.Logger.Infoln("Restarting unhealthy API servers if needed...") + return s.RunTaskOnControlPlane(func(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connection) error { + _, _, err := s.Runner.Run(scripts.RestartKubeAPIServer(), nil) + if err != nil { + return err + } + + return nil + }, state.RunSequentially) +} diff --git a/pkg/tasks/tasks.go b/pkg/tasks/tasks.go index 88e1d8a80..6768c29a0 100644 --- a/pkg/tasks/tasks.go +++ b/pkg/tasks/tasks.go @@ -114,6 +114,7 @@ func WithFullInstall(t Tasks) Tasks { {Fn: repairClusterIfNeeded, ErrMsg: "failed to repair cluster"}, {Fn: joinControlplaneNode, ErrMsg: "failed to join other masters a cluster"}, {Fn: saveKubeconfig, ErrMsg: "failed to save kubeconfig to the local machine"}, + {Fn: restartKubeAPIServer, ErrMsg: "failed to restart unhealthy kube-apiserver"}, }...). append(kubernetesResources()...). append( @@ -188,6 +189,7 @@ func WithUpgrade(t Tasks) Tasks { }...). append(kubernetesResources()...). append( + Task{Fn: restartKubeAPIServer, ErrMsg: "failed to restart unhealthy kube-apiserver"}, Task{Fn: upgradeStaticWorkers, ErrMsg: "unable to upgrade static worker nodes"}, Task{ Fn: upgradeMachineDeployments, From d2095b7f226cfeac95a6c7c06f4c4dc3e6f56191 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Mon, 8 Feb 2021 19:01:41 +0100 Subject: [PATCH 14/21] Add the Kubernetes 1.20 jobs (#1244) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- .prow.yaml | 374 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 336 insertions(+), 38 deletions(-) diff --git a/.prow.yaml b/.prow.yaml index 632a8b05e..bf2f3ecc0 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -123,7 +123,6 @@ presubmits: ######################################################### # E2E/Conformance tests (AWS, 1.18-1.19) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-aws-containerd-conformance-1.18 @@ -135,7 +134,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -163,7 +162,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -189,7 +188,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -205,10 +204,35 @@ presubmits: resources: requests: cpu: 1 + + - name: pull-kubeone-e2e-aws-conformance-1.20 + run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-aws: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "aws" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "conformance" + resources: + requests: + cpu: 1 ######################################################### # E2E/Conformance tests (DigitalOcean, 1.18-1.19) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-digitalocean-conformance-1.18 @@ -220,7 +244,7 @@ presubmits: preset-digitalocean: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -246,7 +270,7 @@ presubmits: preset-digitalocean: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -263,9 +287,34 @@ presubmits: requests: cpu: 1 + - name: pull-kubeone-e2e-digitalocean-conformance-1.20 + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-digitalocean: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "digitalocean" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "conformance" + resources: + requests: + cpu: 1 + ######################################################### # E2E/Conformance tests (Hetzner, 1.18-1.19) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-hetzner-conformance-1.18 @@ -277,7 +326,7 @@ presubmits: preset-hetzner: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -303,7 +352,7 @@ presubmits: preset-hetzner: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -320,9 +369,34 @@ presubmits: requests: cpu: 1 + - name: pull-kubeone-e2e-hetzner-conformance-1.20 + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-hetzner: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "hetzner" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "conformance" + resources: + requests: + cpu: 1 + ######################################################### # E2E/Conformance tests (GCE, 1.18-1.19) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-gce-conformance-1.18 @@ -334,7 +408,7 @@ presubmits: preset-gce: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -362,7 +436,7 @@ presubmits: preset-gce: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -381,9 +455,36 @@ presubmits: requests: cpu: 1 + - name: pull-kubeone-e2e-gce-conformance-1.20 + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-gce: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "gce" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "conformance" + - name: TF_VAR_project + value: "kubeone-terraform-test" + resources: + requests: + cpu: 1 + ######################################################### # E2E/Conformance tests (Packet, 1.16-1.19) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-packet-conformance-1.18 @@ -395,7 +496,7 @@ presubmits: preset-packet: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -421,7 +522,7 @@ presubmits: preset-packet: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -437,10 +538,35 @@ presubmits: resources: requests: cpu: 1 + + - name: pull-kubeone-e2e-packet-conformance-1.20 + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-packet: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "packet" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "conformance" + resources: + requests: + cpu: 1 ######################################################### # E2E/Conformance tests (OpenStack, 1.18-1.19) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-openstack-conformance-1.18 @@ -452,7 +578,7 @@ presubmits: preset-openstack: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -478,7 +604,7 @@ presubmits: preset-openstack: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -494,10 +620,35 @@ presubmits: resources: requests: cpu: 1 + + - name: pull-kubeone-e2e-openstack-conformance-1.20 + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-openstack: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "openstack" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "conformance" + resources: + requests: + cpu: 1 ######################################################### # E2E/Upgrade tests (AWS) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-aws-upgrade-containerd-1.17-1.18 @@ -509,7 +660,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -536,7 +687,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -561,7 +712,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -576,10 +727,34 @@ presubmits: value: "1.19.7" - name: TEST_SET value: "upgrades" + + - name: pull-kubeone-e2e-aws-upgrade-1.19-1.20 + run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-aws: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "aws" + - name: TEST_CLUSTER_INITIAL_VERSION + value: "1.19.7" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "upgrades" ######################################################### # E2E/Upgrade tests (DigitalOcean) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-digitalocean-upgrade-1.17-1.18 @@ -591,7 +766,7 @@ presubmits: preset-digitalocean: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -616,7 +791,7 @@ presubmits: preset-digitalocean: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -631,10 +806,34 @@ presubmits: value: "1.19.7" - name: TEST_SET value: "upgrades" + + - name: pull-kubeone-e2e-digitalocean-upgrade-1.19-1.20 + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-digitalocean: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "digitalocean" + - name: TEST_CLUSTER_INITIAL_VERSION + value: "1.19.7" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "upgrades" ######################################################### # E2E/Upgrade tests (Hetzner) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-hetzner-upgrade-1.17-1.18 @@ -646,7 +845,7 @@ presubmits: preset-hetzner: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -671,7 +870,7 @@ presubmits: preset-hetzner: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -686,10 +885,34 @@ presubmits: value: "1.19.7" - name: TEST_SET value: "upgrades" + + - name: pull-kubeone-e2e-hetzner-upgrade-1.19-1.20 + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-hetzner: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "hetzner" + - name: TEST_CLUSTER_INITIAL_VERSION + value: "1.19.7" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "upgrades" ######################################################### # E2E/Upgrade tests (GCE) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-gce-upgrade-1.17-1.18 @@ -701,7 +924,7 @@ presubmits: preset-gce: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -728,7 +951,7 @@ presubmits: preset-gce: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -745,10 +968,36 @@ presubmits: value: "upgrades" - name: TF_VAR_project value: "kubeone-terraform-test" + + - name: pull-kubeone-e2e-gce-upgrade-1.19-1.20 + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-gce: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "gce" + - name: TEST_CLUSTER_INITIAL_VERSION + value: "1.19.7" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "upgrades" + - name: TF_VAR_project + value: "kubeone-terraform-test" ######################################################### # E2E/Upgrade tests (Packet) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-packet-upgrade-1.17-1.18 @@ -760,7 +1009,7 @@ presubmits: preset-packet: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -785,7 +1034,7 @@ presubmits: preset-packet: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -800,10 +1049,34 @@ presubmits: value: "1.19.7" - name: TEST_SET value: "upgrades" + + - name: pull-kubeone-e2e-packet-upgrade-1.19-1.20 + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-packet: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "packet" + - name: TEST_CLUSTER_INITIAL_VERSION + value: "1.19.7" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "upgrades" ######################################################### # E2E/Upgrade tests (OpenStack) - # TODO: Add 1.20 tests once #1222 is fixed ######################################################### - name: pull-kubeone-e2e-openstack-upgrade-1.17-1.18 @@ -815,7 +1088,7 @@ presubmits: preset-openstack: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -840,7 +1113,7 @@ presubmits: preset-openstack: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -855,6 +1128,31 @@ presubmits: value: "1.19.7" - name: TEST_SET value: "upgrades" + + - name: pull-kubeone-e2e-openstack-upgrade-1.19-1.20 + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" + labels: + preset-goproxy: "true" + preset-openstack: "true" + spec: + containers: + - image: kubermatic/kubeone-e2e:v0.1.13 + imagePullPolicy: Always + command: + - make + args: + - e2e-test + env: + - name: PROVIDER + value: "openstack" + - name: TEST_CLUSTER_INITIAL_VERSION + value: "1.19.7" + - name: TEST_CLUSTER_TARGET_VERSION + value: "1.20.2" + - name: TEST_SET + value: "upgrades" postsubmits: - name: ci-push-kubeone-e2e-image From c6f210b29599760101c0850ebf9760496f998d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Tue, 9 Feb 2021 12:33:47 +0100 Subject: [PATCH 15/21] Use Docker for restarting API server on Flatcar (#1245) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/scripts/node.go | 21 ++++++++++++++++++--- pkg/tasks/nodes.go | 31 +++++++++++++++++++++++++------ 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/pkg/scripts/node.go b/pkg/scripts/node.go index 6f13606ef..38f172997 100644 --- a/pkg/scripts/node.go +++ b/pkg/scripts/node.go @@ -30,7 +30,7 @@ var ( kubectl drain {{ .NODE_NAME }} --ignore-daemonsets --delete-local-data `) - restartKubeAPIServerTemplate = heredoc.Doc(` + restartKubeAPIServerCrictlTemplate = heredoc.Doc(` apiserver_id=$(sudo crictl ps --name=kube-apiserver -q) [ -z "$apiserver_id" ] && exit 1 @@ -40,6 +40,17 @@ var ( sleep 10 fi `) + + restartKubeAPIServerDockerTemplate = heredoc.Doc(` + apiserver_id=$(sudo docker ps --filter="name=k8s_kube-apiserver" -q) + [ -z "$apiserver_id" ] && exit 1 + + sudo docker logs "$apiserver_id" > /tmp/kube-apiserver.log 2>&1 + if sudo grep -q "etcdserver: no leader" /tmp/kube-apiserver.log; then + sudo docker rm -f "$apiserver_id" + sleep 10 + fi + `) ) func DrainNode(nodeName string) (string, error) { @@ -52,6 +63,10 @@ func Hostname() string { return hostnameScript } -func RestartKubeAPIServer() string { - return restartKubeAPIServerTemplate +func RestartKubeAPIServerCrictl() string { + return restartKubeAPIServerCrictlTemplate +} + +func RestartKubeAPIServerDocker() string { + return restartKubeAPIServerDockerTemplate } diff --git a/pkg/tasks/nodes.go b/pkg/tasks/nodes.go index 043f74d05..5b7fc8752 100644 --- a/pkg/tasks/nodes.go +++ b/pkg/tasks/nodes.go @@ -59,12 +59,31 @@ func uncordonNode(s *state.State, host kubeoneapi.HostConfig) error { func restartKubeAPIServer(s *state.State) error { s.Logger.Infoln("Restarting unhealthy API servers if needed...") - return s.RunTaskOnControlPlane(func(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connection) error { - _, _, err := s.Runner.Run(scripts.RestartKubeAPIServer(), nil) - if err != nil { - return err - } - return nil + return s.RunTaskOnControlPlane(func(s *state.State, node *kubeoneapi.HostConfig, _ ssh.Connection) error { + return restartKubeAPIServerOnOS(s, *node) }, state.RunSequentially) } + +func restartKubeAPIServerOnOS(s *state.State, node kubeoneapi.HostConfig) error { + return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ + kubeoneapi.OperatingSystemNameAmazon: restartKubeAPIServerCrictl, + kubeoneapi.OperatingSystemNameCentOS: restartKubeAPIServerCrictl, + kubeoneapi.OperatingSystemNameDebian: restartKubeAPIServerCrictl, + kubeoneapi.OperatingSystemNameFlatcar: restartKubeAPIServerDocker, + kubeoneapi.OperatingSystemNameRHEL: restartKubeAPIServerCrictl, + kubeoneapi.OperatingSystemNameUbuntu: restartKubeAPIServerCrictl, + }) +} + +func restartKubeAPIServerCrictl(s *state.State) error { + _, _, err := s.Runner.RunRaw(scripts.RestartKubeAPIServerCrictl()) + + return errors.WithStack(err) +} + +func restartKubeAPIServerDocker(s *state.State) error { + _, _, err := s.Runner.RunRaw(scripts.RestartKubeAPIServerDocker()) + + return errors.WithStack(err) +} From b9fa601a7d105c8dc332df18702cc40f32fc1600 Mon Sep 17 00:00:00 2001 From: Mohamed El Sayed Date: Thu, 11 Feb 2021 18:04:50 +0100 Subject: [PATCH 16/21] Add encryption providers proposal (#1213) * Add encryption providers proposal * remove custom configuration * add apply flag * updated custom config --- .../proposals/20210112-encryption-roviders.md | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/proposals/20210112-encryption-roviders.md diff --git a/docs/proposals/20210112-encryption-roviders.md b/docs/proposals/20210112-encryption-roviders.md new file mode 100644 index 000000000..eb4ac7c2b --- /dev/null +++ b/docs/proposals/20210112-encryption-roviders.md @@ -0,0 +1,115 @@ +# Encryption Providers for encrypted secrets at rest + +**Auther**: Mohamed Elsayed (@moelsayed) +**Status**: Draft + + +## Abstract + +By default, all Kubernetes secret objects are stored on disk in plain text inside etcd. The [Encryption Providers](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/ +) feature was added to Kubernets starting with version 1.13. + +At rest data encryption is a requirement for security compliance and adds an additional layer of security for secret data, especially when etcd nodes are separated from the control plan and in off-node backups. + +KubeOne needs to support this feature natively. Meaning the user should be able to enable, disable the feature and rotate keys when needed without having to apply any actions manually. + +## Goals + +* Provide a safe path to enable/disable Encryption Providers. +* Support atomic(?) rotation for existing keys. +* Rewriting all secret resources (no just secrets) after enable/disable/rotate operations. + +## Non-Goals + +* Deploy External KMS. +* Safely manage (disable/enable/rotate) configuration when a custom configuration file is used. + +## Challenges + +The feature has a lot of moving parts; as it requires performing a specific sequence of actions, including changing the KubeAPI configuration, restarting KubeAPI and rewriting all secret resources to apply the encryption. This requires the implementation to be as idempotent as possible with ability to rollback on failure, with out breaking the cluster. + +## Implementation + +Unfortunately, it's not possible to simply update the KubeAPI configuration and expect the configuration to reconcile. KubeOne will have to _read_ the _current_ configuration on the cluster, _mutate_ it based on the _required_ state and then apply it. Additionally, KubeOne will have to be able to revert changes on any errors and recover safely if the process is interrupted at any point. + +The configuration for this will be added under `features` in the KubeOneCluster spec: + +```yaml +apiVersion: kubeone.io/v1beta1 +kind: KubeOneCluster +features: + encryptionProviders: + enabled: true + customProvidersFile: | + apiVersion: apiserver.config.k8s.io/v1 + kind: EncryptionConfiguration + resources: + - resources: + - secrets + providers: + - identity: {} + - aescbc: + keys: + - name: key1 + secret: +``` + +To allow users to rotate the keys, a new flag will be added to the `apply` command: + +```bash +--rotate-encryption-key automatically rotate encryption provider key +``` + +### pre-flight checks + + * Cluster is healthy. + * Current Encryption Providers state/configuration is valid and identical on all control plane nodes. + +### Enable Encryption Providers for new cluster + +* Generate a valid configuration file with the `identity` provider set last. +* Sync the configuration file to all Control Plane nodes. +* Set the required KubeAPI configuration and deploy KubeAPI. + +### Enable Encryption Providers for existing cluster + +* Ensure there is no Encryption Provider Config (manually added by the user, broken previous enable process, etc..) present. +* Generate a valid configuration file with the `identity` provider set last. +* Sync the configuration file to all Control Plane nodes. +* Update and restart KubeAPI on all nodes. +* Rewrite secrets to ensure they are encrypted successfully. + +### Disable Encryption Providers for existing cluster + +* Read the current active Encryption Provider configuration from control plane nodes. +* Mutate the configuration to add `identity` provider first and the active provider last. +* Sync the configuration file to all Control Plane nodes. +* Restart KubeAPI on all control plane nodes. +* Rewrite secrets to ensure they are decrypted successfully. +* Update KubeAPI configuration to remove the Encryption Provider configuration and restart KubeAPI on all control plane nodes. +* Remove the old configuration file from all control plane nodes. + +### Rotate Encryption Provider Key for existing cluster + +* Read the current active Encryption Provider configuration from control plane nodes. +* Generate a new encryption key. +* Mutate the configuration file to include the new key first, current key second and `identity` last. +* Sync the updated configuration file to all control plane nodes and restart KubeAPI. +* Rewrite all secrets to ensure they are encrypted with the new key. +* Mutate the configuration file again to remove the old key. +* Sync the updated configuration file to all control plane nodes and restart KubeAPI. + +### Apply Custom Encryption Provider file +This use case is useful for users who would like to utilize an external KMS provider or specify resources other than secrets for encryption. In this case, KubeOne will not manage the content of the file, it will only validate it to make sure it's syntactically valid. Additionally, KubeOne will not rewrite the resources in this case. + +* Ensure the configuration file is valid. +* Sync the configuration file to all control plane nodes. +* Restart KubeAPI on all nodes. + +## Tasks & effort + +* Implement the needed pre-flight checks. +* Implement validation for Encryption Provider configuration files. +* Implement the workflow for each use case. +* Add e2e tests for each workflow. +* Add documentation for the feature. \ No newline at end of file From eb97eae6cd11f6b723a93bc24c8b4df7868e350b Mon Sep 17 00:00:00 2001 From: Christian Rebischke Date: Tue, 16 Feb 2021 12:46:52 +0100 Subject: [PATCH 17/21] fix(docs): fix broken master documentation link (#1246) Signed-off-by: Christian Rebischke --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7a2679a53..bea6569f5 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Amazon gift card among the respondents of our KubeOne Survey. ## Getting Started -All user documentation is available at the +All user documentation for the latest stable version is available at the [KubeOne docs website][docs]. Information about the support policy (natively-supported providers, supported @@ -118,12 +118,12 @@ See [the list of releases][changelog] to find out about feature changes. [upstream-supported-versions]: https://kubernetes.io/docs/setup/release/version-skew-policy/#supported-versions [cluster-api]: https://github.com/kubernetes-sigs/cluster-api [machine-controller]: https://github.com/kubermatic/machine-controller -[docs]: https://docs.kubermatic.com/kubeone/master/ -[docs-compatibility]: https://docs.kubermatic.com/kubeone/master/compatibility_info/ -[docs-prerequisistes]: https://docs.kubermatic.com/kubeone/master/prerequisites/ -[docs-infrastructure]: https://docs.kubermatic.com/kubeone/master/infrastructure/ -[docs-provisioning]: https://docs.kubermatic.com/kubeone/master/provisioning/ -[docs-install]: https://docs.kubermatic.com/kubeone/master/getting_kubeone/ +[docs]: https://docs.kubermatic.com/kubeone/ +[docs-compatibility]: https://docs.kubermatic.com/kubeone/v1.2/compatibility_info/ +[docs-prerequisistes]: https://docs.kubermatic.com/kubeone/v1.2/prerequisites/ +[docs-infrastructure]: https://docs.kubermatic.com/kubeone/v1.2/infrastructure/ +[docs-provisioning]: https://docs.kubermatic.com/kubeone/v1.2/provisioning/ +[docs-install]: https://docs.kubermatic.com/kubeone/v1.2/getting_kubeone/ [contributing-guide]: https://github.com/kubermatic/KubeOne/blob/master/CONTRIBUTING.md [k8s-slack-kubeone]: https://kubernetes.slack.com/messages/CNEV2UMT7 [k8s-slack]: http://slack.k8s.io/ From c824810769d4ce55b3cfdc560b46b6563c8c509e Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Wed, 17 Feb 2021 12:07:57 +0200 Subject: [PATCH 18/21] Fix credentials in addons (#1248) Signed-off-by: Artiom Diomin --- pkg/addons/addons.go | 3 +- pkg/credentials/credentials.go | 163 +++++++++++++++++++-------------- 2 files changed, 97 insertions(+), 69 deletions(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index acf3a882b..6c0c2356e 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -49,10 +49,11 @@ func Ensure(s *state.State) error { } s.Logger.Infoln("Applying addons...") - creds, err := credentials.ProviderCredentials(s.Cluster.CloudProvider, s.CredentialsFilePath) + creds, err := credentials.Any(s.CredentialsFilePath) if err != nil { return errors.Wrap(err, "unable to fetch credentials") } + templateData := TemplateData{ Config: s.Cluster, Credentials: creds, diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index 4354b67df..3e261a50c 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -22,7 +22,7 @@ import ( "os" "strings" - "github.com/aws/aws-sdk-go/aws/credentials" + awscredentials "github.com/aws/aws-sdk-go/aws/credentials" "github.com/pkg/errors" "gopkg.in/yaml.v2" @@ -68,6 +68,32 @@ const ( VSphereUsernameMC = "VSPHERE_USERNAME" ) +var ( + allKeys = []string{ + AWSAccessKeyID, + AWSSecretAccessKey, + AzureClientID, + AzureClientSecret, + AzureTenantID, + AzureSubscribtionID, + DigitalOceanTokenKey, + GoogleServiceAccountKey, + HetznerTokenKey, + OpenStackAuthURL, + OpenStackDomainName, + OpenStackPassword, + OpenStackRegionName, + OpenStackTenantID, + OpenStackTenantName, + OpenStackUserName, + PacketAPIKey, + PacketProjectID, + VSphereAddress, + VSpherePassword, + VSphereUsername, + } +) + // ProviderEnvironmentVariable is used to match environment variable used by KubeOne to environment variable used by // machine-controller. type ProviderEnvironmentVariable struct { @@ -75,29 +101,46 @@ type ProviderEnvironmentVariable struct { MachineControllerName string } +func Any(credentialsFilePath string) (map[string]string, error) { + credentialsFinder, err := newCredsFinder(credentialsFilePath) + if err != nil { + return nil, err + } + + creds := map[string]string{} + + for _, key := range allKeys { + if val := credentialsFinder(key); val != "" { + creds[key] = val + } + } + + return creds, nil +} + // ProviderCredentials implements fetching credentials for each supported provider -func ProviderCredentials(p kubeone.CloudProviderSpec, credentialsFilePath string) (map[string]string, error) { - f, err := newFetcher(credentialsFilePath) +func ProviderCredentials(cloudProvider kubeone.CloudProviderSpec, credentialsFilePath string) (map[string]string, error) { + credentialsFinder, err := newCredsFinder(credentialsFilePath) if err != nil { return nil, err } switch { - case p.AWS != nil: - return f.parseAWSCredentials() - case p.Azure != nil: - return f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.AWS != nil: + return credentialsFinder.aws() + case cloudProvider.Azure != nil: + return credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: AzureClientID, MachineControllerName: AzureClientIDMC}, {Name: AzureClientSecret, MachineControllerName: AzureClientSecretMC}, {Name: AzureTenantID, MachineControllerName: AzureTenantIDMC}, {Name: AzureSubscribtionID, MachineControllerName: AzureSubscribtionIDMC}, }, defaultValidationFunc) - case p.DigitalOcean != nil: - return f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.DigitalOcean != nil: + return credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: DigitalOceanTokenKey, MachineControllerName: DigitalOceanTokenKeyMC}, }, defaultValidationFunc) - case p.GCE != nil: - gsa, err := f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.GCE != nil: + gsa, err := credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: GoogleServiceAccountKey, MachineControllerName: GoogleServiceAccountKeyMC}, }, defaultValidationFunc) if err != nil { @@ -107,12 +150,12 @@ func ProviderCredentials(p kubeone.CloudProviderSpec, credentialsFilePath string // machine-controller, as machine-controller assumes it will be double encoded gsa[GoogleServiceAccountKeyMC] = base64.StdEncoding.EncodeToString([]byte(gsa[GoogleServiceAccountKeyMC])) return gsa, nil - case p.Hetzner != nil: - return f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.Hetzner != nil: + return credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: HetznerTokenKey, MachineControllerName: HetznerTokenKeyMC}, }, defaultValidationFunc) - case p.Openstack != nil: - return f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.Openstack != nil: + return credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: OpenStackAuthURL}, {Name: OpenStackUserName, MachineControllerName: OpenStackUserNameMC}, {Name: OpenStackPassword}, @@ -121,13 +164,13 @@ func ProviderCredentials(p kubeone.CloudProviderSpec, credentialsFilePath string {Name: OpenStackTenantID}, {Name: OpenStackTenantName}, }, openstackValidationFunc) - case p.Packet != nil: - return f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.Packet != nil: + return credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: PacketAPIKey, MachineControllerName: PacketAPIKeyMC}, {Name: PacketProjectID}, }, defaultValidationFunc) - case p.Vsphere != nil: - vscreds, err := f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.Vsphere != nil: + vscreds, err := credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: VSphereAddress, MachineControllerName: VSphereAddressMC}, {Name: VSphereUsername, MachineControllerName: VSphereUsernameMC}, {Name: VSpherePassword}, @@ -138,67 +181,50 @@ func ProviderCredentials(p kubeone.CloudProviderSpec, credentialsFilePath string // force scheme, as machine-controller requires it while terraform does not vscreds[VSphereAddressMC] = "https://" + vscreds[VSphereAddressMC] return vscreds, nil - case p.None != nil: + case cloudProvider.None != nil: return map[string]string{}, nil } return nil, errors.New("no provider matched") } -type fetcher struct { - // Source is custom source for credentials, by default environment is used - Source map[string]string - // F is function that retrieves variable from the source - F func(string) string -} +func newCredsFinder(credentialsFilePath string) (lookupFunc, error) { + staticMap := map[string]string{} + finder := func(name string) string { + if val := os.Getenv(name); val != "" { + return val + } + return staticMap[name] + } -func newFetcher(credentialsFilePath string) (*fetcher, error) { - f := &fetcher{ - F: os.Getenv, + if credentialsFilePath == "" { + return finder, nil } - if credentialsFilePath != "" { - b, err := ioutil.ReadFile(credentialsFilePath) - if err != nil { - return nil, errors.Wrap(err, "unable to load credentials file") - } - m := make(map[string]string) - err = yaml.Unmarshal(b, &m) - if err != nil { - return nil, errors.Wrap(err, "unable to unmarshal credentials file") - } - f.Source = m - f.F = func(name string) string { - return m[name] - } + buf, err := ioutil.ReadFile(credentialsFilePath) + if err != nil { + return nil, errors.Wrap(err, "unable to load credentials file") } - return f, nil + if err = yaml.Unmarshal(buf, &staticMap); err != nil { + return nil, errors.Wrap(err, "unable to unmarshal credentials file") + } + + return finder, nil } -func (f *fetcher) parseAWSCredentials() (map[string]string, error) { - if f.Source != nil { - return map[string]string{ - AWSAccessKeyID: f.F(AWSAccessKeyID), - AWSSecretAccessKey: f.F(AWSSecretAccessKey), - }, nil - } +// lookupFunc is function that retrieves credentials from the sources +type lookupFunc func(string) string +func (lookup lookupFunc) aws() (map[string]string, error) { creds := make(map[string]string) - envCredsProvider := credentials.NewEnvCredentials() + accessKeyID := lookup(AWSAccessKeyID) + secretAccessKey := lookup(AWSSecretAccessKey) - // will error out in case when ether ID or KEY are missing from ENV - envCreds, err := envCredsProvider.Get() - - switch err { - case nil: - creds[AWSAccessKeyID] = envCreds.AccessKeyID - creds[AWSSecretAccessKey] = envCreds.SecretAccessKey + if accessKeyID != "" && secretAccessKey != "" { + creds[AWSAccessKeyID] = accessKeyID + creds[AWSSecretAccessKey] = secretAccessKey return creds, nil - case credentials.ErrSecretAccessKeyNotFound, credentials.ErrAccessKeyIDNotFound: - // ignore above errors to continue to shared credentials method - default: - return nil, errors.WithStack(err) } if os.Getenv("AWS_PROFILE") == "" { @@ -209,10 +235,10 @@ func (f *fetcher) parseAWSCredentials() (map[string]string, error) { } // If env fails resort to config file - configCredsProvider := credentials.NewSharedCredentials("", "") + sharedCredsProvider := awscredentials.NewSharedCredentials("", "") // will error out in case when ether ID or KEY are missing from shared file - configCreds, err := configCredsProvider.Get() + configCreds, err := sharedCredsProvider.Get() if err != nil { return nil, errors.WithStack(err) } @@ -224,12 +250,13 @@ func (f *fetcher) parseAWSCredentials() (map[string]string, error) { return creds, nil } -func (f fetcher) parseCredentialVariables(envVars []ProviderEnvironmentVariable, validationFunc func(map[string]string) error) (map[string]string, error) { - // Validate credentials using given validation function +func (lookup lookupFunc) parseCredentialVariables(envVars []ProviderEnvironmentVariable, validationFunc func(map[string]string) error) (map[string]string, error) { creds := make(map[string]string) for _, env := range envVars { - creds[env.Name] = strings.TrimSpace(f.F(env.Name)) + creds[env.Name] = strings.TrimSpace(lookup(env.Name)) } + + // Validate credentials using given validation function if err := validationFunc(creds); err != nil { return nil, errors.Wrap(err, "unable to validate credentials") } From ed0282cc587cfc5313c92db2384605827981253e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Wed, 17 Feb 2021 13:52:57 +0100 Subject: [PATCH 19/21] Add the changelog for the v1.2.0-beta.1 release (#1249) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- CHANGELOG.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffca720dd..4a50609dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,64 @@ # Changelog +# [v1.2.0-beta.1](https://github.com/kubermatic/kubeone/releases/tag/v1.2.0-beta.1) - 2021-02-17 + +## Attention Needed + +* [**Breaking**] Support for CoreOS has been removed from KubeOne and machine-controller + * CoreOS has reached End-of-Life on May 26, 2020 + * As an alternative to CoreOS, KubeOne supports Flatcar Linux + * We recommend migrating your CoreOS clusters to the Flatcar Linux or other supported operating system +* [**Breaking**] Default values for OpenIDConnect has been corrected to match what's advised by the example configuration + * Previously, there were no default values for the OpenIDConnect fields + * This might only affect users using the OpenIDConnect feature +* [**Breaking**] Disallow and deprecate the PodPresets feature + * [**Action Required**] If you're upgrading a cluster that uses the PodPresets feature from Kubernetes 1.19 to 1.20, you have to disable the PodPresets feature in the KubeOne configuration manifest + * The PodPresets feature has been removed from Kubernetes 1.20 with no built-in replacement + * It's not possible to use the PodPresets feature starting with Kubernetes 1.20, however, it currently remains possible to use it for older Kubernetes versions + * The PodPresets feature will be removed from the KubeOneCluster API once Kubernetes 1.19 reaches End-of-Life (EOL) + * As an alternative to the PodPresets feature, Kubernetes recommends using the MutatingAdmissionWebhooks. + +## Added + +* Add support for Kubernetes 1.20 + * Previously, we've shared that there is an issue affecting newly created clusters where the first control plane node is unhealthy/broken for the first 5-10 minutes. We've investigated the issue and found out that the issue can be successfully mitigated by restarting the first API server. We've implemented a task that automatically restarts the API server if it's affected by the issue ([#1243](https://github.com/kubermatic/kubeone/pull/1243), [#1245](https://github.com/kubermatic/kubeone/pull/1245)) +* Add support for Debian on control plane and static worker nodes ([#1233](https://github.com/kubermatic/kubeone/pull/1233)) + * Debian is currently not supported by machine-controller, so it's not possible to use it on worker nodes managed by machine-controller + +## Changed + +### API Changes + +* [**Breaking**] Default values for OpenIDConnect has been corrected to match what's advised by the example configuration ([#1235](https://github.com/kubermatic/kubeone/pull/1235)) + * Previously, there were no default values for the OpenIDConnect fields + * This might only affect users using the OpenIDConnect feature +* [**Breaking**] Disallow and deprecate the PodPresets feature ([#1236](https://github.com/kubermatic/kubeone/pull/1236)) + * [**Action Required**] If you're upgrading a cluster that uses the PodPresets feature from Kubernetes 1.19 to 1.20, you have to disable the PodPresets feature in the KubeOne configuration manifest + * The PodPresets feature has been removed from Kubernetes 1.20 with no built-in replacement + * It's not possible to use the PodPresets feature starting with Kubernetes 1.20, however, it currently remains possible to use it for older Kubernetes versions + * The PodPresets feature will be removed from the KubeOneCluster API once Kubernetes 1.19 reaches End-of-Life (EOL) + * As an alternative to the PodPresets feature, Kubernetes recommends using the MutatingAdmissionWebhooks. + +### General + +* Add rsync on CentOS and Amazon Linux ([#1240](https://github.com/kubermatic/kubeone/pull/1240)) + +### Bug Fixes + +* Drop mounting Flexvolume plugins into the OpenStack CCM. This fixes the issue with deploying the OpenStack CCM on the clusters running Flatcar Linux ([#1234](https://github.com/kubermatic/kubeone/pull/1234)) +* Ensure all credentials are available to be used in addons. This fixes the issue with the Backups addon not working on non-AWS providers ([#1248](https://github.com/kubermatic/kubeone/pull/1248)) + +### Updated + +* Update machine-controller to v1.25.0 ([#1238](https://github.com/kubermatic/kubeone/pull/1238)) + +## Removed + +* [**Breaking**] Support for CoreOS has been removed from KubeOne and machine-controller ([#1232](https://github.com/kubermatic/kubeone/pull/1232)) + * CoreOS has reached End-of-Life on May 26, 2020 + * As an alternative to CoreOS, KubeOne supports Flatcar Linux + * We recommend migrating your CoreOS clusters to the Flatcar Linux or other supported operating system + # [v1.2.0-beta.0](https://github.com/kubermatic/kubeone/releases/tag/v1.2.0-beta.0) - 2021-01-27 ## Attention Needed From 168338870a5eb01a4c4d2831ad84ac415ccd479e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Fri, 19 Feb 2021 12:49:57 +0100 Subject: [PATCH 20/21] Update README.md (#1250) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index bea6569f5..98b1f0094 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,6 @@ Kubermatic KubeOne automates cluster operations on all your cloud, on-prem, edge, and IoT environments. KubeOne can install high-available (HA) master clusters as well single master clusters. -## KubeOne User Survey - -**We're organizing the [KubeOne User Survey][survey]!** -This survey is intended to shape the future roadmap of KubeOne. Your answers -will help us determine future features and schedules. We’re raffling one 10€ -Amazon gift card among the respondents of our KubeOne Survey. - ## Getting Started All user documentation for the latest stable version is available at the From 0774acca90a8ca127114701a2766444f2ac7cd2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Wed, 24 Feb 2021 14:02:08 +0100 Subject: [PATCH 21/21] Improve the installation script (#1253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- README.md | 5 +++-- install.sh | 45 +++++++++++++++++++++------------------------ 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 98b1f0094..5d0ca5cd2 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,9 @@ curl -sfL get.kubeone.io | sh ``` The installation script downloads the release archive from GitHub, installs the -KubeOne binary in your `/usr/local/bin` directory and unpacks the example -Terraform configs in your current working directory. +KubeOne binary in your `/usr/local/bin` directory, and unpacks the example +Terraform configs, addons, and helper scripts in your current working +directory. For other installation methods, check the [Getting KubeOne guide][docs-install] on our documentation website. diff --git a/install.sh b/install.sh index 9b93a3bbb..15374eb9e 100755 --- a/install.sh +++ b/install.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh # Copyright 2019 The KubeOne Authors. # @@ -14,38 +14,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -# -# This is a simple installer script for KubeOne # -# +# This is a simple installer script for KubeOne. + +set -eu # What OS is used -OS=$(uname) -# find out what's the latest version -VERSION=$(curl -w '%{url_effective}' -I -L -s -S https://github.com/kubermatic/kubeone/releases/latest -o /dev/null | sed -e 's|.*/v||') -# download URL for the latest version +OS="$(uname | tr '[:upper:]' '[:lower:]')" +# Find out what's the latest version +VERSION="$(curl -w '%{url_effective}' -I -L -s -S https://github.com/kubermatic/kubeone/releases/latest -o /dev/null | sed -e 's|.*/v||')" +# Download URL for the latest version URL="https://github.com/kubermatic/kubeone/releases/download/v${VERSION}/kubeone_${VERSION}_${OS}_amd64.zip" -# 'kubeone' will be installed into this dir: +# 'kubeone' will be installed into this dir DEST=/usr/local/bin # Download the latest version for the OS and save it as zip - if curl -LO "$URL" -then - echo "Copying kubeone binary into $DEST" - # unpack: - - - if unzip "kubeone_${VERSION}_${OS}_amd64.zip" -d "kubeone_${VERSION}_${OS}_amd64" - then - sudo mv "kubeone_${VERSION}_${OS}_amd64/kubeone" "$DEST" - rm "kubeone_${VERSION}_${OS}_amd64.zip" - rm -rf "kubeone_${VERSION}_${OS}_amd64" - echo "kubeone has been installed into $DEST/kubeone" - exit 0 - fi +then + echo "Copying kubeone binary into $DEST" + + if unzip "kubeone_${VERSION}_${OS}_amd64.zip" -d "kubeone_${VERSION}_${OS}_amd64" + then + sudo mv "kubeone_${VERSION}_${OS}_amd64/kubeone" "$DEST" + rm "kubeone_${VERSION}_${OS}_amd64.zip" + echo "Kubermatic KubeOne has been installed into $DEST/kubeone" + echo "Terraform example configs, addons, and helper scripts have been downloaded into the ./kubeone_${VERSION}_${OS}_amd64 directory" + exit 0 + fi else - printf "Failed to determine your platform.\n Try downloading from https://github.com/kubermatic/kubeone/releases" + printf "Failed to determine your platform.\n Try downloading from https://github.com/kubermatic/kubeone/releases" fi exit 1