From d911c8c1ca0f3d501cd82803c43c0ffdea260c3b Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Wed, 12 Jun 2019 17:53:07 +0100 Subject: [PATCH] Use new CoreDNS manifests --- pkg/addons/default/coredns.go | 92 ++++++++++++++--------- pkg/addons/default/coredns_test.go | 114 +++++++++++++++++------------ pkg/ctl/utils/install_corends.go | 7 +- pkg/ctl/utils/update_coredns.go | 9 ++- 4 files changed, 139 insertions(+), 83 deletions(-) diff --git a/pkg/addons/default/coredns.go b/pkg/addons/default/coredns.go index 7f98c96fd57..87143d90017 100644 --- a/pkg/addons/default/coredns.go +++ b/pkg/addons/default/coredns.go @@ -8,8 +8,8 @@ import ( "github.com/kris-nova/logger" "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - extensionsv1beta1 "k8s.io/api/extensions/v1beta1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -17,8 +17,6 @@ import ( "k8s.io/apimachinery/pkg/watch" "github.com/weaveworks/eksctl/pkg/kubernetes" - "github.com/weaveworks/eksctl/pkg/printers" - k8s "k8s.io/client-go/kubernetes" ) const ( @@ -27,9 +25,6 @@ const ( // KubeDNS is the name of the kube-dns addon KubeDNS = "kube-dns" - // CoreDNSVersion Current latest coredns version supported - CoreDNSVersion = "v1.2.2" - componentLabel = "eks.amazonaws.com/component" coreDNSImagePrefix = "602401143452.dkr.ecr." @@ -37,7 +32,7 @@ const ( ) // InstallCoreDNS will install the `coredns` add-on in place of `kube-dns` -func InstallCoreDNS(rawClient kubernetes.RawClientInterface, region string, waitTimeout *time.Duration, plan bool) (bool, error) { +func InstallCoreDNS(rawClient kubernetes.RawClientInterface, region, controlPlaneVersion string, waitTimeout *time.Duration, plan bool) (bool, error) { kubeDNSSevice, err := rawClient.ClientSet().CoreV1().Services(metav1.NamespaceSystem).Get(KubeDNS, metav1.GetOptions{}) if err != nil { if apierrs.IsNotFound(err) { @@ -67,7 +62,7 @@ func InstallCoreDNS(rawClient kubernetes.RawClientInterface, region string, wait } // if kube-dns is present, go ahead and try to replace it with coredns - list, err := LoadAsset(CoreDNS, "yaml") + list, err := loadAssetCoreDNS(controlPlaneVersion) if err != nil { return false, err } @@ -81,7 +76,7 @@ func InstallCoreDNS(rawClient kubernetes.RawClientInterface, region string, wait } switch resource.GVK.Kind { case "Deployment": - coreDNSDeployemnt := resource.Info.Object.(*extensionsv1beta1.Deployment) + coreDNSDeployemnt := resource.Info.Object.(*appsv1.Deployment) listPodsOptions.LabelSelector = labels.FormatLabels(coreDNSDeployemnt.Spec.Selector.MatchLabels) replicas = int(*coreDNSDeployemnt.Spec.Replicas) image := &coreDNSDeployemnt.Spec.Template.Spec.Containers[0].Image @@ -167,11 +162,18 @@ func InstallCoreDNS(rawClient kubernetes.RawClientInterface, region string, wait return false, nil } -// UpdateCoreDNSImageTag updates image tag for kube-system:deployment/coredns based to match the latest release -func UpdateCoreDNSImageTag(clientSet k8s.Interface, plan bool) (bool, error) { - printer := printers.NewJSONPrinter() +// UpdateCoreDNS will update the `coredns` add-on +func UpdateCoreDNS(rawClient kubernetes.RawClientInterface, region, controlPlaneVersion string, plan bool) (bool, error) { + kubeDNSSevice, err := rawClient.ClientSet().CoreV1().Services(metav1.NamespaceSystem).Get(KubeDNS, metav1.GetOptions{}) + if err != nil { + if apierrs.IsNotFound(err) { + logger.Warning("%q service was not found", KubeDNS) + return false, nil + } + return false, errors.Wrapf(err, "getting %q service", KubeDNS) + } - d, err := clientSet.AppsV1().Deployments(metav1.NamespaceSystem).Get(CoreDNS, metav1.GetOptions{}) + _, err = rawClient.ClientSet().AppsV1().Deployments(metav1.NamespaceSystem).Get(CoreDNS, metav1.GetOptions{}) if err != nil { if apierrs.IsNotFound(err) { logger.Warning("%q was not found", CoreDNS) @@ -179,25 +181,41 @@ func UpdateCoreDNSImageTag(clientSet k8s.Interface, plan bool) (bool, error) { } return false, errors.Wrapf(err, "getting %q", CoreDNS) } - if numContainers := len(d.Spec.Template.Spec.Containers); !(numContainers >= 1) { - return false, fmt.Errorf("%s has %d containers, expected at least 1", CoreDNS, numContainers) - } - if err := printer.LogObj(logger.Debug, CoreDNS+" [current] = \\\n%s\n", d); err != nil { + // if Deployment is present, go through our list of assets + list, err := loadAssetCoreDNS(controlPlaneVersion) + if err != nil { return false, err } - image := &d.Spec.Template.Spec.Containers[0].Image - imageParts := strings.Split(*image, ":") + for _, rawObj := range list.Items { + resource, err := rawClient.NewRawResource(rawObj) + if err != nil { + return false, err + } + switch resource.GVK.Kind { + case "Deployment": + image := &resource.Info.Object.(*appsv1.Deployment).Spec.Template.Spec.Containers[0].Image + imageParts := strings.Split(*image, ":") - if len(imageParts) != 2 { - return false, fmt.Errorf("unexpected image format %q for %q", *image, CoreDNS) - } + if len(imageParts) != 2 { + return false, fmt.Errorf("unexpected image format %q for %q", *image, KubeProxy) + } - if imageParts[1] == CoreDNSVersion { - logger.Debug("imageParts = %v, desiredTag = %s", imageParts, CoreDNSVersion) - logger.Info("%q is already up-to-date", CoreDNS) - return false, nil + if strings.HasPrefix(imageParts[0], coreDNSImagePrefix) && + strings.HasSuffix(imageParts[0], coreDNSImageSuffix) { + *image = coreDNSImagePrefix + region + coreDNSImageSuffix + ":" + imageParts[1] + } + case "Service": + resource.Info.Object.(*corev1.Service).SetResourceVersion(kubeDNSSevice.GetResourceVersion()) + resource.Info.Object.(*corev1.Service).Spec.ClusterIP = kubeDNSSevice.Spec.ClusterIP + } + + status, err := resource.CreateOrReplace(plan) + if err != nil { + return false, err + } + logger.Info(status) } if plan { @@ -205,16 +223,20 @@ func UpdateCoreDNSImageTag(clientSet k8s.Interface, plan bool) (bool, error) { return true, nil } - imageParts[1] = CoreDNSVersion - *image = strings.Join(imageParts, ":") + logger.Info("%q is now up-to-date", CoreDNS) + return false, nil +} - if err := printer.LogObj(logger.Debug, CoreDNS+" [updated] = \\\n%s\n", d); err != nil { - return false, err +func loadAssetCoreDNS(controlPlaneVersion string) (*metav1.List, error) { + assetName := CoreDNS + if strings.HasPrefix(controlPlaneVersion, "1.10.") { + return nil, fmt.Errorf("CoreDNS is not supported on Kubernetes 1.10") } - if _, err := clientSet.AppsV1().Deployments(metav1.NamespaceSystem).Update(d); err != nil { - return false, err + if strings.HasPrefix(controlPlaneVersion, "1.11.") { + assetName += "-1.11" } - - logger.Info("%q is now up-to-date", CoreDNS) - return false, nil + if strings.HasPrefix(controlPlaneVersion, "1.12.") { + assetName += "-1.12" + } + return LoadAsset(assetName, "json") } diff --git a/pkg/addons/default/coredns_test.go b/pkg/addons/default/coredns_test.go index 0e9d3ad5c89..7ccf7e55b3e 100644 --- a/pkg/addons/default/coredns_test.go +++ b/pkg/addons/default/coredns_test.go @@ -10,21 +10,21 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/fake" ) var _ = Describe("default addons - coredns", func() { - Describe("can update from kubedns to coredns add-on", func() { - var ( - rawClient *testutils.FakeRawClient - ct *testutils.CollectionTracker - ) - - It("can load sample for 1.10 and create objests that don't exist", func() { - sampleAddons := testutils.LoadSamples("testdata/sample-1.10.json") - + var ( + rawClient *testutils.FakeRawClient + ct *testutils.CollectionTracker + ) + + loadSample := func(v string, expectCount int) { + samplePath := "testdata/sample-" + v + ".json" + It("can load "+samplePath+" and create objests that don't exist", func() { rawClient = testutils.NewFakeRawClient() + sampleAddons := testutils.LoadSamples(samplePath) + rawClient.UseUnionTracker = true for _, item := range sampleAddons { @@ -38,10 +38,13 @@ var _ = Describe("default addons - coredns", func() { Expect(ct.Updated()).To(BeEmpty()) Expect(ct.Created()).ToNot(BeEmpty()) - Expect(ct.CreatedItems()).To(HaveLen(6)) - - Expect(ct.CreatedItems()).To(HaveLen(6)) + Expect(ct.CreatedItems()).To(HaveLen(expectCount)) }) + } + + Describe("[1.10 –> 1.11] can update from kube-dns to coredns", func() { + + loadSample("1.10", 6) It("has newly created objects", func() { kubeDNS, err := rawClient.ClientSet().AppsV1().Deployments(metav1.NamespaceSystem).Get(KubeDNS, metav1.GetOptions{}) @@ -63,10 +66,10 @@ var _ = Describe("default addons - coredns", func() { Expect(err).ToNot(HaveOccurred()) // test client doesn't support watching, and we would have to create some pods, so we set nil timeout - _, err = InstallCoreDNS(rawClient, "eu-west-1", nil, false) + _, err = InstallCoreDNS(rawClient, "eu-west-1", "1.11.x", nil, false) Expect(err).ToNot(HaveOccurred()) - updateReqs := []string{ + createReqs := []string{ "POST [/namespaces/kube-system/deployments] (kube-dns)", "POST [/clusterrolebindings] (aws-node)", "POST [/namespaces/kube-system/serviceaccounts] (coredns)", @@ -79,8 +82,8 @@ var _ = Describe("default addons - coredns", func() { "POST [/namespaces/kube-system/daemonsets] (kube-proxy)", "POST [/clusterroles] (aws-node)", } - Expect(rawClient.Collection.Created()).To(HaveLen(len(updateReqs))) - for _, k := range updateReqs { + Expect(rawClient.Collection.Created()).To(HaveLen(len(createReqs))) + for _, k := range createReqs { Expect(rawClient.Collection.Created()).To(HaveKey(k)) } @@ -94,7 +97,7 @@ var _ = Describe("default addons - coredns", func() { Expect(svcNew.Spec.ClusterIP).To(Equal(svcOld.Spec.ClusterIP)) - coreDNS, err := rawClient.ClientSet().ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Get(CoreDNS, metav1.GetOptions{}) + coreDNS, err := rawClient.ClientSet().AppsV1().Deployments(metav1.NamespaceSystem).Get(CoreDNS, metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(coreDNS.Spec.Replicas).ToNot(BeNil()) Expect(*coreDNS.Spec.Replicas == 2).To(BeTrue()) @@ -106,41 +109,62 @@ var _ = Describe("default addons - coredns", func() { }) - Describe("can update coredns", func() { - var ( - clientSet *fake.Clientset - ) - - check := func(imageTag string) { - coreDNS, err := clientSet.AppsV1().Deployments(metav1.NamespaceSystem).Get(CoreDNS, metav1.GetOptions{}) + Context("[1.11 –> 1.12] can update coredns", func() { - Expect(err).ToNot(HaveOccurred()) - Expect(coreDNS).ToNot(BeNil()) - Expect(coreDNS.Spec.Template.Spec.Containers).To(HaveLen(1)) - - Expect(coreDNS.Spec.Template.Spec.Containers[0].Image).To( - Equal("602401143452.dkr.ecr.eu-west-1.amazonaws.com/eks/coredns:" + imageTag), - ) - } - - BeforeEach(func() { - clientSet, _ = testutils.NewFakeClientSetWithSamples("testdata/sample-1.11.json") - }) + loadSample("1.11", 10) It("can load 1.11 sample", func() { - check("v1.1.3") + checkCoreDNSImage(rawClient, "eu-west-1", "v1.1.3") }) It("can update based to latest version", func() { - _, err := UpdateCoreDNSImageTag(clientSet, false) + _, err := UpdateCoreDNS(rawClient, "eu-west-2", "1.12.x", false) Expect(err).ToNot(HaveOccurred()) - check("v1.2.2") - }) + checkCoreDNSImage(rawClient, "eu-west-2", "v1.2.2") - It("can dry-run update to latest version", func() { - _, err := UpdateCoreDNSImageTag(clientSet, true) - Expect(err).ToNot(HaveOccurred()) - check("v1.1.3") + createReqs := []string{ + "POST [/clusterrolebindings] (aws-node)", + "POST [/namespaces/kube-system/serviceaccounts] (coredns)", + "POST [/namespaces/kube-system/configmaps] (coredns)", + "POST [/namespaces/kube-system/services] (kube-dns)", + "POST [/namespaces/kube-system/daemonsets] (aws-node)", + "POST [/clusterroles] (system:coredns)", + "POST [/clusterrolebindings] (system:coredns)", + "POST [/namespaces/kube-system/deployments] (coredns)", + "POST [/namespaces/kube-system/daemonsets] (kube-proxy)", + "POST [/clusterroles] (aws-node)", + } + + Expect(rawClient.Collection.Created()).To(HaveLen(len(createReqs))) + for _, k := range createReqs { + Expect(rawClient.Collection.Created()).To(HaveKey(k)) + } + + updateReqs := []string{ + "PUT [/namespaces/kube-system/serviceaccounts/coredns] (coredns)", + "PUT [/namespaces/kube-system/configmaps/coredns] (coredns)", + "PUT [/namespaces/kube-system/services/kube-dns] (kube-dns)", + "PUT [/clusterroles/system:coredns] (system:coredns)", + "PUT [/clusterrolebindings/system:coredns] (system:coredns)", + "PUT [/namespaces/kube-system/deployments/coredns] (coredns)", + } + + Expect(rawClient.Collection.Updated()).To(HaveLen(len(updateReqs))) + for _, k := range updateReqs { + Expect(rawClient.Collection.Updated()).To(HaveKey(k)) + } }) }) }) + +func checkCoreDNSImage(rawClient *testutils.FakeRawClient, region, imageTag string) { + coreDNS, err := rawClient.ClientSet().AppsV1().Deployments(metav1.NamespaceSystem).Get(CoreDNS, metav1.GetOptions{}) + + Expect(err).ToNot(HaveOccurred()) + Expect(coreDNS).ToNot(BeNil()) + Expect(coreDNS.Spec.Template.Spec.Containers).To(HaveLen(1)) + + Expect(coreDNS.Spec.Template.Spec.Containers[0].Image).To( + Equal("602401143452.dkr.ecr." + region + ".amazonaws.com/eks/coredns:" + imageTag), + ) +} diff --git a/pkg/ctl/utils/install_corends.go b/pkg/ctl/utils/install_corends.go index 8e5425a9d02..f97298d0967 100644 --- a/pkg/ctl/utils/install_corends.go +++ b/pkg/ctl/utils/install_corends.go @@ -69,9 +69,14 @@ func doInstallCoreDNS(rc *cmdutils.ResourceCmd) error { return err } + kubernetesVersion, err := rawClient.ServerVersion() + if err != nil { + return err + } + waitTimeout := ctl.Provider.WaitTimeout() - updateRequired, err := defaultaddons.InstallCoreDNS(rawClient, meta.Region, &waitTimeout, rc.Plan) + updateRequired, err := defaultaddons.InstallCoreDNS(rawClient, meta.Region, kubernetesVersion, &waitTimeout, rc.Plan) if err != nil { return err } diff --git a/pkg/ctl/utils/update_coredns.go b/pkg/ctl/utils/update_coredns.go index 47f7eb194d2..3ab2e8698e5 100644 --- a/pkg/ctl/utils/update_coredns.go +++ b/pkg/ctl/utils/update_coredns.go @@ -54,12 +54,17 @@ func doUpdateCoreDNS(rc *cmdutils.ResourceCmd) error { return errors.Wrapf(err, "getting credentials for cluster %q", meta.Name) } - clientSet, err := ctl.NewStdClientSet(cfg) + rawClient, err := ctl.NewRawClient(cfg) if err != nil { return err } - updateRequired, err := defaultaddons.UpdateCoreDNSImageTag(clientSet, rc.Plan) + kubernetesVersion, err := rawClient.ServerVersion() + if err != nil { + return err + } + + updateRequired, err := defaultaddons.UpdateCoreDNS(rawClient, meta.Region, kubernetesVersion, rc.Plan) if err != nil { return err }