From b62c12b7f2b19283f419f0de3dd6212eb121f77f Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 20 Feb 2020 12:09:58 -0800 Subject: [PATCH 1/7] label minikube nodes --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 9eb428d8fa1e..eb296c5ded3a 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -54,6 +54,7 @@ import ( "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/vmpath" "k8s.io/minikube/pkg/util/retry" + "k8s.io/minikube/pkg/version" ) // Bootstrapper is a bootstrapper using kubeadm @@ -230,6 +231,10 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error { return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges") } } + fmt.Println("about to apply node labels") + if err := k.applyNodeLabels(cfg); err != nil { + glog.Warningf("unable to apply node labels: %v", err) + } if err := bsutil.AdjustResourceLimits(k.c); err != nil { glog.Warningf("unable to adjust resource limits: %v", err) @@ -483,3 +488,30 @@ func (k *Bootstrapper) applyKicOverlay(cfg config.MachineConfig) error { } return nil } + +// applyNodeLabels applies minikube labels to all the nodes +func (k *Bootstrapper) applyNodeLabels(cfg config.MachineConfig) error { + start := time.Now() + // based on ISO 8601 (RFC 3339) except converting - and : to _ because of kubernetes label restriction + createdAtLbl := "k8s.minikube.io/updated_at=" + start.Format("2006_01_02T15_04_05_0700") + verLbl := "k8s.minikube.io/version=" + version.GetVersion() + commitLbl := "k8s.minikube.io/commit=" + version.GetGitCommitID() + nameLbl := "k8s.minikube.io/name=" + cfg.Name + + // example: + // /var/lib/minikube/binaries/v1.17.3/kubectl label nodes --kubeconfig /var/lib/minikube/kubeconfig k8s.minikube.io/version=1.7.3 --all --overwrite + cmd := exec.Command("sudo", + path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"), + "label", "nodes", verLbl, commitLbl, nameLbl, createdAtLbl, "--all", "--overwrite", + fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig"))) + + rr, err := k.c.RunCmd(cmd) + elapsed := time.Since(start) + if elapsed > (1 * time.Second) { + glog.Infof("Completed: %s: (%s)", rr.Command(), elapsed) + } + if err != nil { + return errors.Wrapf(err, "applying node labels") + } + return nil +} From c85fe4955d3872aab3dd4efdb5ba36553380873f Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 20 Feb 2020 13:57:09 -0800 Subject: [PATCH 2/7] Add integration tests for node labels --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 21 +++++------- test/integration/functional_test.go | 34 ++++++++++++++++++++ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index eb296c5ded3a..bb920a7ba7b3 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -491,26 +491,21 @@ func (k *Bootstrapper) applyKicOverlay(cfg config.MachineConfig) error { // applyNodeLabels applies minikube labels to all the nodes func (k *Bootstrapper) applyNodeLabels(cfg config.MachineConfig) error { - start := time.Now() - // based on ISO 8601 (RFC 3339) except converting - and : to _ because of kubernetes label restriction - createdAtLbl := "k8s.minikube.io/updated_at=" + start.Format("2006_01_02T15_04_05_0700") - verLbl := "k8s.minikube.io/version=" + version.GetVersion() - commitLbl := "k8s.minikube.io/commit=" + version.GetGitCommitID() - nameLbl := "k8s.minikube.io/name=" + cfg.Name + // timne cluster was created. time format is based on ISO 8601 (RFC 3339) + // converting - and : to _ because of kubernetes label restriction + createdAtLbl := "minikube.k8s.io/updated_at=" + time.Now().Format("2006_01_02T15_04_05_0700") + verLbl := "minikube.k8s.io/version=" + version.GetVersion() + commitLbl := "minikube.k8s.io/commit=" + version.GetGitCommitID() + nameLbl := "minikube.k8s.io/name=" + cfg.Name // example: - // /var/lib/minikube/binaries/v1.17.3/kubectl label nodes --kubeconfig /var/lib/minikube/kubeconfig k8s.minikube.io/version=1.7.3 --all --overwrite + // sudo /var/lib/minikube/binaries/v1.17.3/kubectl label nodes minikube.k8s.io/version=v1.7.3 minikube.k8s.io/commit=aa91f39ffbcf27dcbb93c4ff3f457c54e585cf4a-dirty minikube.k8s.io/name=p1 minikube.k8s.io/updated_at=2020_02_20T12_05_35_0700 --all --overwrite --kubeconfig=/var/lib/minikube/kubeconfig cmd := exec.Command("sudo", path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl"), "label", "nodes", verLbl, commitLbl, nameLbl, createdAtLbl, "--all", "--overwrite", fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig"))) - rr, err := k.c.RunCmd(cmd) - elapsed := time.Since(start) - if elapsed > (1 * time.Second) { - glog.Infof("Completed: %s: (%s)", rr.Command(), elapsed) - } - if err != nil { + if _, err := k.c.RunCmd(cmd); err != nil { return errors.Wrapf(err, "applying node labels") } return nil diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 1534abffde52..86f95fbf7803 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -112,6 +112,7 @@ func TestFunctional(t *testing.T) { {"FileSync", validateFileSync}, {"UpdateContextCmd", validateUpdateContextCmd}, {"DockerEnv", validateDockerEnv}, + {"NodeLabels", validateNodeLabels}, } for _, tc := range tests { tc := tc @@ -123,6 +124,39 @@ func TestFunctional(t *testing.T) { }) } +// validateNodeLabels checks if minikube cluster is created with correct kubernetes's node label +func validateNodeLabels(ctx context.Context, t *testing.T, profile string) { + mctx, cancel := context.WithTimeout(ctx, 13*time.Second) + defer cancel() + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "nodes", "--output", "jsonpath={.items[0].metadata.labels}")) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + + // json output would look like this + // [beta.kubernetes.io/arch:amd64 beta.kubernetes.io/os:linux minikube.k8s.io/commit:aa91f39ffbcf27dcbb93c4ff3f457c54e585cf4a-dirty minikube.k8s.io/name:p1 minikube.k8s.io/updated_at:2020_02_20T12_05_35_0700 minikube.k8s.io/version:v1.7.3 kubernetes.io/arch:amd64 kubernetes.io/hostname:p1 kubernetes.io/os:linux node-role.kubernetes.io/master:] + + var labels []string + err = json.Unmarshal(rr.Stdout.Bytes(), &labels) + if err != nil { + t.Errorf("%s umarshaling node label from json failed: %v", rr.Args, err) + } + + expectedLabels := []string{"minikube.k8s.io/commit", "minikube.k8s.io/version", "minikube.k8s.io/updated_at", "minikube.k8s.io/name"} + for _, el := range expectedLabels { + found := false + for _, l := range labels { + if strings.Contains(l, el) { + found = true + break + } + } + if !found { + t.Errorf("Failed to have label %q in node labels %+v", expectedLabels, labels) + } + } +} + // check functionality of minikube after evaling docker-env func validateDockerEnv(ctx context.Context, t *testing.T, profile string) { mctx, cancel := context.WithTimeout(ctx, 13*time.Second) From a2be045f1695a5e18fb013e53766af107a06ec49 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 20 Feb 2020 14:01:04 -0800 Subject: [PATCH 3/7] remove debugging --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index bb920a7ba7b3..0f9ac0596bd2 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -231,7 +231,6 @@ func (k *Bootstrapper) StartCluster(cfg config.MachineConfig) error { return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges") } } - fmt.Println("about to apply node labels") if err := k.applyNodeLabels(cfg); err != nil { glog.Warningf("unable to apply node labels: %v", err) } From 7c831eb2de1a714795e15eda0156e87d43abdd0a Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 20 Feb 2020 14:24:37 -0800 Subject: [PATCH 4/7] lint --- test/integration/functional_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 86f95fbf7803..1195e98dd93c 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -126,7 +126,6 @@ func TestFunctional(t *testing.T) { // validateNodeLabels checks if minikube cluster is created with correct kubernetes's node label func validateNodeLabels(ctx context.Context, t *testing.T, profile string) { - mctx, cancel := context.WithTimeout(ctx, 13*time.Second) defer cancel() rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "nodes", "--output", "jsonpath={.items[0].metadata.labels}")) if err != nil { From 4c14c48304a8291520b577ce8cf551c54f62bed6 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 21 Feb 2020 09:46:37 -0800 Subject: [PATCH 5/7] lint --- test/integration/functional_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 1195e98dd93c..a54bcd304f76 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -126,16 +126,14 @@ func TestFunctional(t *testing.T) { // validateNodeLabels checks if minikube cluster is created with correct kubernetes's node label func validateNodeLabels(ctx context.Context, t *testing.T, profile string) { - defer cancel() rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "nodes", "--output", "jsonpath={.items[0].metadata.labels}")) if err != nil { t.Errorf("%s failed: %v", rr.Args, err) } - // json output would look like this - // [beta.kubernetes.io/arch:amd64 beta.kubernetes.io/os:linux minikube.k8s.io/commit:aa91f39ffbcf27dcbb93c4ff3f457c54e585cf4a-dirty minikube.k8s.io/name:p1 minikube.k8s.io/updated_at:2020_02_20T12_05_35_0700 minikube.k8s.io/version:v1.7.3 kubernetes.io/arch:amd64 kubernetes.io/hostname:p1 kubernetes.io/os:linux node-role.kubernetes.io/master:] - var labels []string + // json output: + // [beta.kubernetes.io/arch:amd64 beta.kubernetes.io/os:linux minikube.k8s.io/commit:aa91f39ffbcf27dcbb93c4ff3f457c54e585cf4a-dirty minikube.k8s.io/name:p1 minikube.k8s.io/updated_at:2020_02_20T12_05_35_0700 minikube.k8s.io/version:v1.7.3 kubernetes.io/arch:amd64 kubernetes.io/hostname:p1 kubernetes.io/os:linux node-role.kubernetes.io/master:] err = json.Unmarshal(rr.Stdout.Bytes(), &labels) if err != nil { t.Errorf("%s umarshaling node label from json failed: %v", rr.Args, err) From 6ffba941d33992be431448a48ff1f74db4c0aca7 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 21 Feb 2020 09:55:32 -0800 Subject: [PATCH 6/7] lint --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 0f9ac0596bd2..1a12f50decf3 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -490,7 +490,7 @@ func (k *Bootstrapper) applyKicOverlay(cfg config.MachineConfig) error { // applyNodeLabels applies minikube labels to all the nodes func (k *Bootstrapper) applyNodeLabels(cfg config.MachineConfig) error { - // timne cluster was created. time format is based on ISO 8601 (RFC 3339) + // time cluster was created. time format is based on ISO 8601 (RFC 3339) // converting - and : to _ because of kubernetes label restriction createdAtLbl := "minikube.k8s.io/updated_at=" + time.Now().Format("2006_01_02T15_04_05_0700") verLbl := "minikube.k8s.io/version=" + version.GetVersion() From e6535896b4cbd78f679ea016458ba93abb182b2a Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 21 Feb 2020 16:01:47 -0800 Subject: [PATCH 7/7] integration test for show labels --- test/integration/functional_test.go | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index a54bcd304f76..4469631e2727 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -126,30 +126,14 @@ func TestFunctional(t *testing.T) { // validateNodeLabels checks if minikube cluster is created with correct kubernetes's node label func validateNodeLabels(ctx context.Context, t *testing.T, profile string) { - rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "nodes", "--output", "jsonpath={.items[0].metadata.labels}")) + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "nodes", "--output=go-template", "--template='{{range $k, $v := (index .items 0).metadata.labels}}{{$k}} {{end}}'")) if err != nil { t.Errorf("%s failed: %v", rr.Args, err) } - - var labels []string - // json output: - // [beta.kubernetes.io/arch:amd64 beta.kubernetes.io/os:linux minikube.k8s.io/commit:aa91f39ffbcf27dcbb93c4ff3f457c54e585cf4a-dirty minikube.k8s.io/name:p1 minikube.k8s.io/updated_at:2020_02_20T12_05_35_0700 minikube.k8s.io/version:v1.7.3 kubernetes.io/arch:amd64 kubernetes.io/hostname:p1 kubernetes.io/os:linux node-role.kubernetes.io/master:] - err = json.Unmarshal(rr.Stdout.Bytes(), &labels) - if err != nil { - t.Errorf("%s umarshaling node label from json failed: %v", rr.Args, err) - } - expectedLabels := []string{"minikube.k8s.io/commit", "minikube.k8s.io/version", "minikube.k8s.io/updated_at", "minikube.k8s.io/name"} for _, el := range expectedLabels { - found := false - for _, l := range labels { - if strings.Contains(l, el) { - found = true - break - } - } - if !found { - t.Errorf("Failed to have label %q in node labels %+v", expectedLabels, labels) + if !strings.Contains(rr.Output(), el) { + t.Errorf("expected to have label %q in node labels: %q", expectedLabels, rr.Output()) } } }