Replace MachineAnnotations with NodeAnnotations and add MachineObject…

…Annotations (#2077) * Replace MachineAnnotations with NodeAnnotations and introduce MachineObjectAnnotations Signed-off-by: Marko Mudrinić <mudrinic.mare@gmail.com> * Update example Terraform configs with NodeAnnotations and MachineObjectAnnotations Signed-off-by: Marko Mudrinić <mudrinic.mare@gmail.com> * Update API docs Signed-off-by: Marko Mudrinić <mudrinic.mare@gmail.com>
kubermatic · Jun 1, 2022 · 829cbb6 · 829cbb6
1 parent e96db99
commit 829cbb6
Show file tree

Hide file tree

Showing 14 changed files with 194 additions and 26 deletions.
diff --git a/docs/api_reference/v1beta2.en.md b/docs/api_reference/v1beta2.en.md
@@ -1,6 +1,6 @@
 +++
 title = "v1beta2 API Reference"
-date = 2022-02-04T17:38:06+02:00
+date = 2022-06-01T15:57:00+02:00
 weight = 11
 +++
 ## v1beta2
@@ -618,7 +618,9 @@ ProviderSpec describes a worker node
 | ----- | ----------- | ------ | -------- |
 | cloudProviderSpec | CloudProviderSpec | [json.RawMessage](https://golang.org/pkg/encoding/json/#RawMessage) | true |
 | annotations | Annotations set MachineDeployment.ObjectMeta.Annotations | map[string]string | false |
-| machineAnnotations | MachineAnnotations set MachineDeployment.Spec.Template.Spec.ObjectMeta.Annotations a way to annotate resulted Nodes | map[string]string | false |
+| machineAnnotations | MachineAnnotations set MachineDeployment.Spec.Template.Spec.ObjectMeta.Annotations as a way to annotate resulting Nodes Deprecated: Use NodeAnnotations instead. | map[string]string | false |
+| nodeAnnotations | NodeAnnotations set MachineDeployment.Spec.Template.Spec.ObjectMeta.Annotations as a way to annotate resulting Nodes | map[string]string | false |
+| machineObjectAnnotations | MachineObjectAnnotations set MachineDeployment.Spec.Template.Metadata.Annotations as a way to annotate resulting Machine objects. Those annotations are not propagated to Node objects. If you want to annotate resulting Nodes as well, see NodeAnnotations | map[string]string | false |
 | labels | Labels | map[string]string | false |
 | taints | Taints | [][corev1.Taint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#taint-v1-core) | false |
 | sshPublicKeys | SSHPublicKeys | []string | false |

diff --git a/examples/terraform/aws/output.tf b/examples/terraform/aws/output.tf
@@ -88,12 +88,18 @@ output "kubeone_workers" {
         labels = {
           isSpotInstance = format("%t", var.initial_machinedeployment_spotinstances)
         }
+        # nodeAnnotations are applied on resulting Node objects
+        # nodeAnnotations = {
+        #   "key" = "value"
+        # }
+        # machineObjectAnnotations are applied on resulting Machine objects
         # uncomment to following to set those kubelet parameters. More into at:
         # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
-        # machineAnnotations = {
-        #  "v1.kubelet-config.machine-controller.kubermatic.io/SystemReserved" = "cpu=200m,memory=200Mi"
-        #  "v1.kubelet-config.machine-controller.kubermatic.io/KubeReserved"   = "cpu=200m,memory=300Mi"
-        #  "v1.kubelet-config.machine-controller.kubermatic.io/EvictionHard"   = ""
+        # machineObjectAnnotations = {
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/SystemReserved" = "cpu=200m,memory=200Mi"
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/KubeReserved"   = "cpu=200m,memory=300Mi"
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/EvictionHard"   = ""
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/MaxPods"        = "110"
         # }
         cloudProviderSpec = {
           # provider specific fields:
@@ -136,6 +142,19 @@ output "kubeone_workers" {
         labels = {
           isSpotInstance = format("%t", var.initial_machinedeployment_spotinstances)
         }
+        # nodeAnnotations are applied on resulting Node objects
+        # nodeAnnotations = {
+        #   "key" = "value"
+        # }
+        # machineObjectAnnotations are applied on resulting Machine objects
+        # uncomment to following to set those kubelet parameters. More into at:
+        # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
+        # machineObjectAnnotations = {
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/SystemReserved" = "cpu=200m,memory=200Mi"
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/KubeReserved"   = "cpu=200m,memory=300Mi"
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/EvictionHard"   = ""
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/MaxPods"        = "110"
+        # }
         cloudProviderSpec = {
           # provider specific fields:
           # see example under `cloudProviderSpec` section at:
@@ -177,6 +196,19 @@ output "kubeone_workers" {
         labels = {
           isSpotInstance = format("%t", var.initial_machinedeployment_spotinstances)
         }
+        # nodeAnnotations are applied on resulting Node objects
+        # nodeAnnotations = {
+        #   "key" = "value"
+        # }
+        # machineObjectAnnotations are applied on resulting Machine objects
+        # uncomment to following to set those kubelet parameters. More into at:
+        # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
+        # machineObjectAnnotations = {
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/SystemReserved" = "cpu=200m,memory=200Mi"
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/KubeReserved"   = "cpu=200m,memory=300Mi"
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/EvictionHard"   = ""
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/MaxPods"        = "110"
+        # }
         cloudProviderSpec = {
           # provider specific fields:
           # see example under `cloudProviderSpec` section at:

diff --git a/examples/terraform/nutanix/output.tf b/examples/terraform/nutanix/output.tf
@@ -60,12 +60,18 @@ output "kubeone_workers" {
         operatingSystemSpec = {
           distUpgradeOnBoot   = false
         }
+        # nodeAnnotations are applied on resulting Node objects
+        # nodeAnnotations = {
+        #   "key" = "value"
+        # }
+        # machineObjectAnnotations are applied on resulting Machine objects
         # uncomment to following to set those kubelet parameters. More into at:
         # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
-        # machineAnnotations = {
-        #  "v1.kubelet-config.machine-controller.kubermatic.io/SystemReserved" = "cpu=200m,memory=200Mi"
-        #  "v1.kubelet-config.machine-controller.kubermatic.io/KubeReserved"   = "cpu=200m,memory=300Mi"
-        #  "v1.kubelet-config.machine-controller.kubermatic.io/EvictionHard"   = ""
+        # machineObjectAnnotations = {
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/SystemReserved" = "cpu=200m,memory=200Mi"
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/KubeReserved"   = "cpu=200m,memory=300Mi"
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/EvictionHard"   = ""
+        #   "v1.kubelet-config.machine-controller.kubermatic.io/MaxPods"        = "110"
         # }
         cloudProviderSpec = {
           # provider specific fields:

diff --git a/pkg/apis/kubeone/config/config.go b/pkg/apis/kubeone/config/config.go
@@ -256,6 +256,19 @@ func SetKubeOneClusterDynamicDefaults(cluster *kubeoneapi.KubeOneCluster, creden
 	// Default the AssetsConfiguration internal API
 	cluster.DefaultAssetConfiguration()
 
+	// Copy MachineAnnotations to NodeAnnotations.
+	// MachineAnnotations has been deprecated in favor of NodeAnnotations.
+	// This is supposed to handle renaming of MachineAnnotations to
+	// NodeAnnotations in non backwards-compatibility breaking way.
+	for i, workerset := range cluster.DynamicWorkers {
+		// NB: We don't want to allow both MachineAnnotations and NodeAnnotations
+		// to be set, so we explicitly handle this scenario here and in validation.
+		if len(workerset.Config.MachineAnnotations) > 0 && len(workerset.Config.NodeAnnotations) == 0 {
+			cluster.DynamicWorkers[i].Config.NodeAnnotations = cluster.DynamicWorkers[i].Config.MachineAnnotations
+			cluster.DynamicWorkers[i].Config.MachineAnnotations = nil
+		}
+	}
+
 	return nil
 }
 

diff --git a/pkg/apis/kubeone/types.go b/pkg/apis/kubeone/types.go
@@ -433,8 +433,17 @@ type ProviderSpec struct {
 	// Annotations set MachineDeployment.ObjectMeta.Annotations
 	Annotations map[string]string `json:"annotations,omitempty"`
 	// MachineAnnotations set MachineDeployment.Spec.Template.Spec.ObjectMeta.Annotations
-	// a way to annotate resulted Nodes
+	// as a way to annotate resulting Nodes
+	// Deprecated: Use NodeAnnotations instead.
 	MachineAnnotations map[string]string `json:"machineAnnotations,omitempty"`
+	// NodeAnnotations set MachineDeployment.Spec.Template.Spec.ObjectMeta.Annotations
+	// as a way to annotate resulting Nodes
+	NodeAnnotations map[string]string `json:"nodeAnnotations,omitempty"`
+	// MachineObjectAnnotations set MachineDeployment.Spec.Template.Metadata.Annotations
+	// as a way to annotate resulting Machine objects. Those annotations are not
+	// propagated to Node objects. If you want to annotate resulting Nodes as well,
+	// see NodeAnnotations
+	MachineObjectAnnotations map[string]string `json:"machineObjectAnnotations,omitempty"`
 	// Labels
 	Labels map[string]string `json:"labels,omitempty"`
 	// Taints

diff --git a/pkg/apis/kubeone/v1beta1/conversion.go b/pkg/apis/kubeone/v1beta1/conversion.go
@@ -76,3 +76,8 @@ func Convert_kubeone_KubeOneCluster_To_v1beta1_KubeOneCluster(in *kubeoneapi.Kub
 	// LoggingConfig was introduced only in new v1beta2 API, so we skip it here
 	return autoConvert_kubeone_KubeOneCluster_To_v1beta1_KubeOneCluster(in, out, s)
 }
+
+func Convert_kubeone_ProviderSpec_To_v1beta1_ProviderSpec(in *kubeoneapi.ProviderSpec, out *ProviderSpec, s conversion.Scope) error {
+	// NodeAnnotations and MachineObjectAnnotations were introduced only in new v1beta2 API, so we skip them here
+	return autoConvert_kubeone_ProviderSpec_To_v1beta1_ProviderSpec(in, out, s)
+}
diff --git a/pkg/apis/kubeone/v1beta1/zz_generated.conversion.go b/pkg/apis/kubeone/v1beta1/zz_generated.conversion.go
diff --git a/pkg/apis/kubeone/v1beta2/types.go b/pkg/apis/kubeone/v1beta2/types.go
@@ -431,8 +431,17 @@ type ProviderSpec struct {
 	// Annotations set MachineDeployment.ObjectMeta.Annotations
 	Annotations map[string]string `json:"annotations,omitempty"`
 	// MachineAnnotations set MachineDeployment.Spec.Template.Spec.ObjectMeta.Annotations
-	// a way to annotate resulted Nodes
+	// as a way to annotate resulting Nodes
+	// Deprecated: Use NodeAnnotations instead.
 	MachineAnnotations map[string]string `json:"machineAnnotations,omitempty"`
+	// NodeAnnotations set MachineDeployment.Spec.Template.Spec.ObjectMeta.Annotations
+	// as a way to annotate resulting Nodes
+	NodeAnnotations map[string]string `json:"nodeAnnotations,omitempty"`
+	// MachineObjectAnnotations set MachineDeployment.Spec.Template.Metadata.Annotations
+	// as a way to annotate resulting Machine objects. Those annotations are not
+	// propagated to Node objects. If you want to annotate resulting Nodes as well,
+	// see NodeAnnotations
+	MachineObjectAnnotations map[string]string `json:"machineObjectAnnotations,omitempty"`
 	// Labels
 	Labels map[string]string `json:"labels,omitempty"`
 	// Taints

diff --git a/pkg/apis/kubeone/v1beta2/zz_generated.conversion.go b/pkg/apis/kubeone/v1beta2/zz_generated.conversion.go
diff --git a/pkg/apis/kubeone/v1beta2/zz_generated.deepcopy.go b/pkg/apis/kubeone/v1beta2/zz_generated.deepcopy.go
diff --git a/pkg/apis/kubeone/validation/validation.go b/pkg/apis/kubeone/validation/validation.go
@@ -434,6 +434,9 @@ func ValidateDynamicWorkerConfig(workerset []kubeoneapi.DynamicWorkerConfig, fld
 		if w.Replicas == nil || *w.Replicas < 0 {
 			allErrs = append(allErrs, field.Invalid(fldPath.Child("replicas"), w.Replicas, ".dynamicWorkers.replicas must be specified and >= 0"))
 		}
+		if len(w.Config.MachineAnnotations) > 0 && len(w.Config.NodeAnnotations) > 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("machineAnnotations"), w.Config.MachineAnnotations, "machineAnnotations has been replaced with nodeAnnotations, only one of those two can be set"))
+		}
 	}
 
 	return allErrs

diff --git a/pkg/apis/kubeone/validation/validation_test.go b/pkg/apis/kubeone/validation/validation_test.go
@@ -1211,6 +1211,46 @@ func TestValidateDynamicWorkerConfig(t *testing.T) {
 			},
 			expectedError: true,
 		},
+		{
+			name: "only machineAnnotations set",
+			dynamicWorkerConfig: []kubeoneapi.DynamicWorkerConfig{
+				{
+					Name:     "test-1",
+					Replicas: intPtr(3),
+					Config: kubeoneapi.ProviderSpec{
+						MachineAnnotations: map[string]string{"test": "test"},
+					},
+				},
+			},
+			expectedError: false,
+		},
+		{
+			name: "only nodeAnnotations set",
+			dynamicWorkerConfig: []kubeoneapi.DynamicWorkerConfig{
+				{
+					Name:     "test-1",
+					Replicas: intPtr(3),
+					Config: kubeoneapi.ProviderSpec{
+						NodeAnnotations: map[string]string{"test": "test"},
+					},
+				},
+			},
+			expectedError: false,
+		},
+		{
+			name: "both machineAnnotations and nodeAnnotations set",
+			dynamicWorkerConfig: []kubeoneapi.DynamicWorkerConfig{
+				{
+					Name:     "test-1",
+					Replicas: intPtr(3),
+					Config: kubeoneapi.ProviderSpec{
+						MachineAnnotations: map[string]string{"test": "test"},
+						NodeAnnotations:    map[string]string{"test": "test"},
+					},
+				},
+			},
+			expectedError: true,
+		},
 	}
 
 	for _, tc := range tests {

diff --git a/pkg/apis/kubeone/zz_generated.deepcopy.go b/pkg/apis/kubeone/zz_generated.deepcopy.go
diff --git a/pkg/templates/machinecontroller/machines.go b/pkg/templates/machinecontroller/machines.go
@@ -152,11 +152,11 @@ func createMachineDeployment(cluster *kubeoneapi.KubeOneCluster, workerset kubeo
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      labels.Merge(workerset.Config.Labels, workersetNameLabels),
 					Namespace:   metav1.NamespaceSystem,
-					Annotations: machineAnnotations,
+					Annotations: labels.Merge(workerset.Config.MachineObjectAnnotations, machineAnnotations),
 				},
 				Spec: clusterv1alpha1.MachineSpec{
 					ObjectMeta: metav1.ObjectMeta{
-						Annotations: workerset.Config.MachineAnnotations,
+						Annotations: workerset.Config.NodeAnnotations,
 						Labels:      labels.Merge(workerset.Config.Labels, workersetNameLabels),
 					},
 					Versions: clusterv1alpha1.MachineVersionInfo{