From e63244a13612e1553d78319f0e47e2328e49ec9f Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Fri, 3 Feb 2023 11:18:16 -0600 Subject: [PATCH] Use capacity annotations to set labels and taints for cluster api node groups --- .../cloudprovider/clusterapi/README.md | 20 ++++- .../clusterapi/clusterapi_unstructured.go | 73 +++++++++++++++++-- .../clusterapi_unstructured_test.go | 14 ++++ .../clusterapi/clusterapi_utils.go | 2 + 4 files changed, 100 insertions(+), 9 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/README.md b/cluster-autoscaler/cloudprovider/clusterapi/README.md index d2e4cce5cd83..6bbfa1914612 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/README.md +++ b/cluster-autoscaler/cloudprovider/clusterapi/README.md @@ -255,10 +255,22 @@ rules: #### Pre-defined labels and taints on nodes scaled from zero -The Cluster API provider currently does not support the addition of pre-defined -labels and taints for node groups that are scaling from zero. This work is on-going -and will be included in a future release once the API for specifying those -labels and taints has been accepted by the community. +To provide labels or taint information for scale from zero, the optional +capacity annotations may be supplied as a comma separated list, as +demonstrated in the example below: + +```yaml +apiVersion: cluster.x-k8s.io/v1alpha4 +kind: MachineDeployment +metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0" + capacity.cluster-autoscaler.kubernetes.io/memory: "128G" + capacity.cluster-autoscaler.kubernetes.io/cpu: "16" + capacity.cluster-autoscaler.kubernetes.io/labels: "key1=value1,key2=value2" + capacity.cluster-autoscaler.kubernetes.io/taints: "key1=value1:NoSchedule,key2=value2:NoExecute" +``` ## Specifying a Custom Resource Group diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go index 7e6d75b78a18..81fccf43e28d 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured.go @@ -30,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation" klog "k8s.io/klog/v2" ) @@ -169,15 +170,36 @@ func (r unstructuredScalableResource) MarkMachineForDeletion(machine *unstructur } func (r unstructuredScalableResource) Labels() map[string]string { - // TODO implement this once the community has decided how they will handle labels - // this issue is related, https://github.com/kubernetes-sigs/cluster-api/issues/7006 - + annotations := r.unstructured.GetAnnotations() + // annotation value of the form "key1=value1,key2=value2" + if val, found := annotations[labelsKey]; found { + labels := strings.Split(val, ",") + kv := make(map[string]string, len(labels)) + for _, label := range labels { + split := strings.SplitN(label, "=", 2) + if len(split) == 2 { + kv[split[0]] = split[1] + } + } + return kv + } return nil } func (r unstructuredScalableResource) Taints() []apiv1.Taint { - // TODO implement this once the community has decided how they will handle taints - + annotations := r.unstructured.GetAnnotations() + // annotation value the form of "key1=value1:condition,key2=value2:condition" + if val, found := annotations[taintsKey]; found { + taints := strings.Split(val, ",") + ret := make([]apiv1.Taint, 0, len(taints)) + for _, taintStr := range taints { + taint, err := parseTaint(taintStr) + if err == nil { + ret = append(ret, taint) + } + } + return ret + } return nil } @@ -359,3 +381,44 @@ func resourceCapacityFromInfrastructureObject(infraobj *unstructured.Unstructure return capacity } + +// adapted from https://github.com/kubernetes/kubernetes/blob/release-1.25/pkg/util/taints/taints.go#L39 +func parseTaint(st string) (apiv1.Taint, error) { + var taint apiv1.Taint + + var key string + var value string + var effect apiv1.TaintEffect + + parts := strings.Split(st, ":") + switch len(parts) { + case 1: + key = parts[0] + case 2: + effect = apiv1.TaintEffect(parts[1]) + + partsKV := strings.Split(parts[0], "=") + if len(partsKV) > 2 { + return taint, fmt.Errorf("invalid taint spec: %v", st) + } + key = partsKV[0] + if len(partsKV) == 2 { + value = partsKV[1] + if errs := validation.IsValidLabelValue(value); len(errs) > 0 { + return taint, fmt.Errorf("invalid taint spec: %v, %s", st, strings.Join(errs, "; ")) + } + } + default: + return taint, fmt.Errorf("invalid taint spec: %v", st) + } + + if errs := validation.IsQualifiedName(key); len(errs) > 0 { + return taint, fmt.Errorf("invalid taint spec: %v, %s", st, strings.Join(errs, "; ")) + } + + taint.Key = key + taint.Value = value + taint.Effect = effect + + return taint, nil +} diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go index 437f32a0efb8..c80524c8010f 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -287,12 +289,15 @@ func TestAnnotations(t *testing.T) { diskQuantity := resource.MustParse("100Gi") gpuQuantity := resource.MustParse("1") maxPodsQuantity := resource.MustParse("42") + expectedTaints := []v1.Taint{{Key: "key1", Effect: v1.TaintEffectNoSchedule, Value: "value1"}, {Key: "key2", Effect: v1.TaintEffectNoExecute, Value: "value2"}} annotations := map[string]string{ cpuKey: cpuQuantity.String(), memoryKey: memQuantity.String(), diskCapacityKey: diskQuantity.String(), gpuCountKey: gpuQuantity.String(), maxPodsKey: maxPodsQuantity.String(), + taintsKey: "key1=value1:NoSchedule,key2=value2:NoExecute", + labelsKey: "key3=value3,key4=value4,key5=value5", } test := func(t *testing.T, testConfig *testConfig, testResource *unstructured.Unstructured) { @@ -333,6 +338,15 @@ func TestAnnotations(t *testing.T) { } else if maxPodsQuantity.Cmp(maxPods) != 0 { t.Errorf("expected %v, got %v", maxPodsQuantity, maxPods) } + + taints := sr.Taints() + assert.Equal(t, expectedTaints, taints) + + labels := sr.Labels() + assert.Len(t, labels, 3) + assert.Equal(t, "value3", labels["key3"]) + assert.Equal(t, "value4", labels["key4"]) + assert.Equal(t, "value5", labels["key5"]) } t.Run("MachineSet", func(t *testing.T) { diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go index 72a0e611fb18..d10c8ce21ed5 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go @@ -34,6 +34,8 @@ const ( gpuTypeKey = "capacity.cluster-autoscaler.kubernetes.io/gpu-type" gpuCountKey = "capacity.cluster-autoscaler.kubernetes.io/gpu-count" maxPodsKey = "capacity.cluster-autoscaler.kubernetes.io/maxPods" + taintsKey = "capacity.cluster-autoscaler.kubernetes.io/taints" + labelsKey = "capacity.cluster-autoscaler.kubernetes.io/labels" ) var (