From 0aa9eeee586df3bca83057c1fbcbe7d5772dc33c Mon Sep 17 00:00:00 2001 From: Karl Heins Date: Tue, 18 Jan 2022 09:19:33 -0600 Subject: [PATCH] Update metadata during subsequent promote Signed-off-by: Karl Heins Support updating primary Deployment/DaemonSet/HPA/Service labels and annotations after first-time rollout --- pkg/canary/daemonset_controller.go | 99 ++++++++++------- pkg/canary/daemonset_controller_test.go | 8 ++ pkg/canary/daemonset_fixture_test.go | 12 ++- pkg/canary/deployment_controller.go | 130 +++++++++++++++-------- pkg/canary/deployment_controller_test.go | 16 +++ pkg/canary/deployment_fixture_test.go | 23 +++- pkg/canary/service_controller.go | 58 ++++++---- 7 files changed, 235 insertions(+), 111 deletions(-) diff --git a/pkg/canary/daemonset_controller.go b/pkg/canary/daemonset_controller.go index f2f1a1923..3a6b3eb26 100644 --- a/pkg/canary/daemonset_controller.go +++ b/pkg/canary/daemonset_controller.go @@ -27,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned" @@ -119,59 +120,75 @@ func (c *DaemonSetController) Promote(cd *flaggerv1.Canary) error { targetName := cd.Spec.TargetRef.Name primaryName := fmt.Sprintf("%s-primary", targetName) - canary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("damonset %s.%s get query error: %v", targetName, cd.Namespace, err) - } + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + canary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("damonset %s.%s get query error: %v", targetName, cd.Namespace, err) + } - label, labelValue, err := c.getSelectorLabel(canary) - primaryLabelValue := fmt.Sprintf("%s-primary", labelValue) - if err != nil { - return fmt.Errorf("getSelectorLabel failed: %w", err) - } + label, labelValue, err := c.getSelectorLabel(canary) + primaryLabelValue := fmt.Sprintf("%s-primary", labelValue) + if err != nil { + return fmt.Errorf("getSelectorLabel failed: %w", err) + } - primary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("daemonset %s.%s get query error: %w", primaryName, cd.Namespace, err) - } + primary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("daemonset %s.%s get query error: %w", primaryName, cd.Namespace, err) + } - // promote secrets and config maps - configRefs, err := c.configTracker.GetTargetConfigs(cd) - if err != nil { - return fmt.Errorf("GetTargetConfigs failed: %w", err) - } - if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil { - return fmt.Errorf("CreatePrimaryConfigs failed: %w", err) - } + // promote secrets and config maps + configRefs, err := c.configTracker.GetTargetConfigs(cd) + if err != nil { + return fmt.Errorf("GetTargetConfigs failed: %w", err) + } + if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil { + return fmt.Errorf("CreatePrimaryConfigs failed: %w", err) + } - primaryCopy := primary.DeepCopy() - primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds - primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit - primaryCopy.Spec.UpdateStrategy = canary.Spec.UpdateStrategy + primaryCopy := primary.DeepCopy() + primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds + primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit + primaryCopy.Spec.UpdateStrategy = canary.Spec.UpdateStrategy - // update spec with primary secrets and config maps - primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs) + // update spec with primary secrets and config maps + primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs) - // ignore `daemonSetScaleDownNodeSelector` node selector - for key := range daemonSetScaleDownNodeSelector { - delete(primaryCopy.Spec.Template.Spec.NodeSelector, key) - } + // ignore `daemonSetScaleDownNodeSelector` node selector + for key := range daemonSetScaleDownNodeSelector { + delete(primaryCopy.Spec.Template.Spec.NodeSelector, key) + } - // update pod annotations to ensure a rolling update - annotations, err := makeAnnotations(canary.Spec.Template.Annotations) - if err != nil { - return fmt.Errorf("makeAnnotations failed: %w", err) - } + // update pod annotations to ensure a rolling update + annotations, err := makeAnnotations(canary.Spec.Template.Annotations) + if err != nil { + return fmt.Errorf("makeAnnotations failed: %w", err) + } - primaryCopy.Spec.Template.Annotations = annotations - primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label) + primaryCopy.Spec.Template.Annotations = annotations + primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label) - // apply update - _, err = c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{}) + // update deploy annotations + primaryCopy.ObjectMeta.Annotations = make(map[string]string) + for k, v := range canary.ObjectMeta.Annotations { + primaryCopy.ObjectMeta.Annotations[k] = v + } + // update deploy labels + primaryCopy.ObjectMeta.Labels = make(map[string]string) + filteredLabels := includeLabelsByPrefix(canary.ObjectMeta.Labels, c.includeLabelPrefix) + for k, v := range filteredLabels { + primaryCopy.ObjectMeta.Labels[k] = v + } + + // apply update + _, err = c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{}) + return err + }) if err != nil { return fmt.Errorf("updating daemonset %s.%s template spec failed: %w", - primaryCopy.GetName(), primaryCopy.Namespace, err) + primaryName, cd.Namespace, err) } + return nil } diff --git a/pkg/canary/daemonset_controller_test.go b/pkg/canary/daemonset_controller_test.go index f27ce6842..29585bb2a 100644 --- a/pkg/canary/daemonset_controller_test.go +++ b/pkg/canary/daemonset_controller_test.go @@ -96,6 +96,14 @@ func TestDaemonSetController_Promote(t *testing.T) { sourceImage := dae2.Spec.Template.Spec.Containers[0].Image assert.Equal(t, primaryImage, sourceImage) + daePrimaryLabels := daePrimary.ObjectMeta.Labels + daeSourceLabels := dae2.ObjectMeta.Labels + assert.Equal(t, daeSourceLabels["app.kubernetes.io/test-label-1"], daePrimaryLabels["app.kubernetes.io/test-label-1"]) + + daePrimaryAnnotations := daePrimary.ObjectMeta.Annotations + daeSourceAnnotations := dae2.ObjectMeta.Annotations + assert.Equal(t, daeSourceAnnotations["app.kubernetes.io/test-annotation-1"], daePrimaryAnnotations["app.kubernetes.io/test-annotation-1"]) + configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), "podinfo-config-env-primary", metav1.GetOptions{}) if assert.NoError(t, err) { assert.Equal(t, configPrimary.Data["color"], config2.Data["color"]) diff --git a/pkg/canary/daemonset_fixture_test.go b/pkg/canary/daemonset_fixture_test.go index f1a04598d..01c21be81 100644 --- a/pkg/canary/daemonset_fixture_test.go +++ b/pkg/canary/daemonset_fixture_test.go @@ -369,6 +369,12 @@ func newDaemonSetControllerTestPodInfo(dc daemonsetConfigs) *appsv1.DaemonSet { ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: dc.name, + Annotations: map[string]string{ + "test-annotation-1": "test-annotation-value-1", + }, + Labels: map[string]string{ + "test-label-1": "test-label-value-1", + }, }, Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ @@ -379,7 +385,11 @@ func newDaemonSetControllerTestPodInfo(dc daemonsetConfigs) *appsv1.DaemonSet { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - dc.label: dc.labelValue, + dc.label: dc.labelValue, + "test-label-1": "test-label-value-1", + }, + Annotations: map[string]string{ + "test-annotation-1": "test-annotation-value-1", }, }, Spec: corev1.PodSpec{ diff --git a/pkg/canary/deployment_controller.go b/pkg/canary/deployment_controller.go index e6e990f06..695869c7a 100644 --- a/pkg/canary/deployment_controller.go +++ b/pkg/canary/deployment_controller.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned" @@ -84,54 +85,69 @@ func (c *DeploymentController) Promote(cd *flaggerv1.Canary) error { targetName := cd.Spec.TargetRef.Name primaryName := fmt.Sprintf("%s-primary", targetName) - canary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err) - } + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + canary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err) + } - label, labelValue, err := c.getSelectorLabel(canary) - primaryLabelValue := fmt.Sprintf("%s-primary", labelValue) - if err != nil { - return fmt.Errorf("getSelectorLabel failed: %w", err) - } + label, labelValue, err := c.getSelectorLabel(canary) + primaryLabelValue := fmt.Sprintf("%s-primary", labelValue) + if err != nil { + return fmt.Errorf("getSelectorLabel failed: %w", err) + } - primary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("deployment %s.%s get query error: %w", primaryName, cd.Namespace, err) - } + primary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("deployment %s.%s get query error: %w", primaryName, cd.Namespace, err) + } - // promote secrets and config maps - configRefs, err := c.configTracker.GetTargetConfigs(cd) - if err != nil { - return fmt.Errorf("GetTargetConfigs failed: %w", err) - } - if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil { - return fmt.Errorf("CreatePrimaryConfigs failed: %w", err) - } + // promote secrets and config maps + configRefs, err := c.configTracker.GetTargetConfigs(cd) + if err != nil { + return fmt.Errorf("GetTargetConfigs failed: %w", err) + } + if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil { + return fmt.Errorf("CreatePrimaryConfigs failed: %w", err) + } - primaryCopy := primary.DeepCopy() - primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds - primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds - primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit - primaryCopy.Spec.Strategy = canary.Spec.Strategy + primaryCopy := primary.DeepCopy() + primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds + primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds + primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit + primaryCopy.Spec.Strategy = canary.Spec.Strategy - // update spec with primary secrets and config maps - primaryCopy.Spec.Template.Spec = c.getPrimaryDeploymentTemplateSpec(canary, configRefs) + // update spec with primary secrets and config maps + primaryCopy.Spec.Template.Spec = c.getPrimaryDeploymentTemplateSpec(canary, configRefs) - // update pod annotations to ensure a rolling update - annotations, err := makeAnnotations(canary.Spec.Template.Annotations) - if err != nil { - return fmt.Errorf("makeAnnotations failed: %w", err) - } + // update pod annotations to ensure a rolling update + podAnnotations, err := makeAnnotations(canary.Spec.Template.Annotations) + if err != nil { + return fmt.Errorf("makeAnnotations for podAnnotations failed: %w", err) + } + + primaryCopy.Spec.Template.Annotations = podAnnotations + primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label) - primaryCopy.Spec.Template.Annotations = annotations - primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label) + // update deploy annotations + primaryCopy.ObjectMeta.Annotations = make(map[string]string) + for k, v := range canary.ObjectMeta.Annotations { + primaryCopy.ObjectMeta.Annotations[k] = v + } + // update deploy labels + primaryCopy.ObjectMeta.Labels = make(map[string]string) + filteredLabels := includeLabelsByPrefix(canary.ObjectMeta.Labels, c.includeLabelPrefix) + for k, v := range filteredLabels { + primaryCopy.ObjectMeta.Labels[k] = v + } - // apply update - _, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{}) + // apply update + _, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{}) + return err + }) if err != nil { return fmt.Errorf("updating deployment %s.%s template spec failed: %w", - primaryCopy.GetName(), primaryCopy.Namespace, err) + primaryName, cd.Namespace, err) } // update HPA @@ -364,18 +380,40 @@ func (c *DeploymentController) reconcilePrimaryHpa(cd *flaggerv1.Canary, init bo if !init && primaryHpa != nil { diffMetrics := cmp.Diff(hpaSpec.Metrics, primaryHpa.Spec.Metrics) diffBehavior := cmp.Diff(hpaSpec.Behavior, primaryHpa.Spec.Behavior) - if diffMetrics != "" || diffBehavior != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas { + diffLabels := cmp.Diff(hpa.ObjectMeta.Labels, primaryHpa.ObjectMeta.Labels) + diffAnnotations := cmp.Diff(hpa.ObjectMeta.Annotations, primaryHpa.ObjectMeta.Annotations) + if diffMetrics != "" || diffBehavior != "" || diffLabels != "" || diffAnnotations != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas { fmt.Println(diffMetrics, diffBehavior, hpaSpec.MinReplicas, primaryHpa.Spec.MinReplicas, hpaSpec.MaxReplicas, primaryHpa.Spec.MaxReplicas) - hpaClone := primaryHpa.DeepCopy() - hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas - hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas - hpaClone.Spec.Metrics = hpaSpec.Metrics - hpaClone.Spec.Behavior = hpaSpec.Behavior - _, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Update(context.TODO(), hpaClone, metav1.UpdateOptions{}) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + primaryHpa, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), primaryHpaName, metav1.GetOptions{}) + if err != nil { + return err + } + hpaClone := primaryHpa.DeepCopy() + hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas + hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas + hpaClone.Spec.Metrics = hpaSpec.Metrics + hpaClone.Spec.Behavior = hpaSpec.Behavior + + // update hpa annotations + hpaClone.ObjectMeta.Annotations = make(map[string]string) + for k, v := range hpa.ObjectMeta.Annotations { + hpaClone.ObjectMeta.Annotations[k] = v + } + // update hpa labels + hpaClone.ObjectMeta.Labels = make(map[string]string) + filteredLabels := includeLabelsByPrefix(hpa.ObjectMeta.Labels, c.includeLabelPrefix) + for k, v := range filteredLabels { + hpaClone.ObjectMeta.Labels[k] = v + } + + _, err = c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Update(context.TODO(), hpaClone, metav1.UpdateOptions{}) + return err + }) if err != nil { return fmt.Errorf("updating HorizontalPodAutoscaler %s.%s failed: %w", - hpaClone.Name, hpaClone.Namespace, err) + primaryHpa.Name, primaryHpa.Namespace, err) } c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)). Infof("HorizontalPodAutoscaler %s.%s updated", primaryHpa.GetName(), cd.Namespace) diff --git a/pkg/canary/deployment_controller_test.go b/pkg/canary/deployment_controller_test.go index f8af8cf20..0191b140d 100644 --- a/pkg/canary/deployment_controller_test.go +++ b/pkg/canary/deployment_controller_test.go @@ -109,6 +109,14 @@ func TestDeploymentController_Promote(t *testing.T) { sourceImage := dep2.Spec.Template.Spec.Containers[0].Image assert.Equal(t, sourceImage, primaryImage) + depPrimaryLabels := depPrimary.ObjectMeta.Labels + depSourceLabels := dep2.ObjectMeta.Labels + assert.Equal(t, depSourceLabels["app.kubernetes.io/test-label-1"], depPrimaryLabels["app.kubernetes.io/test-label-1"]) + + depPrimaryAnnotations := depPrimary.ObjectMeta.Annotations + depSourceAnnotations := dep2.ObjectMeta.Annotations + assert.Equal(t, depSourceAnnotations["app.kubernetes.io/test-annotation-1"], depPrimaryAnnotations["app.kubernetes.io/test-annotation-1"]) + configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), "podinfo-config-env-primary", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, config2.Data["color"], configPrimary.Data["color"]) @@ -117,6 +125,14 @@ func TestDeploymentController_Promote(t *testing.T) { require.NoError(t, err) assert.Equal(t, int32(2), hpaPrimary.Spec.MaxReplicas) + hpaPrimaryLabels := hpaPrimary.ObjectMeta.Labels + hpaSourceLabels := hpa.ObjectMeta.Labels + assert.Equal(t, hpaSourceLabels["app.kubernetes.io/test-label-1"], hpaPrimaryLabels["app.kubernetes.io/test-label-1"]) + + hpaPrimaryAnnotations := hpaPrimary.ObjectMeta.Annotations + hpaSourceAnnotations := hpa.ObjectMeta.Annotations + assert.Equal(t, hpaSourceAnnotations["app.kubernetes.io/test-annotation-1"], hpaPrimaryAnnotations["app.kubernetes.io/test-annotation-1"]) + value := depPrimary.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.LabelSelector.MatchExpressions[0].Values[0] assert.Equal(t, "podinfo-primary", value) diff --git a/pkg/canary/deployment_fixture_test.go b/pkg/canary/deployment_fixture_test.go index 718b76d34..ebe3951d0 100644 --- a/pkg/canary/deployment_fixture_test.go +++ b/pkg/canary/deployment_fixture_test.go @@ -122,6 +122,7 @@ func newCustomizableFixture(dc deploymentConfigs) (deploymentControllerFixture, KubeClient: kubeClient, FlaggerClient: flaggerClient, }, + includeLabelPrefix: []string{"app.kubernetes.io"}, } return deploymentControllerFixture{ @@ -421,6 +422,10 @@ func newDeploymentControllerTest(dc deploymentConfigs) *appsv1.Deployment { Name: dc.name, Annotations: map[string]string{ "kustomize.toolkit.fluxcd.io/checksum": "0a40893bfdc545d62125bd3e74eeb2ebaa7097c2", + "test-annotation-1": "test-annotation-value-1", + }, + Labels: map[string]string{ + "test-label-1": "test-label-value-1", }, }, Spec: appsv1.DeploymentSpec{ @@ -431,8 +436,12 @@ func newDeploymentControllerTest(dc deploymentConfigs) *appsv1.Deployment { }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "test-annotation-1": "test-annotation-value-1", + }, Labels: map[string]string{ - dc.label: dc.labelValue, + dc.label: dc.labelValue, + "test-label-1": "test-label-value-1", }, }, Spec: corev1.PodSpec{ @@ -757,6 +766,12 @@ func newDeploymentControllerTestV2() *appsv1.Deployment { ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "podinfo", + Annotations: map[string]string{ + "test-annotation-1": "test-annotation-value-1", + }, + Labels: map[string]string{ + "test-label-1": "test-label-value-1", + }, }, Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -767,7 +782,11 @@ func newDeploymentControllerTestV2() *appsv1.Deployment { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "name": "podinfo", + "name": "podinfo", + "test-label-1": "test-label-value-1", + }, + Annotations: map[string]string{ + "test-annotation-1": "test-annotation-value-1", }, }, Spec: corev1.PodSpec{ diff --git a/pkg/canary/service_controller.go b/pkg/canary/service_controller.go index 2f53edcab..5a2dc330a 100644 --- a/pkg/canary/service_controller.go +++ b/pkg/canary/service_controller.go @@ -26,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned" @@ -171,29 +172,44 @@ func (c *ServiceController) Promote(cd *flaggerv1.Canary) error { targetName := cd.Spec.TargetRef.Name primaryName := fmt.Sprintf("%s-primary", targetName) - canary, err := c.kubeClient.CoreV1().Services(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("service %s.%s get query error: %w", targetName, cd.Namespace, err) - } - - primary, err := c.kubeClient.CoreV1().Services(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("service %s.%s get query error: %w", primaryName, cd.Namespace, err) - } - - primaryCopy := canary.DeepCopy() - primaryCopy.ObjectMeta.Name = primary.ObjectMeta.Name - if primaryCopy.Spec.Type == "ClusterIP" { - primaryCopy.Spec.ClusterIP = primary.Spec.ClusterIP - } - primaryCopy.ObjectMeta.ResourceVersion = primary.ObjectMeta.ResourceVersion - primaryCopy.ObjectMeta.UID = primary.ObjectMeta.UID - - // apply update - _, err = c.kubeClient.CoreV1().Services(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{}) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + canary, err := c.kubeClient.CoreV1().Services(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("service %s.%s get query error: %w", targetName, cd.Namespace, err) + } + + primary, err := c.kubeClient.CoreV1().Services(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("service %s.%s get query error: %w", primaryName, cd.Namespace, err) + } + + primaryCopy := canary.DeepCopy() + primaryCopy.ObjectMeta.Name = primary.ObjectMeta.Name + if primaryCopy.Spec.Type == "ClusterIP" { + primaryCopy.Spec.ClusterIP = primary.Spec.ClusterIP + } + primaryCopy.ObjectMeta.ResourceVersion = primary.ObjectMeta.ResourceVersion + primaryCopy.ObjectMeta.UID = primary.ObjectMeta.UID + + // update service annotations + primaryCopy.ObjectMeta.Annotations = make(map[string]string) + for k, v := range canary.ObjectMeta.Annotations { + primaryCopy.ObjectMeta.Annotations[k] = v + } + // update service labels + primaryCopy.ObjectMeta.Labels = make(map[string]string) + filteredLabels := includeLabelsByPrefix(canary.ObjectMeta.Labels, []string{"*"}) + for k, v := range filteredLabels { + primaryCopy.ObjectMeta.Labels[k] = v + } + + // apply update + _, err = c.kubeClient.CoreV1().Services(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{}) + return err + }) if err != nil { return fmt.Errorf("updating service %s.%s spec failed: %w", - primaryCopy.GetName(), primaryCopy.Namespace, err) + primaryName, cd.Namespace, err) } return nil