Merge pull request #586 from weaveworks/retry-initializing-status

Retry canary initialization on conflict

pkg/controller/controller.go

@@ -1,7 +1,6 @@
 package controller
 
 import (
-	"context"
 	"fmt"
 	"sync"
 	"time"
@@ -10,7 +9,6 @@ import (
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
@@ -265,16 +263,9 @@ func (c *Controller) syncHandler(key string) error {
 
 	// set status condition for new canaries
 	if cd.Status.Conditions == nil {
-		if ok, conditions := canary.MakeStatusConditions(cd, flaggerv1.CanaryPhaseInitializing); ok {
-			cdCopy := cd.DeepCopy()
-			cdCopy.Status.Conditions = conditions
-			cdCopy.Status.LastTransitionTime = metav1.Now()
-			cdCopy.Status.Phase = flaggerv1.CanaryPhaseInitializing
-			_, err := c.flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).UpdateStatus(context.TODO(), cdCopy, metav1.UpdateOptions{})
-			if err != nil {
-				c.logger.Errorf("%s status condition update error: %v", key, err)
-				return fmt.Errorf("%s status condition update error: %w", key, err)
-			}
+		if err := c.setPhaseInitializing(cd); err != nil {
+			c.logger.Errorf("%s unable to set initializing status: %v", key, err)
+			return fmt.Errorf("%s initializing error: %w", key, err)
 		}
 	}
 

pkg/controller/finalizer.go

@@ -44,7 +44,7 @@ func (c *Controller) finalize(old interface{}) error {
 	c.logger.Infof("%s.%s kind %s reverted", canary.Name, canary.Namespace, canary.Spec.TargetRef.Kind)
 
 	// Ensure that targetRef has met a ready state
-	c.logger.Infof("Checking is canary is ready %s.%s", canary.Name, canary.Namespace)
+	c.logger.Infof("Checking if canary is ready %s.%s", canary.Name, canary.Namespace)
 	_, err = canaryController.IsCanaryReady(canary)
 	if err != nil {
 		return fmt.Errorf("canary not ready during finalizing: %w", err)

pkg/controller/scheduler.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
 
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
 	"github.com/weaveworks/flagger/pkg/canary"
@@ -738,3 +739,32 @@ func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary.
 	c.recorder.SetStatus(canary, flaggerv1.CanaryPhaseFailed)
 	c.runPostRolloutHooks(canary, flaggerv1.CanaryPhaseFailed)
 }
+
+func (c *Controller) setPhaseInitializing(cd *flaggerv1.Canary) error {
+	phase := flaggerv1.CanaryPhaseInitializing
+	firstTry := true
+	name, ns := cd.GetName(), cd.GetNamespace()
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		if !firstTry {
+			cd, err = c.flaggerClient.FlaggerV1beta1().Canaries(ns).Get(context.TODO(), name, metav1.GetOptions{})
+			if err != nil {
+				return fmt.Errorf("canary %s.%s get query failed: %w", name, ns, err)
+			}
+		}
+
+		if ok, conditions := canary.MakeStatusConditions(cd, phase); ok {
+			cdCopy := cd.DeepCopy()
+			cdCopy.Status.Conditions = conditions
+			cdCopy.Status.LastTransitionTime = metav1.Now()
+			cdCopy.Status.Phase = phase
+			_, err = c.flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).UpdateStatus(context.TODO(), cdCopy, metav1.UpdateOptions{})
+		}
+		firstTry = false
+		return
+	})
+
+	if err != nil {
+		return fmt.Errorf("failed after retries: %w", err)
+	}
+	return nil
+}

pkg/router/kubernetes_default.go

@@ -184,7 +184,6 @@ func (c *KubernetesDefaultRouter) reconcileService(canary *flaggerv1.Canary, nam
 		if portsDiff != "" || selectorsDiff != "" {
 			svcClone.Spec.Ports = svcSpec.Ports
 			svcClone.Spec.Selector = svcSpec.Selector
-			_, err = c.kubeClient.CoreV1().Services(canary.Namespace).Update(context.TODO(), svcClone, metav1.UpdateOptions{})
 			updateService = true
 		}