fix: auto-sync fails with 'another operation is already in progress' …

…error (#15638)

Signed-off-by: Alexander Matyushentsev <AMatyushentsev@gmail.com>

controller/appcontroller.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	"encoding/json"
+	goerrors "errors"
 	"fmt"
 	"math"
 	"net/http"
@@ -1801,6 +1802,13 @@ func (ctrl *ApplicationController) autoSync(app *appv1.Application, syncStatus *
 	_, err := argo.SetAppOperation(appIf, app.Name, &op)
 	setOpTime := time.Since(start)
 	if err != nil {
+		if goerrors.Is(err, argo.ErrAnotherOperationInProgress) {
+			// skipping auto-sync because another operation is in progress and was not noticed due to stale data in informer
+			// it is safe to skip auto-sync because it is already running
+			logCtx.Warnf("Failed to initiate auto-sync to %s: %v", desiredCommitSHA, err)
+			return nil, 0
+		}
+
 		logCtx.Errorf("Failed to initiate auto-sync to %s: %v", desiredCommitSHA, err)
 		return &appv1.ApplicationCondition{Type: appv1.ApplicationConditionSyncError, Message: err.Error()}, setOpTime
 	}

util/argo/argo.go

@@ -35,6 +35,10 @@ const (
 	errDestinationMissing = "Destination server missing from app spec"
 )
 
+var (
+	ErrAnotherOperationInProgress = status.Errorf(codes.FailedPrecondition, "another operation is already in progress")
+)
+
 // AugmentSyncMsg enrich the K8s message with user-relevant information
 func AugmentSyncMsg(res common.ResourceSyncResult, apiResourceInfoGetter func() ([]kube.APIResourceInfo, error)) (string, error) {
 	switch res.Message {
@@ -800,7 +804,7 @@ func SetAppOperation(appIf v1alpha1.ApplicationInterface, appName string, op *ar
 			return nil, fmt.Errorf("error getting application %q: %w", appName, err)
 		}
 		if a.Operation != nil {
-			return nil, status.Errorf(codes.FailedPrecondition, "another operation is already in progress")
+			return nil, ErrAnotherOperationInProgress
 		}
 		a.Operation = op
 		a.Status.OperationState = nil