Skip restart of pods with proxies with istio's custom image annotation

Makefile

@@ -96,7 +96,7 @@ build: generate fmt vet ## Build manager binary.
 	go build -o bin/manager main.go
 
 .PHONY: run
-run: manifests install create-kyma-system-ns build ## Run a controller from your host.
+run: manifests install create-kyma-system-ns ## Run a controller from your host.
 	go run ./main.go
 
 .PHONY: docker-build

docs/release-notes/1.7.0.md

@@ -2,4 +2,5 @@
 
 - Allow for opting out of the **ENABLE_EXTERNAL_NAME_ALIAS** Istio pilot environment variable in the Istio custom resource. This allows for retaining behavior that was present in Istio prior to version 1.21. See issue [#787](https://github.com/kyma-project/istio/issues/787 ).
 - Update the Istio version to 1.21.2 [#802](https://github.com/kyma-project/istio/pull/802). Read [Istio 1.21.2 Release Announcement](https://istio.io/latest/news/releases/1.21.x/announcing-1.21.2/) and [Change Notes](https://istio.io/latest/news/releases/1.21.x/announcing-1.21/change-notes/) for more details.
-- Add Request Authentication UI for Kyma dashboard [#816](https://github.com/kyma-project/istio/pull/816)
+- Add Request Authentication UI for Kyma dashboard [#816](https://github.com/kyma-project/istio/pull/816)
+- Now, Istio Operator does not restart Pods with Istio Sidecar, which contain custom image annotations. See the issue [#698](https://github.com/kyma-project/istio/issues/698) and [Istio Resource Annotations](https://istio.io/latest/docs/reference/config/annotations/#SidecarProxyImage) for more details.

docs/user/00-10-overview-istio-controller.md

@@ -18,6 +18,7 @@ The `istios.operator.kyma-project.io` CustomResourceDefinition (CRD) describes t
 
 When the Istio version is updated or the configuration of the proxies is changed, the Pods that have Istio injection enabled are automatically restarted. This is possible for all resources that allow for a rolling restart. If Istio is uninstalled, the workloads are restarted again to remove the sidecars.
 However, if a resource is a job, a ReplicaSet that is not managed by any deployment, or a Pod that is not managed by any other resource, the restart cannot be performed automatically. In such cases, a warning is logged, and you must manually restart the resources.
+Istio Operator does not restart an Istio sidecar proxy, if it has a custom image set. See [Resource Annotations](https://istio.io/latest/docs/reference/config/annotations/#SidecarProxyImage).
 
 ## Status Codes
 

docs/user/00-30-overview-istio-sidecars.md

@@ -35,4 +35,4 @@ Application resiliency is an important topic within traffic management. Traditio
 ## Operation Guides and Troubleshooting
 
 [Check if you have automatic Istio sidecar proxy injection enabled](./operation-guides/02-10-check-if-sidecar-injection-is-enabled.md) and learn how to [enable automatic Istio sidecar proxy injection](./operation-guides/02-20-enable-sidecar-injection.md).
-Follow the troubleshooting guides if you experience [issues with Istio sidecar injection](./troubleshooting/03-30-istio-no-sidecar.md) or have [incompatible Istio sidecar version after Kyma Istio Operator's upgrade](./troubleshooting/03-40-incompatible-istio-sidecar-version.md).
+Follow the troubleshooting guides if you experience [issues with Istio sidecar injection](./troubleshooting/03-30-istio-no-sidecar.md) or have [incompatible Istio sidecar version after Kyma Istio Operator's upgrade](./troubleshooting/03-40-incompatible-istio-sidecar-version.md).

pkg/lib/sidecars/pods/filter.go

@@ -7,7 +7,8 @@ import (
 )
 
 const (
-	istioSidecarName = "istio-proxy"
+	istioSidecarName                         = "istio-proxy"
+	istioSidecarCustomImageAnnotation string = "sidecar.istio.io/proxyImage"
 )
 
 type RestartProxyPredicate struct {
@@ -38,6 +39,7 @@ func (r RestartProxyPredicate) NewProxyRestartEvaluator(_ context.Context) (filt
 func needsRestart(pod v1.Pod, expectedImage SidecarImage, expectedResources v1.ResourceRequirements) bool {
 	return HasIstioSidecarStatusAnnotation(pod) &&
 		IsPodReady(pod) &&
+		!hasCustomImageAnnotation(pod) &&
 		(hasSidecarContainerWithWithDifferentImage(pod, expectedImage) || hasDifferentSidecarResources(pod, expectedResources))
 }
 
@@ -64,6 +66,11 @@ func isPodRunning(pod v1.Pod) bool {
 	return pod.Status.Phase == v1.PodRunning
 }
 
+func hasCustomImageAnnotation(pod v1.Pod) bool {
+	_, found := pod.Annotations[istioSidecarCustomImageAnnotation]
+	return found
+}
+
 func hasSidecarContainerWithWithDifferentImage(pod v1.Pod, expectedImage SidecarImage) bool {
 
 	for _, container := range pod.Spec.Containers {

pkg/lib/sidecars/pods/filter_test.go

@@ -0,0 +1,63 @@
+package pods_test
+
+import (
+	"context"
+	"github.com/kyma-project/istio/operator/pkg/lib/sidecars/pods"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Evaluate restart", func() {
+	It("should should return false when pod has custom image annotation", func() {
+		pod := createPodWithProxySidecar("test-pod", "test-namespace", "1.21.0", map[string]string{"sidecar.istio.io/proxyImage": "istio/proxyv2:1.21.0"})
+
+		predicate := pods.NewRestartProxyPredicate(pods.NewSidecarImage("istio", "1.22.0"), v1.ResourceRequirements{})
+		evaluator, err := predicate.NewProxyRestartEvaluator(context.Background())
+		Expect(err).ToNot(HaveOccurred())
+		Expect(evaluator.RequiresProxyRestart(pod)).To(BeFalse())
+
+	})
+
+	It("should should return true when pod does not have custom image annotation", func() {
+		pod := createPodWithProxySidecar("test-pod", "test-namespace", "1.21.0", map[string]string{})
+
+		predicate := pods.NewRestartProxyPredicate(pods.NewSidecarImage("istio", "1.22.0"), v1.ResourceRequirements{})
+		evaluator, err := predicate.NewProxyRestartEvaluator(context.Background())
+		Expect(err).ToNot(HaveOccurred())
+		Expect(evaluator.RequiresProxyRestart(pod)).To(BeTrue())
+
+	})
+})
+
+func createPodWithProxySidecar(name, namespace, proxyIstioVersion string, annotations map[string]string) v1.Pod {
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	annotations["sidecar.istio.io/status"] = "true"
+	return v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        name,
+			Namespace:   namespace,
+			Annotations: annotations,
+		},
+		Status: v1.PodStatus{
+			Phase: v1.PodRunning,
+			Conditions: []v1.PodCondition{
+				{
+					Type:   v1.PodReady,
+					Status: v1.ConditionTrue,
+				},
+			},
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "istio-proxy",
+					Image: "istio/proxyv2:" + proxyIstioVersion,
+				},
+			},
+		},
+	}
+}