Disable istio sidecar injection in Suggestion and Training Jobs (#1050)

* Add istio sidecar inject annotation to Suggestion pod and all training jobs

* Add annotation to Training Jobs from Controller

* Fix comment

pkg/controller.v1alpha3/consts/const.go

@@ -113,6 +113,12 @@ const (
 	JobGroupJob = "batch"
 	// JobGroupKubeflow is the group name of Kubeflow.
 	JobGroupKubeflow = "kubeflow.org"
+
+	// AnnotationIstioSidecarInjectName is the annotation of Istio Sidecar
+	AnnotationIstioSidecarInjectName = "sidecar.istio.io/inject"
+
+	// AnnotationIstioSidecarInjectValue is the value of Istio Sidecar annotation
+	AnnotationIstioSidecarInjectValue = "false"
 )
 
 var (

pkg/controller.v1alpha3/suggestion/composer/composer.go

@@ -67,7 +67,7 @@ func (g *General) DesiredDeployment(s *suggestionsv1alpha3.Suggestion) (*appsv1.
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      util.SuggestionLabels(s),
-					Annotations: s.Annotations,
+					Annotations: util.SuggestionAnnotations(s),
 				},
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{

pkg/controller.v1alpha3/trial/trial_controller.go

@@ -45,6 +45,7 @@ import (
 
 	trialsv1alpha3 "github.com/kubeflow/katib/pkg/apis/controller/trials/v1alpha3"
 	"github.com/kubeflow/katib/pkg/controller.v1alpha3/trial/managerclient"
+	"github.com/kubeflow/katib/pkg/controller.v1alpha3/util"
 	jobv1alpha3 "github.com/kubeflow/katib/pkg/job/v1alpha3"
 )
 
@@ -258,6 +259,13 @@ func (r *ReconcileTrial) reconcileJob(instance *trialsv1alpha3.Trial, desiredJob
 	kind := desiredJob.GetKind()
 	gvk := schema.FromAPIVersionAndKind(apiVersion, kind)
 
+	// Add annotation to desired Job
+	err = util.TrainingJobAnnotations(desiredJob)
+	if err != nil {
+		logger.Error(err, "TrainingJobAnnotations error")
+		return nil, err
+	}
+
 	deployedJob := &unstructured.Unstructured{}
 	deployedJob.SetGroupVersionKind(gvk)
 	err = r.Get(context.TODO(), types.NamespacedName{Name: desiredJob.GetName(), Namespace: desiredJob.GetNamespace()}, deployedJob)

pkg/controller.v1alpha3/util/annotations.go

@@ -0,0 +1,101 @@
+package util
+
+import (
+	"fmt"
+
+	suggestionsv1alpha3 "github.com/kubeflow/katib/pkg/apis/controller/suggestions/v1alpha3"
+	"github.com/kubeflow/katib/pkg/controller.v1alpha3/consts"
+	pytorchv1 "github.com/kubeflow/pytorch-operator/pkg/apis/pytorch/v1"
+	tfv1 "github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+)
+
+var (
+	log = logf.Log.WithName("util-annotations")
+)
+
+// SuggestionAnnotations returns the expected suggestion annotations.
+func SuggestionAnnotations(instance *suggestionsv1alpha3.Suggestion) map[string]string {
+	return appendAnnotation(
+		instance.Annotations,
+		consts.AnnotationIstioSidecarInjectName,
+		consts.AnnotationIstioSidecarInjectValue)
+}
+
+// TrainingJobAnnotations adds annotations to unstructured job.
+func TrainingJobAnnotations(desiredJob *unstructured.Unstructured) error {
+	kind := desiredJob.GetKind()
+	switch kind {
+	case consts.JobKindJob:
+		batchJob := &batchv1.Job{}
+		err := runtime.DefaultUnstructuredConverter.FromUnstructured(desiredJob.Object, &batchJob)
+		if err != nil {
+			log.Error(err, "Convert unstructured to job error")
+			return err
+		}
+		batchJob.Spec.Template.ObjectMeta.Annotations = appendAnnotation(
+			batchJob.Spec.Template.ObjectMeta.Annotations,
+			consts.AnnotationIstioSidecarInjectName,
+			consts.AnnotationIstioSidecarInjectValue)
+		desiredJob.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(batchJob)
+		if err != nil {
+			log.Error(err, "Convert job to unstructured error")
+			return err
+		}
+		return nil
+	case consts.JobKindTF:
+		tfJob := &tfv1.TFJob{}
+		err := runtime.DefaultUnstructuredConverter.FromUnstructured(desiredJob.Object, &tfJob)
+		if err != nil {
+			log.Error(err, "Convert unstructured to TFJob error")
+			return err
+		}
+		for _, replicaSpec := range tfJob.Spec.TFReplicaSpecs {
+			replicaSpec.Template.ObjectMeta.Annotations = appendAnnotation(
+				replicaSpec.Template.ObjectMeta.Annotations,
+				consts.AnnotationIstioSidecarInjectName,
+				consts.AnnotationIstioSidecarInjectValue)
+		}
+		desiredJob.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(tfJob)
+		if err != nil {
+			log.Error(err, "Convert TFJob to unstructured error")
+			return err
+		}
+		return nil
+	case consts.JobKindPyTorch:
+		pytorchJob := &pytorchv1.PyTorchJob{}
+		err := runtime.DefaultUnstructuredConverter.FromUnstructured(desiredJob.Object, &pytorchJob)
+		if err != nil {
+			log.Error(err, "Convert unstructured to PytorchJob error")
+			return err
+		}
+		for _, replicaSpec := range pytorchJob.Spec.PyTorchReplicaSpecs {
+			replicaSpec.Template.ObjectMeta.Annotations = appendAnnotation(
+				replicaSpec.Template.ObjectMeta.Annotations,
+				consts.AnnotationIstioSidecarInjectName,
+				consts.AnnotationIstioSidecarInjectValue)
+		}
+		desiredJob.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(pytorchJob)
+		if err != nil {
+			log.Error(err, "Convert PytorchJob to unstructured error")
+			return err
+		}
+		return nil
+	default:
+		return fmt.Errorf("Invalid Katib Training Job kind %v", kind)
+	}
+
+}
+
+func appendAnnotation(annotations map[string]string, newAnnotationName string, newAnnotationValue string) map[string]string {
+	res := make(map[string]string)
+	for k, v := range annotations {
+		res[k] = v
+	}
+	res[newAnnotationName] = newAnnotationValue
+
+	return res
+}