Disable istio sidecar injection in Suggestion and Training Jobs

pkg/controller.v1alpha3/consts/const.go

@@ -113,6 +113,12 @@ const (
  JobGroupJob = "batch"
  // JobGroupKubeflow is the group name of Kubeflow.
  JobGroupKubeflow = "kubeflow.org"
+
+ // AnnotationIstioSidecarInjectName is the annotation of Istio Sidecar
+ AnnotationIstioSidecarInjectName = "sidecar.istio.io/inject"
+
+ // AnnotationIstioSidecarInjectValue is the value of Istio Sidecar annotation
+ AnnotationIstioSidecarInjectValue = "false"
 )
 
 var (

pkg/controller.v1alpha3/suggestion/composer/composer.go

@@ -67,7 +67,7 @@ func (g *General) DesiredDeployment(s *suggestionsv1alpha3.Suggestion) (*appsv1.
  Template: corev1.PodTemplateSpec{
  ObjectMeta: metav1.ObjectMeta{
  Labels: util.SuggestionLabels(s),
- Annotations: s.Annotations,
+ Annotations: util.SuggestionAnnotations(s),
  },
  Spec: corev1.PodSpec{
  Containers: []corev1.Container{

pkg/controller.v1alpha3/trial/trial_controller.go

@@ -45,6 +45,7 @@ import (
 
  trialsv1alpha3 "github.com/kubeflow/katib/pkg/apis/controller/trials/v1alpha3"
  "github.com/kubeflow/katib/pkg/controller.v1alpha3/trial/managerclient"
+ "github.com/kubeflow/katib/pkg/controller.v1alpha3/util"
  jobv1alpha3 "github.com/kubeflow/katib/pkg/job/v1alpha3"
 )
 
@@ -258,6 +259,13 @@ func (r *ReconcileTrial) reconcileJob(instance *trialsv1alpha3.Trial, desiredJob
  kind := desiredJob.GetKind()
  gvk := schema.FromAPIVersionAndKind(apiVersion, kind)
 
+ // Add annotation to desired Job
+ err = util.TrainingJobAnnotations(desiredJob)
+ if err != nil {
+ logger.Error(err, "TrainingJobAnnotations error")
+ return nil, err
+ }
+
  deployedJob := &unstructured.Unstructured{}
  deployedJob.SetGroupVersionKind(gvk)
  err = r.Get(context.TODO(), types.NamespacedName{Name: desiredJob.GetName(), Namespace: desiredJob.GetNamespace()}, deployedJob)

pkg/controller.v1alpha3/util/annotations.go

@@ -0,0 +1,101 @@
+package util
+
+import (
+ "fmt"
+
+ suggestionsv1alpha3 "github.com/kubeflow/katib/pkg/apis/controller/suggestions/v1alpha3"
+ "github.com/kubeflow/katib/pkg/controller.v1alpha3/consts"
+ pytorchv1 "github.com/kubeflow/pytorch-operator/pkg/apis/pytorch/v1"
+ tfv1 "github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1"
+ batchv1 "k8s.io/api/batch/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/runtime"
+ logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+)
+
+var (
+ log = logf.Log.WithName("util-annotations")
+)
+
+// SuggestionAnnotations returns the expected suggestion annotations.
+func SuggestionAnnotations(instance *suggestionsv1alpha3.Suggestion) map[string]string {
+ return appendAnnotation(
+ instance.Annotations,
+ consts.AnnotationIstioSidecarInjectName,
+ consts.AnnotationIstioSidecarInjectValue)
+}
+
+// TrainingJobAnnotations adds annotations to unstructured job.
+func TrainingJobAnnotations(desiredJob *unstructured.Unstructured) error {
+ kind := desiredJob.GetKind()
+ switch kind {
+ case consts.JobKindJob:
+ batchJob := &batchv1.Job{}
+ err := runtime.DefaultUnstructuredConverter.FromUnstructured(desiredJob.Object, &batchJob)
+ if err != nil {
+ log.Error(err, "Convert unstructured to job error")
+ return err
+ }
+ batchJob.Spec.Template.ObjectMeta.Annotations = appendAnnotation(
+ batchJob.Spec.Template.ObjectMeta.Annotations,
+ consts.AnnotationIstioSidecarInjectName,
+ consts.AnnotationIstioSidecarInjectValue)
+ desiredJob.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(batchJob)
+ if err != nil {
+ log.Error(err, "Convert job to unstructured error")
+ return err
+ }
+ return nil
+ case consts.JobKindTF:
+ tfJob := &tfv1.TFJob{}
+ err := runtime.DefaultUnstructuredConverter.FromUnstructured(desiredJob.Object, &tfJob)
+ if err != nil {
+ log.Error(err, "Convert unstructured to TFJob error")
+ return err
+ }
+ for _, replicaSpec := range tfJob.Spec.TFReplicaSpecs {
+ replicaSpec.Template.ObjectMeta.Annotations = appendAnnotation(
+ replicaSpec.Template.ObjectMeta.Annotations,
+ consts.AnnotationIstioSidecarInjectName,
+ consts.AnnotationIstioSidecarInjectValue)
+ }
+ desiredJob.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(tfJob)
+ if err != nil {
+ log.Error(err, "Convert TFJob to unstructured error")
+ return err
+ }
+ return nil
+ case consts.JobKindPyTorch:
+ pytorchJob := &pytorchv1.PyTorchJob{}
+ err := runtime.DefaultUnstructuredConverter.FromUnstructured(desiredJob.Object, &pytorchJob)
+ if err != nil {
+ log.Error(err, "Convert unstructured to PytorchJob error")
+ return err
+ }
+ for _, replicaSpec := range pytorchJob.Spec.PyTorchReplicaSpecs {
+ replicaSpec.Template.ObjectMeta.Annotations = appendAnnotation(
+ replicaSpec.Template.ObjectMeta.Annotations,
+ consts.AnnotationIstioSidecarInjectName,
+ consts.AnnotationIstioSidecarInjectValue)
+ }
+ desiredJob.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(pytorchJob)
+ if err != nil {
+ log.Error(err, "Convert PytorchJob to unstructured error")
+ return err
+ }
+ return nil
+ default:
+ return fmt.Errorf("Invalid Katib Training Job kind %v", kind)
+ }
+
+}
+
+func appendAnnotation(annotations map[string]string, newAnnotationName string, newAnnotationValue string) map[string]string {
+ res := make(map[string]string)
+ for k, v := range annotations {
+ res[k] = v
+ }
+ res[newAnnotationName] = newAnnotationValue
+
+ return res
+}