diff --git a/pkg/controller.v1alpha3/trial/trial_controller.go b/pkg/controller.v1alpha3/trial/trial_controller.go index fd91b599247..d328e47688a 100644 --- a/pkg/controller.v1alpha3/trial/trial_controller.go +++ b/pkg/controller.v1alpha3/trial/trial_controller.go @@ -206,19 +206,23 @@ func (r *ReconcileTrial) reconcileTrial(instance *trialsv1alpha3.Trial) error { return err } - //Job already exists - //TODO Can desired Spec differ from deployedSpec? + // Job already exists + // TODO Can desired Spec differ from deployedSpec? if deployedJob != nil { - if err = r.UpdateTrialStatusObservation(instance, deployedJob); err != nil { - logger.Error(err, "Update trial status observation error") - return err - } - jobConditionType, err := r.GetDeployedJobStatus(deployedJob) if err != nil { logger.Error(err, "Get deployed status error") return err } + + // Update trial observation when the job is succeeded. + if isTrialSucceeded(instance, *jobConditionType) { + if err = r.UpdateTrialStatusObservation(instance, deployedJob); err != nil { + logger.Error(err, "Update trial status observation error") + return err + } + } + // Update Trial job status only // if job has succeded and if observation field is available. // if job has failed diff --git a/pkg/controller.v1alpha3/trial/trial_controller_util.go b/pkg/controller.v1alpha3/trial/trial_controller_util.go index 3a1d1f12e90..6de61352777 100644 --- a/pkg/controller.v1alpha3/trial/trial_controller_util.go +++ b/pkg/controller.v1alpha3/trial/trial_controller_util.go @@ -155,6 +155,14 @@ func isTrialComplete(instance *trialsv1alpha3.Trial, jobConditionType commonv1.J return false } +func isTrialSucceeded(instance *trialsv1alpha3.Trial, jobConditionType commonv1.JobConditionType) bool { + if jobConditionType == commonv1.JobSucceeded { + return true + } + + return false +} + func getBestObjectiveMetricValue(metricLogs []*api_pb.MetricLog, objectiveType commonv1alpha3.ObjectiveType) *float64 { metricLogSize := len(metricLogs) if metricLogSize == 0 {