fix: Reorder to skip observation collection

Signed-off-by: Ce Gao <gaoce@caicloud.io>

pkg/controller.v1alpha3/trial/trial_controller.go

@@ -206,19 +206,23 @@ func (r *ReconcileTrial) reconcileTrial(instance *trialsv1alpha3.Trial) error {
  return err
  }
 
- //Job already exists
- //TODO Can desired Spec differ from deployedSpec?
+ // Job already exists
+ // TODO Can desired Spec differ from deployedSpec?
  if deployedJob != nil {
- if err = r.UpdateTrialStatusObservation(instance, deployedJob); err != nil {
- logger.Error(err, "Update trial status observation error")
- return err
- }
-
  jobConditionType, err := r.GetDeployedJobStatus(deployedJob)
  if err != nil {
  logger.Error(err, "Get deployed status error")
  return err
  }
+
+ // Update trial observation when the job is succeeded.
+ if isTrialSucceeded(instance, *jobConditionType) {
+ if err = r.UpdateTrialStatusObservation(instance, deployedJob); err != nil {
+ logger.Error(err, "Update trial status observation error")
+ return err
+ }
+ }
+
  // Update Trial job status only
  // if job has succeded and if observation field is available.
  // if job has failed

pkg/controller.v1alpha3/trial/trial_controller_util.go

@@ -25,6 +25,7 @@ import (
  "k8s.io/apimachinery/pkg/runtime"
  "sigs.k8s.io/controller-runtime/pkg/reconcile"
 
+ "github.com/caicloud/nirvana/log"
  commonv1alpha3 "github.com/kubeflow/katib/pkg/apis/controller/common/v1alpha3"
  trialsv1alpha3 "github.com/kubeflow/katib/pkg/apis/controller/trials/v1alpha3"
  api_pb "github.com/kubeflow/katib/pkg/apis/manager/v1alpha3"
@@ -155,6 +156,14 @@ func isTrialComplete(instance *trialsv1alpha3.Trial, jobConditionType commonv1.J
  return false
 }
 
+func isTrialSucceeded(instance *trialsv1alpha3.Trial, jobConditionType commonv1.JobConditionType) bool {
+ if jobConditionType == commonv1.JobSucceeded {
+ return true
+ }
+
+ return false
+}
+
 func getBestObjectiveMetricValue(metricLogs []*api_pb.MetricLog, objectiveType commonv1alpha3.ObjectiveType) *float64 {
  metricLogSize := len(metricLogs)
  if metricLogSize == 0 {