Add Requeue logic for Job Style resources till completion (#10)

* Add Requeue logic for Job Style resources till completion

* Add custom describe such that requeue stops on job completion correctly. Also Debugger check

* address review comments

Author: mbaijal

apis/v1alpha1/endpoint.go

@@ -17,6 +17,9 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
+    - jsonPath: .status.failureReason
+      name: FailureReason
+      type: string
     - jsonPath: .status.secondaryStatus
       name: SecondaryStatus
       type: string
@@ -430,6 +433,24 @@ spec:
                   - type
                   type: object
                 type: array
+              debugRuleEvaluationStatuses:
+                description: Evaluation status of Debugger rules for debugging on
+                  a training job.
+                items:
+                  properties:
+                    lastModifiedTime:
+                      format: date-time
+                      type: string
+                    ruleConfigurationName:
+                      type: string
+                    ruleEvaluationJobARN:
+                      type: string
+                    ruleEvaluationStatus:
+                      type: string
+                    statusDetails:
+                      type: string
+                  type: object
+                type: array
               failureReason:
                 description: If the training job failed, the reason it failed.
                 type: string

apis/v1alpha1/training_job.go

@@ -1,10 +1,22 @@
 operations:
+  CreateTrainingJob:
+    set_output_custom_method_name: customCreateTrainingJobSetOutput
+  DescribeTrainingJob:
+    set_output_custom_method_name: customDescribeTrainingJobSetOutput
   StopTrainingJob:
     operation_type: Delete
     resource_name: TrainingJob
+  CreateProcessingJob:
+    set_output_custom_method_name: customCreateProcessingJobSetOutput
+  DescribeProcessingJob:
+    set_output_custom_method_name: customDescribeProcessingJobSetOutput
   StopProcessingJob:
     operation_type: Delete
     resource_name: ProcessingJob
+  CreateTransformJob:
+    set_output_custom_method_name: customCreateTransformJobSetOutput
+  DescribeTransformJob:
+    set_output_custom_method_name: customDescribeTransformJobSetOutput
   StopTransformJob:
     operation_type: Delete
     resource_name: TransformJob
@@ -19,6 +31,10 @@ operations:
       RetainAllVariantProperties: true
   DeleteEndpoint:
     custom_implementation: customDeleteEndpoint
+  CreateHyperParameterTuningJob:
+    set_output_custom_method_name: customCreateHyperParameterTuningJobSetOutput
+  DescribeHyperParameterTuningJob:
+    set_output_custom_method_name: customDescribeHyperParameterTuningJobSetOutput
   StopHyperParameterTuningJob:
     operation_type: Delete
     resource_name: HyperParameterTuningJob
@@ -155,8 +171,14 @@ resources:
         from:
           operation: DescribeTrainingJob
           path: SecondaryStatus
+      DebugRuleEvaluationStatuses:
+        is_read_only: true
+        from:
+          operation: DescribeTrainingJob
+          path: DebugRuleEvaluationStatuses 
       FailureReason:
         is_read_only: true
+        is_printable: true
         from:
           operation: DescribeTrainingJob
           path: FailureReason 

apis/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,92 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// Use this file if the Status/Spec of the CR needs to be modified after
+// create/describe/update operation
+
+package hyper_parameter_tuning_job
+
+import (
+	"context"
+
+	ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1"
+	svcapitypes "github.com/aws-controllers-k8s/sagemaker-controller/apis/v1alpha1"
+	"github.com/aws/aws-sdk-go/aws"
+	svcsdk "github.com/aws/aws-sdk-go/service/sagemaker"
+	corev1 "k8s.io/api/core/v1"
+)
+
+// customCreateHyperParameterTuningJobSetOutput sets the resource in TempOutofSync if HyperParameterTuningJob is
+// in creating state. At this stage we know call to createHyperParameterTuningJob was successful.
+func (rm *resourceManager) customCreateHyperParameterTuningJobSetOutput(
+	ctx context.Context,
+	r *resource,
+	resp *svcsdk.CreateHyperParameterTuningJobOutput,
+	ko *svcapitypes.HyperParameterTuningJob,
+) (*svcapitypes.HyperParameterTuningJob, error) {
+	rm.customSetOutput(r, aws.String(svcsdk.HyperParameterTuningJobStatusInProgress), ko)
+	return ko, nil
+}
+
+// customDescribeHyperParameterTuningJobSetOutput sets the resource in TempOutofSync if
+// HyperParameterTuningJob is being modified by AWS.
+func (rm *resourceManager) customDescribeHyperParameterTuningJobSetOutput(
+	ctx context.Context,
+	r *resource,
+	resp *svcsdk.DescribeHyperParameterTuningJobOutput,
+	ko *svcapitypes.HyperParameterTuningJob,
+) (*svcapitypes.HyperParameterTuningJob, error) {
+	rm.customSetOutput(r, resp.HyperParameterTuningJobStatus, ko)
+	return ko, nil
+}
+
+// customSetOutput sets ConditionTypeResourceSynced condition to True or False
+// based on the hyperParameterTuningJobStatus on AWS so the reconciler can determine if a
+// requeue is needed
+func (rm *resourceManager) customSetOutput(
+	r *resource,
+	hyperParameterTuningJobStatus *string,
+	ko *svcapitypes.HyperParameterTuningJob,
+) {
+	if hyperParameterTuningJobStatus == nil {
+		return
+	}
+
+	syncConditionStatus := corev1.ConditionUnknown
+	if *hyperParameterTuningJobStatus == svcsdk.HyperParameterTuningJobStatusCompleted || *hyperParameterTuningJobStatus == svcsdk.HyperParameterTuningJobStatusStopped || *hyperParameterTuningJobStatus == svcsdk.HyperParameterTuningJobStatusFailed {
+		syncConditionStatus = corev1.ConditionTrue
+	} else {
+		syncConditionStatus = corev1.ConditionFalse
+	}
+
+	var resourceSyncedCondition *ackv1alpha1.Condition = nil
+	if ko.Status.Conditions == nil {
+		ko.Status.Conditions = []*ackv1alpha1.Condition{}
+	} else {
+		for _, condition := range ko.Status.Conditions {
+			if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
+				resourceSyncedCondition = condition
+				break
+			}
+		}
+	}
+
+	if resourceSyncedCondition == nil {
+		resourceSyncedCondition = &ackv1alpha1.Condition{
+			Type: ackv1alpha1.ConditionTypeResourceSynced,
+		}
+		ko.Status.Conditions = append(ko.Status.Conditions, resourceSyncedCondition)
+	}
+	resourceSyncedCondition.Status = syncConditionStatus
+
+}

config/crd/bases/sagemaker.services.k8s.aws_trainingjobs.yaml

@@ -0,0 +1,92 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// Use this file if the Status/Spec of the CR needs to be modified after
+// create/describe/update operation
+
+package processing_job
+
+import (
+	"context"
+
+	ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1"
+	svcapitypes "github.com/aws-controllers-k8s/sagemaker-controller/apis/v1alpha1"
+	"github.com/aws/aws-sdk-go/aws"
+	svcsdk "github.com/aws/aws-sdk-go/service/sagemaker"
+	corev1 "k8s.io/api/core/v1"
+)
+
+// customCreateProcessingJobSetOutput sets the resource in TempOutofSync if ProcessingJob is
+// in creating state. At this stage we know call to createProcessingJob was successful.
+func (rm *resourceManager) customCreateProcessingJobSetOutput(
+	ctx context.Context,
+	r *resource,
+	resp *svcsdk.CreateProcessingJobOutput,
+	ko *svcapitypes.ProcessingJob,
+) (*svcapitypes.ProcessingJob, error) {
+	rm.customSetOutput(r, aws.String(svcsdk.ProcessingJobStatusInProgress), ko)
+	return ko, nil
+}
+
+// customDescribeProcessingJobSetOutput sets the resource in TempOutofSync if
+// ProcessingJob is being modified by AWS.
+func (rm *resourceManager) customDescribeProcessingJobSetOutput(
+	ctx context.Context,
+	r *resource,
+	resp *svcsdk.DescribeProcessingJobOutput,
+	ko *svcapitypes.ProcessingJob,
+) (*svcapitypes.ProcessingJob, error) {
+	rm.customSetOutput(r, resp.ProcessingJobStatus, ko)
+	return ko, nil
+}
+
+// customSetOutput sets ConditionTypeResourceSynced condition to True or False
+// based on the processingJobStatus on AWS so the reconciler can determine if a
+// requeue is needed
+func (rm *resourceManager) customSetOutput(
+	r *resource,
+	processingJobStatus *string,
+	ko *svcapitypes.ProcessingJob,
+) {
+	if processingJobStatus == nil {
+		return
+	}
+
+	syncConditionStatus := corev1.ConditionUnknown
+	if *processingJobStatus == svcsdk.ProcessingJobStatusCompleted || *processingJobStatus == svcsdk.ProcessingJobStatusStopped || *processingJobStatus == svcsdk.ProcessingJobStatusFailed {
+		syncConditionStatus = corev1.ConditionTrue
+	} else {
+		syncConditionStatus = corev1.ConditionFalse
+	}
+
+	var resourceSyncedCondition *ackv1alpha1.Condition = nil
+	if ko.Status.Conditions == nil {
+		ko.Status.Conditions = []*ackv1alpha1.Condition{}
+	} else {
+		for _, condition := range ko.Status.Conditions {
+			if condition.Type == ackv1alpha1.ConditionTypeResourceSynced {
+				resourceSyncedCondition = condition
+				break
+			}
+		}
+	}
+
+	if resourceSyncedCondition == nil {
+		resourceSyncedCondition = &ackv1alpha1.Condition{
+			Type: ackv1alpha1.ConditionTypeResourceSynced,
+		}
+		ko.Status.Conditions = append(ko.Status.Conditions, resourceSyncedCondition)
+	}
+	resourceSyncedCondition.Status = syncConditionStatus
+
+}