diff --git a/apis/v1alpha1/ack-generate-metadata.yaml b/apis/v1alpha1/ack-generate-metadata.yaml index 026bf99..b629eea 100755 --- a/apis/v1alpha1/ack-generate-metadata.yaml +++ b/apis/v1alpha1/ack-generate-metadata.yaml @@ -1,13 +1,13 @@ ack_generate_info: - build_date: "2022-09-06T12:54:50Z" - build_hash: 585f06bbd6d4cc1b738acb85901e7a009bf452c7 - go_version: go1.17.5 - version: v0.20.0 -api_directory_checksum: bc5fed2df0bfab2a176531654d33d69e8153dc42 + build_date: "2022-09-12T20:46:01Z" + build_hash: 2944c8772f216656d84ee02d392eaca501274c1e + go_version: go1.18.1 + version: v0.20.1 +api_directory_checksum: 2322dd513cbf25a6b2e4b99236b5f3ee0fb8c22e api_version: v1alpha1 aws_sdk_go_version: v1.42.0 generator_config_info: - file_checksum: 7a666081b91b39305bf1eedcc9cc538703e34948 + file_checksum: 8452039ec1ac012add845247a98a2f640647b7b0 original_file_name: generator.yaml last_modification: reason: API generation diff --git a/apis/v1alpha1/generator.yaml b/apis/v1alpha1/generator.yaml index 74bdca9..7821393 100644 --- a/apis/v1alpha1/generator.yaml +++ b/apis/v1alpha1/generator.yaml @@ -34,8 +34,11 @@ resources: fields: Name: is_immutable: true - VirtualClusterID: + VirtualClusterId: is_immutable: true + references: + resource: VirtualCluster + path: Status.ID ExecutionRoleARN: is_immutable: true ReleaseLabel: diff --git a/apis/v1alpha1/job_run.go b/apis/v1alpha1/job_run.go index 9a4e782..fc74c27 100644 --- a/apis/v1alpha1/job_run.go +++ b/apis/v1alpha1/job_run.go @@ -40,8 +40,8 @@ type JobRunSpec struct { // The tags assigned to job runs. Tags map[string]*string `json:"tags,omitempty"` // The virtual cluster ID for which the job run request is submitted. - // +kubebuilder:validation:Required - VirtualClusterID *string `json:"virtualClusterID"` + VirtualClusterID *string `json:"virtualClusterID,omitempty"` + VirtualClusterRef *ackv1alpha1.AWSResourceReferenceWrapper `json:"virtualClusterRef,omitempty"` } // JobRunStatus defines the observed state of JobRun diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index 0784ac3..5563778 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -331,6 +331,11 @@ func (in *JobRunSpec) DeepCopyInto(out *JobRunSpec) { *out = new(string) **out = **in } + if in.VirtualClusterRef != nil { + in, out := &in.VirtualClusterRef, &out.VirtualClusterRef + *out = new(corev1alpha1.AWSResourceReferenceWrapper) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobRunSpec. diff --git a/config/crd/bases/emrcontainers.services.k8s.aws_jobruns.yaml b/config/crd/bases/emrcontainers.services.k8s.aws_jobruns.yaml index 18cec25..6160dc6 100644 --- a/config/crd/bases/emrcontainers.services.k8s.aws_jobruns.yaml +++ b/config/crd/bases/emrcontainers.services.k8s.aws_jobruns.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: jobruns.emrcontainers.services.k8s.aws spec: @@ -73,11 +72,23 @@ spec: description: The virtual cluster ID for which the job run request is submitted. type: string + virtualClusterRef: + description: 'AWSResourceReferenceWrapper provides a wrapper around + *AWSResourceReference type to provide more user friendly syntax + for references using ''from'' field Ex: APIIDRef: from: name: my-api' + properties: + from: + description: AWSResourceReference provides all the values necessary + to reference another k8s resource for finding the identifier(Id/ARN/Name) + properties: + name: + type: string + type: object + type: object required: - executionRoleARN - jobDriver - releaseLabel - - virtualClusterID type: object status: description: JobRunStatus defines the observed state of JobRun @@ -152,9 +163,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/bases/emrcontainers.services.k8s.aws_virtualclusters.yaml b/config/crd/bases/emrcontainers.services.k8s.aws_virtualclusters.yaml index a7e62fa..4cd9644 100644 --- a/config/crd/bases/emrcontainers.services.k8s.aws_virtualclusters.yaml +++ b/config/crd/bases/emrcontainers.services.k8s.aws_virtualclusters.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: virtualclusters.emrcontainers.services.k8s.aws spec: @@ -147,9 +146,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/common/bases/services.k8s.aws_fieldexports.yaml b/config/crd/common/bases/services.k8s.aws_fieldexports.yaml index a435de8..361e01f 100644 --- a/config/crd/common/bases/services.k8s.aws_fieldexports.yaml +++ b/config/crd/common/bases/services.k8s.aws_fieldexports.yaml @@ -66,10 +66,6 @@ spec: description: FieldExportTarget provides the values necessary to identify the output path for a field export. properties: - key: - description: Key overrides the default value (`.`) - for the FieldExport target - type: string kind: description: FieldExportOutputType represents all types that can be produced by a field export operation diff --git a/config/iam/recommended-policy-arn b/config/iam/recommended-policy-arn deleted file mode 100644 index 59881e5..0000000 --- a/config/iam/recommended-policy-arn +++ /dev/null @@ -1 +0,0 @@ -arn:aws:iam::aws:policy/aws-service-role/AmazonEMRContainersServiceRolePolicy diff --git a/config/rbac/cluster-role-controller.yaml b/config/rbac/cluster-role-controller.yaml index f72d496..b130351 100644 --- a/config/rbac/cluster-role-controller.yaml +++ b/config/rbac/cluster-role-controller.yaml @@ -1,4 +1,3 @@ - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/generator.yaml b/generator.yaml index 74bdca9..7821393 100644 --- a/generator.yaml +++ b/generator.yaml @@ -34,8 +34,11 @@ resources: fields: Name: is_immutable: true - VirtualClusterID: + VirtualClusterId: is_immutable: true + references: + resource: VirtualCluster + path: Status.ID ExecutionRoleARN: is_immutable: true ReleaseLabel: diff --git a/helm/crds/emrcontainers.services.k8s.aws_jobruns.yaml b/helm/crds/emrcontainers.services.k8s.aws_jobruns.yaml index 18cec25..6160dc6 100644 --- a/helm/crds/emrcontainers.services.k8s.aws_jobruns.yaml +++ b/helm/crds/emrcontainers.services.k8s.aws_jobruns.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: jobruns.emrcontainers.services.k8s.aws spec: @@ -73,11 +72,23 @@ spec: description: The virtual cluster ID for which the job run request is submitted. type: string + virtualClusterRef: + description: 'AWSResourceReferenceWrapper provides a wrapper around + *AWSResourceReference type to provide more user friendly syntax + for references using ''from'' field Ex: APIIDRef: from: name: my-api' + properties: + from: + description: AWSResourceReference provides all the values necessary + to reference another k8s resource for finding the identifier(Id/ARN/Name) + properties: + name: + type: string + type: object + type: object required: - executionRoleARN - jobDriver - releaseLabel - - virtualClusterID type: object status: description: JobRunStatus defines the observed state of JobRun @@ -152,9 +163,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm/crds/emrcontainers.services.k8s.aws_virtualclusters.yaml b/helm/crds/emrcontainers.services.k8s.aws_virtualclusters.yaml index a7e62fa..4cd9644 100644 --- a/helm/crds/emrcontainers.services.k8s.aws_virtualclusters.yaml +++ b/helm/crds/emrcontainers.services.k8s.aws_virtualclusters.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: virtualclusters.emrcontainers.services.k8s.aws spec: @@ -147,9 +146,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm/crds/services.k8s.aws_adoptedresources.yaml b/helm/crds/services.k8s.aws_adoptedresources.yaml index f764dbb..7dca541 100644 --- a/helm/crds/services.k8s.aws_adoptedresources.yaml +++ b/helm/crds/services.k8s.aws_adoptedresources.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: adoptedresources.services.k8s.aws spec: @@ -170,6 +169,7 @@ spec: - name - uid type: object + x-kubernetes-map-type: atomic type: array type: object required: @@ -224,9 +224,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/helm/crds/services.k8s.aws_fieldexports.yaml b/helm/crds/services.k8s.aws_fieldexports.yaml index a435de8..775f0e6 100644 --- a/helm/crds/services.k8s.aws_fieldexports.yaml +++ b/helm/crds/services.k8s.aws_fieldexports.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: fieldexports.services.k8s.aws spec: @@ -66,10 +65,6 @@ spec: description: FieldExportTarget provides the values necessary to identify the output path for a field export. properties: - key: - description: Key overrides the default value (`.`) - for the FieldExport target - type: string kind: description: FieldExportOutputType represents all types that can be produced by a field export operation @@ -133,9 +128,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/pkg/resource/job_run/delta.go b/pkg/resource/job_run/delta.go index 77aeb44..88e4992 100644 --- a/pkg/resource/job_run/delta.go +++ b/pkg/resource/job_run/delta.go @@ -101,6 +101,9 @@ func newResourceDelta( delta.Add("Spec.VirtualClusterID", a.ko.Spec.VirtualClusterID, b.ko.Spec.VirtualClusterID) } } + if !reflect.DeepEqual(a.ko.Spec.VirtualClusterRef, b.ko.Spec.VirtualClusterRef) { + delta.Add("Spec.VirtualClusterRef", a.ko.Spec.VirtualClusterRef, b.ko.Spec.VirtualClusterRef) + } return delta } diff --git a/pkg/resource/job_run/references.go b/pkg/resource/job_run/references.go index 1152d7e..ed47070 100644 --- a/pkg/resource/job_run/references.go +++ b/pkg/resource/job_run/references.go @@ -17,8 +17,15 @@ package job_run import ( "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1" + ackcondition "github.com/aws-controllers-k8s/runtime/pkg/condition" + ackerr "github.com/aws-controllers-k8s/runtime/pkg/errors" acktypes "github.com/aws-controllers-k8s/runtime/pkg/types" svcapitypes "github.com/aws-controllers-k8s/emrcontainers-controller/apis/v1alpha1" @@ -36,17 +43,95 @@ func (rm *resourceManager) ResolveReferences( apiReader client.Reader, res acktypes.AWSResource, ) (acktypes.AWSResource, error) { - return res, nil + namespace := res.MetaObject().GetNamespace() + ko := rm.concreteResource(res).ko.DeepCopy() + err := validateReferenceFields(ko) + if err == nil { + err = resolveReferenceForVirtualClusterID(ctx, apiReader, namespace, ko) + } + + // If there was an error while resolving any reference, reset all the + // resolved values so that they do not get persisted inside etcd + if err != nil { + ko = rm.concreteResource(res).ko.DeepCopy() + } + if hasNonNilReferences(ko) { + return ackcondition.WithReferencesResolvedCondition(&resource{ko}, err) + } + return &resource{ko}, err } // validateReferenceFields validates the reference field and corresponding // identifier field. func validateReferenceFields(ko *svcapitypes.JobRun) error { + if ko.Spec.VirtualClusterRef != nil && ko.Spec.VirtualClusterID != nil { + return ackerr.ResourceReferenceAndIDNotSupportedFor("VirtualClusterID", "VirtualClusterRef") + } + if ko.Spec.VirtualClusterRef == nil && ko.Spec.VirtualClusterID == nil { + return ackerr.ResourceReferenceOrIDRequiredFor("VirtualClusterID", "VirtualClusterRef") + } return nil } // hasNonNilReferences returns true if resource contains a reference to another // resource func hasNonNilReferences(ko *svcapitypes.JobRun) bool { - return false + return false || (ko.Spec.VirtualClusterRef != nil) +} + +// resolveReferenceForVirtualClusterID reads the resource referenced +// from VirtualClusterRef field and sets the VirtualClusterID +// from referenced resource +func resolveReferenceForVirtualClusterID( + ctx context.Context, + apiReader client.Reader, + namespace string, + ko *svcapitypes.JobRun, +) error { + if ko.Spec.VirtualClusterRef != nil && + ko.Spec.VirtualClusterRef.From != nil { + arr := ko.Spec.VirtualClusterRef.From + if arr == nil || arr.Name == nil || *arr.Name == "" { + return fmt.Errorf("provided resource reference is nil or empty") + } + namespacedName := types.NamespacedName{ + Namespace: namespace, + Name: *arr.Name, + } + obj := svcapitypes.VirtualCluster{} + err := apiReader.Get(ctx, namespacedName, &obj) + if err != nil { + return err + } + var refResourceSynced, refResourceTerminal bool + for _, cond := range obj.Status.Conditions { + if cond.Type == ackv1alpha1.ConditionTypeResourceSynced && + cond.Status == corev1.ConditionTrue { + refResourceSynced = true + } + if cond.Type == ackv1alpha1.ConditionTypeTerminal && + cond.Status == corev1.ConditionTrue { + refResourceTerminal = true + } + } + if refResourceTerminal { + return ackerr.ResourceReferenceTerminalFor( + "VirtualCluster", + namespace, *arr.Name) + } + if !refResourceSynced { + return ackerr.ResourceReferenceNotSyncedFor( + "VirtualCluster", + namespace, *arr.Name) + } + if obj.Status.ID == nil { + return ackerr.ResourceReferenceMissingTargetFieldFor( + "VirtualCluster", + namespace, *arr.Name, + "Status.ID") + } + referencedValue := string(*obj.Status.ID) + ko.Spec.VirtualClusterID = &referencedValue + } + return nil } diff --git a/pkg/resource/job_run/sdk.go b/pkg/resource/job_run/sdk.go index 9429179..24f7633 100644 --- a/pkg/resource/job_run/sdk.go +++ b/pkg/resource/job_run/sdk.go @@ -467,12 +467,6 @@ func (rm *resourceManager) getImmutableFieldChanges( delta *ackcompare.Delta, ) []string { var fields []string - if delta.DifferentAt("Spec.ReleaseLabel") { - fields = append(fields, "ReleaseLabel") - } - if delta.DifferentAt("Spec.VirtualClusterID") { - fields = append(fields, "VirtualClusterID") - } if delta.DifferentAt("Spec.ExecutionRoleARN") { fields = append(fields, "ExecutionRoleARN") } @@ -482,6 +476,12 @@ func (rm *resourceManager) getImmutableFieldChanges( if delta.DifferentAt("Spec.Name") { fields = append(fields, "Name") } + if delta.DifferentAt("Spec.ReleaseLabel") { + fields = append(fields, "ReleaseLabel") + } + if delta.DifferentAt("Spec.VirtualClusterId") { + fields = append(fields, "VirtualClusterId") + } return fields }