From e998ffea376ca178d98a54bccb37d0e5e1e95a99 Mon Sep 17 00:00:00 2001 From: Kyrie Chen Date: Tue, 13 Dec 2022 13:35:29 -0500 Subject: [PATCH] Adds updateTags API for snapshot. --- apis/v1alpha1/ack-generate-metadata.yaml | 4 +- apis/v1alpha1/generator.yaml | 2 + generator.yaml | 2 + pkg/resource/snapshot/hooks.go | 164 +++++++++++++++++- pkg/resource/snapshot/sdk.go | 10 +- .../sdk_read_many_post_set_output.go.tpl | 9 +- .../e2e/scenarios/Snapshot/snapshot_copy.yaml | 6 +- .../scenarios/Snapshot/snapshot_create.yaml | 4 +- .../snapshot_create_terminal_condition.yaml | 4 +- .../Snapshot/snapshot_update_with_tags.yaml | 107 ++++++++++++ 10 files changed, 294 insertions(+), 18 deletions(-) create mode 100644 test/e2e/scenarios/Snapshot/snapshot_update_with_tags.yaml diff --git a/apis/v1alpha1/ack-generate-metadata.yaml b/apis/v1alpha1/ack-generate-metadata.yaml index dd227d2..4127469 100755 --- a/apis/v1alpha1/ack-generate-metadata.yaml +++ b/apis/v1alpha1/ack-generate-metadata.yaml @@ -1,5 +1,5 @@ ack_generate_info: - build_date: "2022-12-23T17:00:07Z" + build_date: "2022-12-24T00:37:23Z" build_hash: 16f0e201b37a06b535370cc69e11adb934a22d33 go_version: go1.19 version: v0.20.1-18-g16f0e20 @@ -7,7 +7,7 @@ api_directory_checksum: a1e396caca4bdd1612fa7d09f0ee56f3e4976ff7 api_version: v1alpha1 aws_sdk_go_version: v1.44.93 generator_config_info: - file_checksum: f50534e33903e1ca74491595fd6c21351988eb1c + file_checksum: d7ad13c5bc8d9e9e2171c92dc3ac51c2b5e3b769 original_file_name: generator.yaml last_modification: reason: API generation diff --git a/apis/v1alpha1/generator.yaml b/apis/v1alpha1/generator.yaml index e7f37eb..f129327 100644 --- a/apis/v1alpha1/generator.yaml +++ b/apis/v1alpha1/generator.yaml @@ -212,6 +212,8 @@ resources: from: operation: CreateSnapshot path: ClusterName + update_operation: + custom_method_name: customUpdate hooks: sdk_create_pre_build_request: template_path: hooks/snapshot/sdk_create_pre_build_request.go.tpl diff --git a/generator.yaml b/generator.yaml index e7f37eb..f129327 100644 --- a/generator.yaml +++ b/generator.yaml @@ -212,6 +212,8 @@ resources: from: operation: CreateSnapshot path: ClusterName + update_operation: + custom_method_name: customUpdate hooks: sdk_create_pre_build_request: template_path: hooks/snapshot/sdk_create_pre_build_request.go.tpl diff --git a/pkg/resource/snapshot/hooks.go b/pkg/resource/snapshot/hooks.go index 976ae87..6c5f5d5 100644 --- a/pkg/resource/snapshot/hooks.go +++ b/pkg/resource/snapshot/hooks.go @@ -17,18 +17,20 @@ import ( "context" "errors" - svcapitypes "github.com/aws-controllers-k8s/memorydb-controller/apis/v1alpha1" - ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1" - ackerr "github.com/aws-controllers-k8s/runtime/pkg/errors" - "github.com/aws/aws-sdk-go/service/memorydb" svcsdk "github.com/aws/aws-sdk-go/service/memorydb" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + svcapitypes "github.com/aws-controllers-k8s/memorydb-controller/apis/v1alpha1" + ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1" + ackcompare "github.com/aws-controllers-k8s/runtime/pkg/compare" + ackerr "github.com/aws-controllers-k8s/runtime/pkg/errors" + ackrtlog "github.com/aws-controllers-k8s/runtime/pkg/runtime/log" + ackutil "github.com/aws-controllers-k8s/runtime/pkg/util" ) func (rm *resourceManager) customDescribeSnapshotSetOutput( - resp *memorydb.DescribeSnapshotsOutput, + resp *svcsdk.DescribeSnapshotsOutput, ko *svcapitypes.Snapshot, ) (*svcapitypes.Snapshot, error) { if len(resp.Snapshots) == 0 { @@ -40,7 +42,7 @@ func (rm *resourceManager) customDescribeSnapshotSetOutput( } func (rm *resourceManager) customCreateSnapshotSetOutput( - resp *memorydb.CreateSnapshotOutput, + resp *svcsdk.CreateSnapshotOutput, ko *svcapitypes.Snapshot, ) (*svcapitypes.Snapshot, error) { rm.customSetOutput(resp.Snapshot, ko) @@ -48,7 +50,7 @@ func (rm *resourceManager) customCreateSnapshotSetOutput( } func (rm *resourceManager) customCopySnapshotSetOutput( - resp *memorydb.CopySnapshotOutput, + resp *svcsdk.CopySnapshotOutput, ko *svcapitypes.Snapshot, ) *svcapitypes.Snapshot { rm.customSetOutput(resp.Snapshot, ko) @@ -56,7 +58,7 @@ func (rm *resourceManager) customCopySnapshotSetOutput( } func (rm *resourceManager) customSetOutput( - respSnapshot *memorydb.Snapshot, + respSnapshot *svcsdk.Snapshot, ko *svcapitypes.Snapshot, ) { if ko.Status.Conditions == nil { @@ -245,3 +247,147 @@ func (rm *resourceManager) newCopySnapshotPayload( return res, nil } + +// getTags gets tags from given ParameterGroup. +func (rm *resourceManager) getTags( + ctx context.Context, + resourceARN string, +) ([]*svcapitypes.Tag, error) { + resp, err := rm.sdkapi.ListTagsWithContext( + ctx, + &svcsdk.ListTagsInput{ + ResourceArn: &resourceARN, + }, + ) + rm.metrics.RecordAPICall("GET", "ListTags", err) + if err != nil { + return nil, err + } + tags := resourceTagsFromSDKTags(resp.TagList) + return tags, nil +} + +func (rm *resourceManager) customUpdate( + ctx context.Context, + desired *resource, + latest *resource, + delta *ackcompare.Delta, +) (updated *resource, err error) { + rlog := ackrtlog.FromContext(ctx) + exit := rlog.Trace("rm.customUpdate") + defer func(err error) { exit(err) }(err) + if delta.DifferentAt("Spec.Tags") { + if err = rm.updateTags(ctx, desired, latest); err != nil { + return nil, err + } + } + return desired, nil +} + +// updateTags updates tags of given ParameterGroup to desired tags. +func (rm *resourceManager) updateTags( + ctx context.Context, + desired *resource, + latest *resource, +) (err error) { + rlog := ackrtlog.FromContext(ctx) + exit := rlog.Trace("rm.updateTags") + defer func(err error) { exit(err) }(err) + + arn := (*string)(latest.ko.Status.ACKResourceMetadata.ARN) + + toAdd, toDelete := computeTagsDelta( + desired.ko.Spec.Tags, latest.ko.Spec.Tags, + ) + + if len(toDelete) > 0 { + rlog.Debug("removing tags from snapshot", "tags", toDelete) + _, err = rm.sdkapi.UntagResourceWithContext( + ctx, + &svcsdk.UntagResourceInput{ + ResourceArn: arn, + TagKeys: toDelete, + }, + ) + rm.metrics.RecordAPICall("UPDATE", "UntagResource", err) + if err != nil { + return err + } + } + + if len(toAdd) > 0 { + rlog.Debug("adding tags to snapshot", "tags", toAdd) + _, err = rm.sdkapi.TagResourceWithContext( + ctx, + &svcsdk.TagResourceInput{ + ResourceArn: arn, + Tags: sdkTagsFromResourceTags(toAdd), + }, + ) + rm.metrics.RecordAPICall("UPDATE", "TagResource", err) + if err != nil { + return err + } + } + + return nil +} + +func computeTagsDelta( + desired []*svcapitypes.Tag, + latest []*svcapitypes.Tag, +) (addedOrUpdated []*svcapitypes.Tag, removed []*string) { + var visitedIndexes []string + + for _, latestElement := range latest { + visitedIndexes = append(visitedIndexes, *latestElement.Key) + for _, desiredElement := range desired { + if equalStrings(latestElement.Key, desiredElement.Key) { + if !equalStrings(latestElement.Value, desiredElement.Value) { + addedOrUpdated = append(addedOrUpdated, desiredElement) + } + continue + } + } + removed = append(removed, latestElement.Key) + } + for _, desiredElement := range desired { + if !ackutil.InStrings(*desiredElement.Key, visitedIndexes) { + addedOrUpdated = append(addedOrUpdated, desiredElement) + } + } + return addedOrUpdated, removed +} + +func sdkTagsFromResourceTags( + rTags []*svcapitypes.Tag, +) []*svcsdk.Tag { + tags := make([]*svcsdk.Tag, len(rTags)) + for i := range rTags { + tags[i] = &svcsdk.Tag{ + Key: rTags[i].Key, + Value: rTags[i].Value, + } + } + return tags +} + +func resourceTagsFromSDKTags( + sdkTags []*svcsdk.Tag, +) []*svcapitypes.Tag { + tags := make([]*svcapitypes.Tag, len(sdkTags)) + for i := range sdkTags { + tags[i] = &svcapitypes.Tag{ + Key: sdkTags[i].Key, + Value: sdkTags[i].Value, + } + } + return tags +} + +func equalStrings(a, b *string) bool { + if a == nil { + return b == nil || *b == "" + } + return (*a == "" && b == nil) || *a == *b +} diff --git a/pkg/resource/snapshot/sdk.go b/pkg/resource/snapshot/sdk.go index a235fc1..8213058 100644 --- a/pkg/resource/snapshot/sdk.go +++ b/pkg/resource/snapshot/sdk.go @@ -200,6 +200,13 @@ func (rm *resourceManager) sdkFind( if err != nil { return nil, err } + + resourceARN := (*string)(ko.Status.ACKResourceMetadata.ARN) + tags, err := rm.getTags(ctx, *resourceARN) + if err != nil { + return nil, err + } + ko.Spec.Tags = tags return &resource{ko}, nil } @@ -411,8 +418,7 @@ func (rm *resourceManager) sdkUpdate( latest *resource, delta *ackcompare.Delta, ) (*resource, error) { - // TODO(jaypipes): Figure this out... - return nil, ackerr.NotImplemented + return rm.customUpdate(ctx, desired, latest, delta) } // sdkDelete deletes the supplied resource in the backend AWS service API diff --git a/templates/hooks/snapshot/sdk_read_many_post_set_output.go.tpl b/templates/hooks/snapshot/sdk_read_many_post_set_output.go.tpl index c6d81fa..b8a676c 100644 --- a/templates/hooks/snapshot/sdk_read_many_post_set_output.go.tpl +++ b/templates/hooks/snapshot/sdk_read_many_post_set_output.go.tpl @@ -2,4 +2,11 @@ ko, err = rm.customDescribeSnapshotSetOutput(resp, ko) if err != nil { return nil, err - } \ No newline at end of file + } + + resourceARN := (*string)(ko.Status.ACKResourceMetadata.ARN) + tags, err := rm.getTags(ctx, *resourceARN) + if err != nil { + return nil, err + } + ko.Spec.Tags = tags \ No newline at end of file diff --git a/test/e2e/scenarios/Snapshot/snapshot_copy.yaml b/test/e2e/scenarios/Snapshot/snapshot_copy.yaml index 0d15296..9822b73 100644 --- a/test/e2e/scenarios/Snapshot/snapshot_copy.yaml +++ b/test/e2e/scenarios/Snapshot/snapshot_copy.yaml @@ -1,5 +1,8 @@ id: "SNAPSHOT_COPY" description: "In this test we copy snapshot from another snapshot" +#marks: +# - slow +# - blocked steps: - id: "CREATE_INITIAL_SNAPSHOT" description: "Create Initial Snapshot" @@ -7,9 +10,8 @@ steps: apiVersion: $CRD_GROUP/$CRD_VERSION kind: Snapshot metadata: - name: snapshot$RANDOM_SUFFIX + name: snapshot$RANDOM_SUFFIX spec: - description: "Create ACK snapshot" clusterName: $SNAPSHOT_CLUSTER_NAME2 name: snapshot$RANDOM_SUFFIX wait: diff --git a/test/e2e/scenarios/Snapshot/snapshot_create.yaml b/test/e2e/scenarios/Snapshot/snapshot_create.yaml index 684c030..d83ced4 100644 --- a/test/e2e/scenarios/Snapshot/snapshot_create.yaml +++ b/test/e2e/scenarios/Snapshot/snapshot_create.yaml @@ -1,5 +1,8 @@ id: "SNAPSHOT_CREATE" description: "In this test we create Snapshot" +#marks: +# - slow +# - blocked steps: - id: "CREATE_SNAPSHOT" description: "ACK Snapshot" @@ -9,7 +12,6 @@ steps: metadata: name: snapshot$RANDOM_SUFFIX spec: - description: "Create ACK snapshot" clusterName: $SNAPSHOT_CLUSTER_NAME1 name: snapshot$RANDOM_SUFFIX wait: diff --git a/test/e2e/scenarios/Snapshot/snapshot_create_terminal_condition.yaml b/test/e2e/scenarios/Snapshot/snapshot_create_terminal_condition.yaml index 633859c..e4b4e31 100644 --- a/test/e2e/scenarios/Snapshot/snapshot_create_terminal_condition.yaml +++ b/test/e2e/scenarios/Snapshot/snapshot_create_terminal_condition.yaml @@ -1,5 +1,8 @@ id: "SNAPSHOT_CREATE_TERMINAL_CONDITION" description: "In this test we try to create snapshot without specifying cluster" +#marks: +# - slow +# - blocked steps: - id: "SNAPSHOT_INITIAL_CREATE" description: "Create snapshot with no clustername " @@ -9,7 +12,6 @@ steps: metadata: name: snapshot$RANDOM_SUFFIX spec: - description: "Create ACK snapshot" name: snapshot$RANDOM_SUFFIX wait: 120 expect: diff --git a/test/e2e/scenarios/Snapshot/snapshot_update_with_tags.yaml b/test/e2e/scenarios/Snapshot/snapshot_update_with_tags.yaml new file mode 100644 index 0000000..51ae9b0 --- /dev/null +++ b/test/e2e/scenarios/Snapshot/snapshot_update_with_tags.yaml @@ -0,0 +1,107 @@ +id: "SNAPSHOT_UPDATE_WITH_TAGS" +description: "In this test we create snapshot and update tags" +#marks: +# - slow +# - blocked +steps: + - id: "CLUSTER_INITIAL_CREATE" + description: "Create cluster" + create: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: Cluster + metadata: + name: cluster$RANDOM_SUFFIX + spec: + name: cluster$RANDOM_SUFFIX + nodeType: db.t4g.small + aclName: open-access + numShards: 1 + wait: + status: + conditions: + ACK.ResourceSynced: + status: "True" + timeout: 7200 + - id: "SNAPSHOT_INITIAL_CREATE" + description: "Create snapshot with no tags" + create: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: Snapshot + metadata: + name: snapshot$RANDOM_SUFFIX + spec: + clusterName: cluster$RANDOM_SUFFIX + name: snapshot$RANDOM_SUFFIX + wait: + status: + conditions: + ACK.ResourceSynced: + status: "True" + timeout: 1800 + - id: "SNAPSHOT_ADD_TAGS" + description: "Add tags in snapshot" + patch: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: Snapshot + metadata: + name: snapshot$RANDOM_SUFFIX + spec: + tags: + - key: "test_key_1" + value: "test_value_1" + - key: "test_key_2" + - key: + wait: + status: + conditions: + ACK.ResourceSynced: + status: "True" + timeout: 100 + - id: "SNAPSHOT_DELETE_TAGS" + description: "Delete tags in snapshot" + patch: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: Snapshot + metadata: + name: snapshot$RANDOM_SUFFIX + spec: + tags: + - key: "test_key_1" + value: "test_value_1" + wait: + status: + conditions: + ACK.ResourceSynced: + status: "True" + timeout: 100 + - id: "SNAPSHOT_ADD_AND_DELETE_TAGS" + description: "Add some tags and delete tags in snapshot" + patch: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: Snapshot + metadata: + name: snapshot$RANDOM_SUFFIX + spec: + tags: + - key: "test_key_2" + value: "test_value_2" + wait: + status: + conditions: + ACK.ResourceSynced: + status: "True" + timeout: 100 + - id: "DELETE_SNAPSHOT" + description: "Delete snapshot" + delete: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: Snapshot + metadata: + name: snapshot$RANDOM_SUFFIX + - id: "DELETE_CLUSTER" + description: "Delete cluster" + delete: + apiVersion: $CRD_GROUP/$CRD_VERSION + kind: Cluster + metadata: + name: cluster$RANDOM_SUFFIX \ No newline at end of file