From 49b87dbfe5a37abad8880e0325f7aa8218705407 Mon Sep 17 00:00:00 2001 From: Tatsuya Yamamoto Date: Thu, 11 Nov 2021 06:46:51 +0900 Subject: [PATCH] feat(iot): add Action to put objects in S3 Buckets (#17307) I'm trying to implement aws-iot L2 Constructs. This PR is one of steps after following PR: - https://github.com/aws/aws-cdk/pull/16681#issuecomment-942233029 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- package.json | 4 + packages/@aws-cdk/aws-iot-actions/NOTICE | 30 ++++ packages/@aws-cdk/aws-iot-actions/README.md | 55 +++++++ .../lib/cloudwatch-logs-action.ts | 9 +- .../lib/common-action-props.ts | 13 ++ .../@aws-cdk/aws-iot-actions/lib/index.ts | 2 + .../lib/s3-put-object-action.ts | 67 ++++++++ .../@aws-cdk/aws-iot-actions/package.json | 6 + .../integ.s3-put-object-action.expected.json | 86 ++++++++++ .../integ.s3-put-object-action.ts | 32 ++++ .../s3-put-object-action.test.ts | 148 ++++++++++++++++++ 11 files changed, 445 insertions(+), 7 deletions(-) create mode 100644 packages/@aws-cdk/aws-iot-actions/lib/common-action-props.ts create mode 100644 packages/@aws-cdk/aws-iot-actions/lib/s3-put-object-action.ts create mode 100644 packages/@aws-cdk/aws-iot-actions/test/s3-put-object/integ.s3-put-object-action.expected.json create mode 100644 packages/@aws-cdk/aws-iot-actions/test/s3-put-object/integ.s3-put-object-action.ts create mode 100644 packages/@aws-cdk/aws-iot-actions/test/s3-put-object/s3-put-object-action.test.ts diff --git a/package.json b/package.json index aea497b0788f1..2dcf60dfa02c0 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,8 @@ "@aws-cdk/assertions/fs-extra/**", "@aws-cdk/aws-amplify-alpha/yaml", "@aws-cdk/aws-amplify-alpha/yaml/**", + "@aws-cdk/aws-iot-actions-alpha/case", + "@aws-cdk/aws-iot-actions-alpha/case/**", "@aws-cdk/aws-amplify/yaml", "@aws-cdk/aws-amplify/yaml/**", "@aws-cdk/aws-codebuild/yaml", @@ -91,6 +93,8 @@ "@aws-cdk/aws-eks/yaml/**", "@aws-cdk/aws-events-targets/aws-sdk", "@aws-cdk/aws-events-targets/aws-sdk/**", + "@aws-cdk/aws-iot-actions/case", + "@aws-cdk/aws-iot-actions/case/**", "@aws-cdk/aws-s3-deployment/case", "@aws-cdk/aws-s3-deployment/case/**", "@aws-cdk/cloud-assembly-schema/jsonschema", diff --git a/packages/@aws-cdk/aws-iot-actions/NOTICE b/packages/@aws-cdk/aws-iot-actions/NOTICE index 5fc3826926b5b..39cd25bf899ae 100644 --- a/packages/@aws-cdk/aws-iot-actions/NOTICE +++ b/packages/@aws-cdk/aws-iot-actions/NOTICE @@ -1,2 +1,32 @@ AWS Cloud Development Kit (AWS CDK) Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +------------------------------------------------------------------------------- + +The AWS CDK includes the following third-party software/licensing: + +** case - https://www.npmjs.com/package/case +Copyright (c) 2013 Nathan Bubna + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +---------------- diff --git a/packages/@aws-cdk/aws-iot-actions/README.md b/packages/@aws-cdk/aws-iot-actions/README.md index b18182a80a9ad..f9043d86bd447 100644 --- a/packages/@aws-cdk/aws-iot-actions/README.md +++ b/packages/@aws-cdk/aws-iot-actions/README.md @@ -22,6 +22,8 @@ supported AWS Services. Instances of these classes should be passed to Currently supported are: - Invoke a Lambda function +- Put objects to a S3 bucket +- Put logs to CloudWatch Logs ## Invoke a Lambda function @@ -49,6 +51,59 @@ new iot.TopicRule(this, 'TopicRule', { }); ``` +## Put objects to a S3 bucket + +The code snippet below creates an AWS IoT Rule that put objects to a S3 bucket +when it is triggered. + +```ts +import * as iot from '@aws-cdk/aws-iot'; +import * as actions from '@aws-cdk/aws-iot-actions'; +import * as s3 from '@aws-cdk/aws-s3'; + +const bucket = new s3.Bucket(this, 'MyBucket'); + +new iot.TopicRule(this, 'TopicRule', { + sql: iot.IotSql.fromStringAsVer20160323("SELECT topic(2) as device_id FROM 'device/+/data'"), + actions: [new actions.S3PutObjectAction(bucket)], +}); +``` + +The property `key` of `S3PutObjectAction` is given the value `${topic()}/${timestamp()}` by default. This `${topic()}` +and `${timestamp()}` is called Substitution templates. For more information see +[this documentation](https://docs.aws.amazon.com/iot/latest/developerguide/iot-substitution-templates.html). +In above sample, `${topic()}` is replaced by a given MQTT topic as `device/001/data`. And `${timestamp()}` is replaced +by the number of the current timestamp in milliseconds as `1636289461203`. So if the MQTT broker receives an MQTT topic +`device/001/data` on `2021-11-07T00:00:00.000Z`, the S3 bucket object will be put to `device/001/data/1636243200000`. + +You can also set specific `key` as following: + +```ts +new iot.TopicRule(this, 'TopicRule', { + sql: iot.IotSql.fromStringAsVer20160323( + "SELECT topic(2) as device_id, year, month, day FROM 'device/+/data'", + ), + actions: [ + new actions.S3PutObjectAction(bucket, { + key: '${year}/${month}/${day}/${topic(2)}', + }), + ], +}); +``` + +If you wanna set access control to the S3 bucket object, you can specify `accessControl` as following: + +```ts +new iot.TopicRule(this, 'TopicRule', { + sql: iot.IotSql.fromStringAsVer20160323("SELECT * FROM 'device/+/data'"), + actions: [ + new actions.S3PutObjectAction(bucket, { + accessControl: s3.BucketAccessControl.PUBLIC_READ, + }), + ], +}); +``` + ## Put logs to CloudWatch Logs The code snippet below creates an AWS IoT Rule that put logs to CloudWatch Logs diff --git a/packages/@aws-cdk/aws-iot-actions/lib/cloudwatch-logs-action.ts b/packages/@aws-cdk/aws-iot-actions/lib/cloudwatch-logs-action.ts index dda14de887774..fb8f2779f32e7 100644 --- a/packages/@aws-cdk/aws-iot-actions/lib/cloudwatch-logs-action.ts +++ b/packages/@aws-cdk/aws-iot-actions/lib/cloudwatch-logs-action.ts @@ -1,18 +1,13 @@ import * as iam from '@aws-cdk/aws-iam'; import * as iot from '@aws-cdk/aws-iot'; import * as logs from '@aws-cdk/aws-logs'; +import { CommonActionProps } from './common-action-props'; import { singletonActionRole } from './private/role'; /** * Configuration properties of an action for CloudWatch Logs. */ -export interface CloudWatchLogsActionProps { - /** - * The IAM role that allows access to the CloudWatch log group. - * - * @default a new role will be created - */ - readonly role?: iam.IRole; +export interface CloudWatchLogsActionProps extends CommonActionProps { } /** diff --git a/packages/@aws-cdk/aws-iot-actions/lib/common-action-props.ts b/packages/@aws-cdk/aws-iot-actions/lib/common-action-props.ts new file mode 100644 index 0000000000000..5a9b52d8b5f27 --- /dev/null +++ b/packages/@aws-cdk/aws-iot-actions/lib/common-action-props.ts @@ -0,0 +1,13 @@ +import * as iam from '@aws-cdk/aws-iam'; + +/** + * Common properties shared by Actions it access to AWS service. + */ +export interface CommonActionProps { + /** + * The IAM role that allows access to AWS service. + * + * @default a new role will be created + */ + readonly role?: iam.IRole; +} diff --git a/packages/@aws-cdk/aws-iot-actions/lib/index.ts b/packages/@aws-cdk/aws-iot-actions/lib/index.ts index ef917fd0e2181..88521265228d4 100644 --- a/packages/@aws-cdk/aws-iot-actions/lib/index.ts +++ b/packages/@aws-cdk/aws-iot-actions/lib/index.ts @@ -1,2 +1,4 @@ export * from './cloudwatch-logs-action'; +export * from './common-action-props'; export * from './lambda-function-action'; +export * from './s3-put-object-action'; diff --git a/packages/@aws-cdk/aws-iot-actions/lib/s3-put-object-action.ts b/packages/@aws-cdk/aws-iot-actions/lib/s3-put-object-action.ts new file mode 100644 index 0000000000000..f690bf813a922 --- /dev/null +++ b/packages/@aws-cdk/aws-iot-actions/lib/s3-put-object-action.ts @@ -0,0 +1,67 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as iot from '@aws-cdk/aws-iot'; +import * as s3 from '@aws-cdk/aws-s3'; +import { kebab as toKebabCase } from 'case'; +import { CommonActionProps } from './common-action-props'; +import { singletonActionRole } from './private/role'; + +/** + * Configuration properties of an action for s3. + */ +export interface S3PutObjectActionProps extends CommonActionProps { + /** + * The Amazon S3 canned ACL that controls access to the object identified by the object key. + * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl + * + * @default None + */ + readonly accessControl?: s3.BucketAccessControl; + + /** + * The path to the file where the data is written. + * + * Supports substitution templates. + * @see https://docs.aws.amazon.com/iot/latest/developerguide/iot-substitution-templates.html + * + * @default '${topic()}/${timestamp()}' + */ + readonly key?: string; +} + +/** + * The action to write the data from an MQTT message to an Amazon S3 bucket. + */ +export class S3PutObjectAction implements iot.IAction { + private readonly accessControl?: string; + private readonly key?: string; + private readonly role?: iam.IRole; + + /** + * @param bucket The Amazon S3 bucket to which to write data. + * @param props Optional properties to not use default + */ + constructor(private readonly bucket: s3.IBucket, props: S3PutObjectActionProps = {}) { + this.accessControl = props.accessControl; + this.key = props.key; + this.role = props.role; + } + + bind(rule: iot.ITopicRule): iot.ActionConfig { + const role = this.role ?? singletonActionRole(rule); + role.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['s3:PutObject'], + resources: [this.bucket.arnForObjects('*')], + })); + + return { + configuration: { + s3: { + bucketName: this.bucket.bucketName, + cannedAcl: this.accessControl && toKebabCase(this.accessControl.toString()), + key: this.key ?? '${topic()}/${timestamp()}', + roleArn: role.roleArn, + }, + }, + }; + } +} diff --git a/packages/@aws-cdk/aws-iot-actions/package.json b/packages/@aws-cdk/aws-iot-actions/package.json index fb40db84577c3..ca5ca2bf1b1f5 100644 --- a/packages/@aws-cdk/aws-iot-actions/package.json +++ b/packages/@aws-cdk/aws-iot-actions/package.json @@ -83,7 +83,9 @@ "@aws-cdk/aws-iot": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", "@aws-cdk/aws-logs": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/core": "0.0.0", + "case": "1.6.3", "constructs": "^3.3.69" }, "homepage": "https://github.com/aws/aws-cdk", @@ -92,9 +94,13 @@ "@aws-cdk/aws-iot": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", "@aws-cdk/aws-logs": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/core": "0.0.0", "constructs": "^3.3.69" }, + "bundledDependencies": [ + "case" + ], "engines": { "node": ">= 10.13.0 <13 || >=13.7.0" }, diff --git a/packages/@aws-cdk/aws-iot-actions/test/s3-put-object/integ.s3-put-object-action.expected.json b/packages/@aws-cdk/aws-iot-actions/test/s3-put-object/integ.s3-put-object-action.expected.json new file mode 100644 index 0000000000000..4e530f04da2c1 --- /dev/null +++ b/packages/@aws-cdk/aws-iot-actions/test/s3-put-object/integ.s3-put-object-action.expected.json @@ -0,0 +1,86 @@ +{ + "Resources": { + "TopicRule40A4EA44": { + "Type": "AWS::IoT::TopicRule", + "Properties": { + "TopicRulePayload": { + "Actions": [ + { + "S3": { + "BucketName": { + "Ref": "MyBucketF68F3FF0" + }, + "CannedAcl": "bucket-owner-full-control", + "Key": "${year}/${month}/${day}/${topic(2)}", + "RoleArn": { + "Fn::GetAtt": [ + "TopicRuleTopicRuleActionRole246C4F77", + "Arn" + ] + } + } + } + ], + "AwsIotSqlVersion": "2016-03-23", + "Sql": "SELECT topic(2) as device_id, year, month, day FROM 'device/+/data'" + } + } + }, + "TopicRuleTopicRuleActionRole246C4F77": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "iot.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "TopicRuleTopicRuleActionRoleDefaultPolicy99ADD687": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:PutObject", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "MyBucketF68F3FF0", + "Arn" + ] + }, + "/*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "TopicRuleTopicRuleActionRoleDefaultPolicy99ADD687", + "Roles": [ + { + "Ref": "TopicRuleTopicRuleActionRole246C4F77" + } + ] + } + }, + "MyBucketF68F3FF0": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-iot-actions/test/s3-put-object/integ.s3-put-object-action.ts b/packages/@aws-cdk/aws-iot-actions/test/s3-put-object/integ.s3-put-object-action.ts new file mode 100644 index 0000000000000..9e100e0254eaf --- /dev/null +++ b/packages/@aws-cdk/aws-iot-actions/test/s3-put-object/integ.s3-put-object-action.ts @@ -0,0 +1,32 @@ +/// !cdk-integ pragma:ignore-assets +import * as iot from '@aws-cdk/aws-iot'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import * as actions from '../../lib'; + +const app = new cdk.App(); + +class TestStack extends cdk.Stack { + constructor(scope: cdk.App, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + const topicRule = new iot.TopicRule(this, 'TopicRule', { + sql: iot.IotSql.fromStringAsVer20160323( + "SELECT topic(2) as device_id, year, month, day FROM 'device/+/data'", + ), + }); + + const bucket = new s3.Bucket(this, 'MyBucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + topicRule.addAction( + new actions.S3PutObjectAction(bucket, { + key: '${year}/${month}/${day}/${topic(2)}', + accessControl: s3.BucketAccessControl.BUCKET_OWNER_FULL_CONTROL, + }), + ); + } +} + +new TestStack(app, 'test-stack'); +app.synth(); diff --git a/packages/@aws-cdk/aws-iot-actions/test/s3-put-object/s3-put-object-action.test.ts b/packages/@aws-cdk/aws-iot-actions/test/s3-put-object/s3-put-object-action.test.ts new file mode 100644 index 0000000000000..567bd59d05083 --- /dev/null +++ b/packages/@aws-cdk/aws-iot-actions/test/s3-put-object/s3-put-object-action.test.ts @@ -0,0 +1,148 @@ +import { Template, Match } from '@aws-cdk/assertions'; +import * as iam from '@aws-cdk/aws-iam'; +import * as iot from '@aws-cdk/aws-iot'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import * as actions from '../../lib'; + +test('Default s3 action', () => { + // GIVEN + const stack = new cdk.Stack(); + const topicRule = new iot.TopicRule(stack, 'MyTopicRule', { + sql: iot.IotSql.fromStringAsVer20160323("SELECT topic(2) as device_id FROM 'device/+/data'"), + }); + const bucket = s3.Bucket.fromBucketArn(stack, 'MyBucket', 'arn:aws:s3::123456789012:test-bucket'); + + // WHEN + topicRule.addAction( + new actions.S3PutObjectAction(bucket), + ); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IoT::TopicRule', { + TopicRulePayload: { + Actions: [ + { + S3: { + BucketName: 'test-bucket', + Key: '${topic()}/${timestamp()}', + RoleArn: { + 'Fn::GetAtt': ['MyTopicRuleTopicRuleActionRoleCE2D05DA', 'Arn'], + }, + }, + }, + ], + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Role', { + AssumeRolePolicyDocument: { + Statement: [ + { + Action: 'sts:AssumeRole', + Effect: 'Allow', + Principal: { + Service: 'iot.amazonaws.com', + }, + }, + ], + Version: '2012-10-17', + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: 's3:PutObject', + Effect: 'Allow', + Resource: 'arn:aws:s3::123456789012:test-bucket/*', + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'MyTopicRuleTopicRuleActionRoleDefaultPolicy54A701F7', + Roles: [ + { Ref: 'MyTopicRuleTopicRuleActionRoleCE2D05DA' }, + ], + }); +}); + +test('can set key of bucket', () => { + // GIVEN + const stack = new cdk.Stack(); + const topicRule = new iot.TopicRule(stack, 'MyTopicRule', { + sql: iot.IotSql.fromStringAsVer20160323("SELECT topic(2) as device_id FROM 'device/+/data'"), + }); + const bucket = s3.Bucket.fromBucketArn(stack, 'MyBucket', 'arn:aws:s3::123456789012:test-bucket'); + + // WHEN + topicRule.addAction( + new actions.S3PutObjectAction(bucket, { + key: 'test-key', + }), + ); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IoT::TopicRule', { + TopicRulePayload: { + Actions: [ + Match.objectLike({ S3: { Key: 'test-key' } }), + ], + }, + }); +}); + +test('can set canned ACL and it convert to kebab case', () => { + // GIVEN + const stack = new cdk.Stack(); + const topicRule = new iot.TopicRule(stack, 'MyTopicRule', { + sql: iot.IotSql.fromStringAsVer20160323("SELECT topic(2) as device_id FROM 'device/+/data'"), + }); + const bucket = s3.Bucket.fromBucketArn(stack, 'MyBucket', 'arn:aws:s3::123456789012:test-bucket'); + + // WHEN + topicRule.addAction( + new actions.S3PutObjectAction(bucket, { + accessControl: s3.BucketAccessControl.BUCKET_OWNER_FULL_CONTROL, + }), + ); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IoT::TopicRule', { + TopicRulePayload: { + Actions: [ + Match.objectLike({ S3: { CannedAcl: 'bucket-owner-full-control' } }), + ], + }, + }); +}); + +test('can set role', () => { + // GIVEN + const stack = new cdk.Stack(); + const topicRule = new iot.TopicRule(stack, 'MyTopicRule', { + sql: iot.IotSql.fromStringAsVer20160323("SELECT topic(2) as device_id FROM 'device/+/data'"), + }); + const bucket = s3.Bucket.fromBucketArn(stack, 'MyBucket', 'arn:aws:s3::123456789012:test-bucket'); + const role = iam.Role.fromRoleArn(stack, 'MyRole', 'arn:aws:iam::123456789012:role/ForTest'); + + // WHEN + topicRule.addAction( + new actions.S3PutObjectAction(bucket, { role }), + ); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IoT::TopicRule', { + TopicRulePayload: { + Actions: [ + Match.objectLike({ S3: { RoleArn: 'arn:aws:iam::123456789012:role/ForTest' } }), + ], + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyName: 'MyRolePolicy64AB00A5', + Roles: ['ForTest'], + }); +});