From 32ed2290f8efb27bf622998f98808ff18a8cdef1 Mon Sep 17 00:00:00 2001 From: Ben Chaimberg Date: Thu, 22 Jul 2021 13:32:25 -0700 Subject: [PATCH 01/15] feat(core): lazy mappings will only synthesize if keys are unresolved (#15617) This feature adds new static methods to the CfnMapping construct that allow the creation of "lazy" mappings. A lazy mapping will only create a Mappings section in the synthesized CFN template if some "find" operation on the mapping was not able to return a value since one or more of the lookup keys were unresolved. Usage: ```ts // Register the mapping as a lazy mapping. CfnMapping.registerLazyMap('UNIQUEMAPPINGID', { TopLevelKey: { SecondLevelKey: 'value', }, }); // Later, find a value from the mapping. Since the keys are both // resolved, this returns a resolved value and does not create a // CfnMapping. CfnMapping.findInLazyMap(scope, 'UNIQUEMAPPINGID', 'TopLevelKey', 'SecondLevelKey'); // > 'value' // If we try to find a value from the mapping using unresolved keys, a // CfnMapping is created and a Fn::FindInMap is returned. CfnMapping.findInLazyMap(scope, 'UNIQUEMAPPINGID', 'TopLevelKey', Aws.REGION); // > { Fn::FindInMap: [ 'UNIQUEMAPPINGID', 'TopLevelKey', { Ref: 'AWS::Region' } ] } ``` ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/core/README.md | 32 ++++ packages/@aws-cdk/core/lib/cfn-mapping.ts | 74 ++++++++-- packages/@aws-cdk/core/test/mappings.test.ts | 147 ++++++++++++++++++- packages/aws-cdk-lib/README.md | 34 ++++- 4 files changed, 269 insertions(+), 18 deletions(-) diff --git a/packages/@aws-cdk/core/README.md b/packages/@aws-cdk/core/README.md index 44fdda5893019..a6d4d2efc5a9b 100644 --- a/packages/@aws-cdk/core/README.md +++ b/packages/@aws-cdk/core/README.md @@ -775,6 +775,38 @@ Mappings: us-east-2: US East (Ohio) ``` +Mappings can also be synthesized "lazily"; lazy mappings will only render a "Mappings" +section in the synthesized CloudFormation template if some `findInMap` call is unable to +immediately return a concrete value due to one or both of the keys being unresolved tokens +(some value only available at deploy-time). + +For example, the following code will not produce anything in the "Mappings" section. The +call to `findInMap` will be able to resolve the value during synthesis and simply return +`'US East (Ohio)'`. + +```ts +const regionTable = new CfnMapping(this, 'RegionTable', { + mapping: { + regionName: { + 'us-east-1': 'US East (N. Virginia)', + 'us-east-2': 'US East (Ohio)', + }, + }, + lazy: true, +}); + +regionTable.findInMap('regionName', 'us-east-2'); +``` + +On the other hand, the following code will produce the "Mappings" section shown above, +since the second-level key is an unresolved token. The call to `findInMap` will return a +token that resolves to `{ Fn::FindInMap: [ 'RegionTable', 'regionName', { Ref: AWS::Region +} ] }`. + +```ts +regionTable.findInMap('regionName', Aws.REGION); +``` + [cfn-mappings]: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/mappings-section-structure.html ### Dynamic References diff --git a/packages/@aws-cdk/core/lib/cfn-mapping.ts b/packages/@aws-cdk/core/lib/cfn-mapping.ts index 196baf3aadb96..18b670a4ff927 100644 --- a/packages/@aws-cdk/core/lib/cfn-mapping.ts +++ b/packages/@aws-cdk/core/lib/cfn-mapping.ts @@ -1,8 +1,11 @@ import { Construct } from 'constructs'; +import { Annotations } from './annotations'; import { CfnRefElement } from './cfn-element'; import { Fn } from './cfn-fn'; import { Token } from './token'; +type Mapping = { [k1: string]: { [k2: string]: any } }; + export interface CfnMappingProps { /** * Mapping of key to a set of corresponding set of named values. @@ -14,18 +17,34 @@ export interface CfnMappingProps { * * @default - No mapping. */ - readonly mapping?: { [k1: string]: { [k2: string]: any } }; + readonly mapping?: Mapping; + + /* + * Synthesize this map in a lazy fashion. + * + * Lazy maps will only synthesize a mapping if a `findInMap` operation is unable to + * immediately return a value because one or both of the requested keys are unresolved + * tokens. In this case, `findInMap` will return a `Fn::FindInMap` CloudFormation + * intrinsic. + * + * @default false + */ + readonly lazy?: boolean; } /** * Represents a CloudFormation mapping. */ export class CfnMapping extends CfnRefElement { - private mapping: { [k1: string]: { [k2: string]: any } } = { }; + private mapping: Mapping; + private readonly lazy?: boolean; + private lazyRender = false; + private lazyInformed = false; constructor(scope: Construct, id: string, props: CfnMappingProps = {}) { super(scope, id); - this.mapping = props.mapping || { }; + this.mapping = props.mapping ?? { }; + this.lazy = props.lazy; } /** @@ -43,16 +62,25 @@ export class CfnMapping extends CfnRefElement { * @returns A reference to a value in the map based on the two keys. */ public findInMap(key1: string, key2: string): string { - // opportunistically check that the key exists (if the key does not contain tokens) - if (!Token.isUnresolved(key1) && !(key1 in this.mapping)) { - throw new Error(`Mapping doesn't contain top-level key '${key1}'`); + let fullyResolved = false; + if (!Token.isUnresolved(key1)) { + if (!(key1 in this.mapping)) { + throw new Error(`Mapping doesn't contain top-level key '${key1}'`); + } + if (!Token.isUnresolved(key2)) { + if (!(key2 in this.mapping[key1])) { + throw new Error(`Mapping doesn't contain second-level key '${key2}'`); + } + fullyResolved = true; + } } - - // opportunistically check that the second key exists (if the key does not contain tokens) - if (!Token.isUnresolved(key1) && !Token.isUnresolved(key2) && !(key2 in this.mapping[key1])) { - throw new Error(`Mapping doesn't contain second-level key '${key2}'`); + if (fullyResolved) { + if (this.lazy) { + return this.mapping[key1][key2]; + } + } else { + this.lazyRender = true; } - return Fn.findInMap(this.logicalId, key1, key2); } @@ -60,10 +88,24 @@ export class CfnMapping extends CfnRefElement { * @internal */ public _toCloudFormation(): object { - return { - Mappings: { - [this.logicalId]: this.mapping, - }, - }; + if (this.lazy === undefined && !this.lazyRender) { + this.informLazyUse(); + } + if (!this.lazy || (this.lazy && this.lazyRender)) { + return { + Mappings: { + [this.logicalId]: this.mapping, + }, + }; + } else { + return {}; + } + } + + private informLazyUse() { + if (!this.lazyInformed) { + Annotations.of(this).addInfo('Consider making this CfnMapping a lazy mapping by providing `lazy: true`: either no findInMap was called or every findInMap could be immediately resolved without using Fn::FindInMap'); + } + this.lazyInformed = true; } } diff --git a/packages/@aws-cdk/core/test/mappings.test.ts b/packages/@aws-cdk/core/test/mappings.test.ts index 7dd67fbcead45..4b2e15c4d4bf2 100644 --- a/packages/@aws-cdk/core/test/mappings.test.ts +++ b/packages/@aws-cdk/core/test/mappings.test.ts @@ -1,5 +1,7 @@ +import { ArtifactMetadataEntryType } from '@aws-cdk/cloud-assembly-schema'; +import { CloudAssembly } from '@aws-cdk/cx-api'; import { nodeunitShim, Test } from 'nodeunit-shim'; -import { Aws, CfnMapping, CfnResource, Fn, Stack } from '../lib'; +import { App, Aws, CfnMapping, CfnResource, Fn, Stack } from '../lib'; import { toCloudFormation } from './util'; nodeunitShim({ @@ -78,6 +80,13 @@ nodeunitShim({ const expected = { 'Fn::FindInMap': ['mapping', 'instanceCount', { Ref: 'AWS::Region' }] }; test.deepEqual(stack.resolve(v1), expected); test.deepEqual(stack.resolve(v2), expected); + test.deepEqual(toCloudFormation(stack).Mappings, { + mapping: { + instanceCount: { + 'us-east-1': 12, + }, + }, + }); test.done(); }, @@ -99,6 +108,13 @@ nodeunitShim({ test.deepEqual(stack.resolve(v), { 'Fn::FindInMap': ['mapping', { Ref: 'AWS::Region' }, 'size'], }); + test.deepEqual(toCloudFormation(stack).Mappings, { + mapping: { + 'us-east-1': { + size: 12, + }, + }, + }); test.done(); }, @@ -119,6 +135,135 @@ nodeunitShim({ // THEN test.throws(() => mapping.findInMap('not-found', Aws.REGION), /Mapping doesn't contain top-level key 'not-found'/); test.deepEqual(stack.resolve(v), { 'Fn::FindInMap': ['mapping', 'size', { Ref: 'AWS::Region' }] }); + test.deepEqual(toCloudFormation(stack).Mappings, { + mapping: { + size: { + 'us-east-1': 12, + }, + }, + }); test.done(); }, }); + +describe('lazy mapping', () => { + let stack: Stack; + let mapping: CfnMapping; + const backing = { + TopLevelKey1: { + SecondLevelKey1: [1, 2, 3], + SecondLevelKey2: { Hello: 'World' }, + }, + }; + + beforeEach(() => { + stack = new Stack(); + mapping = new CfnMapping(stack, 'Lazy Mapping', { + mapping: backing, + lazy: true, + }); + }); + + it('does not create CfnMapping if findInMap keys can be resolved', () => { + const retrievedValue = mapping.findInMap('TopLevelKey1', 'SecondLevelKey1'); + + expect(stack.resolve(retrievedValue)).toStrictEqual([1, 2, 3]); + expect(toCloudFormation(stack)).toStrictEqual({}); + }); + + it('does not create CfnMapping if findInMap is not called', () => { + expect(toCloudFormation(stack)).toStrictEqual({}); + }); + + it('creates CfnMapping if top level key cannot be resolved', () => { + const retrievedValue = mapping.findInMap(Aws.REGION, 'SecondLevelKey1'); + + expect(stack.resolve(retrievedValue)).toStrictEqual({ 'Fn::FindInMap': ['LazyMapping', { Ref: 'AWS::Region' }, 'SecondLevelKey1'] }); + expect(toCloudFormation(stack)).toStrictEqual({ + Mappings: { + LazyMapping: backing, + }, + }); + }); + + it('creates CfnMapping if second level key cannot be resolved', () => { + const retrievedValue = mapping.findInMap('TopLevelKey1', Aws.REGION); + + expect(stack.resolve(retrievedValue)).toStrictEqual({ 'Fn::FindInMap': ['LazyMapping', 'TopLevelKey1', { Ref: 'AWS::Region' }] }); + expect(toCloudFormation(stack)).toStrictEqual({ + Mappings: { + LazyMapping: backing, + }, + }); + }); + + it('throws if keys can be resolved but are not found in backing', () => { + expect(() => mapping.findInMap('NonExistentKey', 'SecondLevelKey1')) + .toThrowError(/Mapping doesn't contain top-level key .*/); + expect(() => mapping.findInMap('TopLevelKey1', 'NonExistentKey')) + .toThrowError(/Mapping doesn't contain second-level key .*/); + }); +}); + +describe('eager by default', () => { + const backing = { + TopLevelKey1: { + SecondLevelKey1: [1, 2, 3], + SecondLevelKey2: { Hello: 'World' }, + }, + }; + + let app: App; + let stack: Stack; + let mapping: CfnMapping; + + beforeEach(() => { + app = new App(); + stack = new Stack(app, 'Stack'); + mapping = new CfnMapping(stack, 'Lazy Mapping', { + mapping: backing, + }); + }); + + it('emits warning if no findInMap called', () => { + const assembly = app.synth(); + + expect(getInfoAnnotations(assembly)).toStrictEqual([{ + path: '/Stack/Lazy Mapping', + message: 'Consider making this CfnMapping a lazy mapping by providing `lazy: true`: either no findInMap was called or every findInMap could be immediately resolved without using Fn::FindInMap', + }]); + }); + + it('emits warning if every findInMap resolves immediately', () => { + mapping.findInMap('TopLevelKey1', 'SecondLevelKey1'); + + const assembly = app.synth(); + + expect(getInfoAnnotations(assembly)).toStrictEqual([{ + path: '/Stack/Lazy Mapping', + message: 'Consider making this CfnMapping a lazy mapping by providing `lazy: true`: either no findInMap was called or every findInMap could be immediately resolved without using Fn::FindInMap', + }]); + }); + + it('does not emit warning if a findInMap could not resolve immediately', () => { + mapping.findInMap('TopLevelKey1', Aws.REGION); + + const assembly = app.synth(); + + expect(getInfoAnnotations(assembly)).toStrictEqual([]); + }); +}); + +function getInfoAnnotations(casm: CloudAssembly) { + const result = new Array<{ path: string, message: string }>(); + for (const stack of Object.values(casm.manifest.artifacts ?? {})) { + for (const [path, md] of Object.entries(stack.metadata ?? {})) { + for (const x of md) { + if (x.type === ArtifactMetadataEntryType.INFO) { + result.push({ path, message: x.data as string }); + } + } + } + } + return result; +} diff --git a/packages/aws-cdk-lib/README.md b/packages/aws-cdk-lib/README.md index 445346a4a90bc..48d4cd65f93a5 100644 --- a/packages/aws-cdk-lib/README.md +++ b/packages/aws-cdk-lib/README.md @@ -808,6 +808,38 @@ Mappings: us-east-2: US East (Ohio) ``` +Mappings can also be synthesized "lazily"; lazy mappings will only render a "Mappings" +section in the synthesized CloudFormation template if some `findInMap` call is unable to +immediately return a concrete value due to one or both of the keys being unresolved tokens +(some value only available at deploy-time). + +For example, the following code will not produce anything in the "Mappings" section. The +call to `findInMap` will be able to resolve the value during synthesis and simply return +`'US East (Ohio)'`. + +```ts +const regionTable = new CfnMapping(this, 'RegionTable', { + mapping: { + regionName: { + 'us-east-1': 'US East (N. Virginia)', + 'us-east-2': 'US East (Ohio)', + }, + }, + lazy: true, +}); + +regionTable.findInMap('regionName', 'us-east-2'); +``` + +On the other hand, the following code will produce the "Mappings" section shown above, +since the second-level key is an unresolved token. The call to `findInMap` will return a +token that resolves to `{ Fn::FindInMap: [ 'RegionTable', 'regionName', { Ref: AWS::Region +} ] }`. + +```ts +regionTable.findInMap('regionName', Aws.REGION); +``` + [cfn-mappings]: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/mappings-section-structure.html ### Dynamic References @@ -947,4 +979,4 @@ It's possible to synthesize the project with more Resources than the allowed (or Set the context key `@aws-cdk/core:stackResourceLimit` with the proper value, being 0 for disable the limit of resources. - \ No newline at end of file + From 91cf79bc55ffd72b1c79e2218eb76921fbac32b4 Mon Sep 17 00:00:00 2001 From: Joel Cox Date: Fri, 23 Jul 2021 08:46:37 +1000 Subject: [PATCH 02/15] fix(elasticsearch): slow logs incorrectly disabled for Elasticsearch versions lower than 5.1 (#15714) Fixes #15532 As discussed in #15532, this error should not have applied to slow logs in the first place, as they're supported by all Elasticsearch versions. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../@aws-cdk/aws-elasticsearch/lib/domain.ts | 8 ++------ .../aws-elasticsearch/test/domain.test.ts | 16 ++-------------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/packages/@aws-cdk/aws-elasticsearch/lib/domain.ts b/packages/@aws-cdk/aws-elasticsearch/lib/domain.ts index 676d6984234c4..a29b530128c51 100644 --- a/packages/@aws-cdk/aws-elasticsearch/lib/domain.ts +++ b/packages/@aws-cdk/aws-elasticsearch/lib/domain.ts @@ -1415,12 +1415,8 @@ export class Domain extends DomainBase implements IDomain, ec2.IConnectable { // Validate feature support for the given Elasticsearch version, per // https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-features-by-version.html if (elasticsearchVersionNum < 5.1) { - if ( - props.logging?.slowIndexLogEnabled - || props.logging?.appLogEnabled - || props.logging?.slowSearchLogEnabled - ) { - throw new Error('Error and slow logs publishing requires Elasticsearch version 5.1 or later.'); + if (props.logging?.appLogEnabled) { + throw new Error('Error logs publishing requires Elasticsearch version 5.1 or later.'); } if (props.encryptionAtRest?.enabled) { throw new Error('Encryption of data at rest requires Elasticsearch version 5.1 or later.'); diff --git a/packages/@aws-cdk/aws-elasticsearch/test/domain.test.ts b/packages/@aws-cdk/aws-elasticsearch/test/domain.test.ts index 038eb1ebc82db..719341aa29408 100644 --- a/packages/@aws-cdk/aws-elasticsearch/test/domain.test.ts +++ b/packages/@aws-cdk/aws-elasticsearch/test/domain.test.ts @@ -1317,26 +1317,14 @@ describe('custom error responses', () => { })).toThrow(/Unknown Elasticsearch version: 5\.4/); }); - test('error when log publishing is enabled for elasticsearch version < 5.1', () => { - const error = /logs publishing requires Elasticsearch version 5.1 or later/; + test('error when error log publishing is enabled for elasticsearch version < 5.1', () => { + const error = /Error logs publishing requires Elasticsearch version 5.1 or later/; expect(() => new Domain(stack, 'Domain1', { version: ElasticsearchVersion.V2_3, logging: { appLogEnabled: true, }, })).toThrow(error); - expect(() => new Domain(stack, 'Domain2', { - version: ElasticsearchVersion.V1_5, - logging: { - slowSearchLogEnabled: true, - }, - })).toThrow(error); - expect(() => new Domain(stack, 'Domain3', { - version: ElasticsearchVersion.V1_5, - logging: { - slowIndexLogEnabled: true, - }, - })).toThrow(error); }); test('error when encryption at rest is enabled for elasticsearch version < 5.1', () => { From 1b5d525cef8ef4209074156c56077eebaa38d57c Mon Sep 17 00:00:00 2001 From: Madeline Kusters <80541297+madeline-k@users.noreply.github.com> Date: Thu, 22 Jul 2021 17:33:27 -0700 Subject: [PATCH 03/15] feat(aws-kinesisfirehose): DeliveryStream API and basic S3 destination (#15544) This PR implements the minimum DeliveryStream API and S3 destination. More features for DeliveryStream and the S3 destination will follow in future PRs. This work is being tracked in https://github.com/aws/aws-cdk/milestone/16 For more context, see: https://github.com/aws/aws-cdk/pull/15505 and the RFC: https://github.com/aws/aws-cdk-rfcs/pull/342 closes #10810, #15499 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-ec2/lib/connections.ts | 3 + packages/@aws-cdk/aws-ec2/package.json | 1 - .../.eslintrc.js | 3 + .../.gitignore | 19 + .../.npmignore | 28 ++ .../aws-kinesisfirehose-destinations/LICENSE | 201 +++++++++ .../aws-kinesisfirehose-destinations/NOTICE | 2 + .../README.md | 22 + .../jest.config.js | 2 + .../lib/common.ts | 32 ++ .../lib/index.ts | 2 + .../lib/private/helpers.ts | 66 +++ .../lib/s3-bucket.ts | 42 ++ .../package.json | 119 +++++ .../rosetta/default.ts-fixture | 11 + .../test/integ.s3-bucket.expected.json | 408 ++++++++++++++++++ .../test/integ.s3-bucket.ts | 28 ++ .../test/s3-bucket.test.ts | 223 ++++++++++ .../@aws-cdk/aws-kinesisfirehose/README.md | 236 +++++++++- .../lib/delivery-stream.ts | 266 ++++++++++++ .../aws-kinesisfirehose/lib/destination.ts | 40 ++ .../@aws-cdk/aws-kinesisfirehose/lib/index.ts | 3 + .../@aws-cdk/aws-kinesisfirehose/package.json | 16 +- .../rosetta/default.ts-fixture | 11 + .../rosetta/with-bucket.ts-fixture | 13 + .../rosetta/with-delivery-stream.ts-fixture | 12 + .../rosetta/with-destination.ts-fixture | 12 + .../test/delivery-stream.test.ts | 314 ++++++++++++++ .../test/integ.delivery-stream.expected.json | 194 +++++++++ .../test/integ.delivery-stream.ts | 37 ++ .../test/kinesisfirehose.test.ts | 6 - .../region-info/build-tools/fact-tables.ts | 29 ++ .../build-tools/generate-static-data.ts | 10 +- packages/@aws-cdk/region-info/lib/fact.ts | 5 + .../@aws-cdk/region-info/lib/region-info.ts | 7 + packages/aws-cdk-lib/package.json | 1 + packages/decdk/package.json | 1 + packages/monocdk/package.json | 1 + tools/pkglint/lib/rules.ts | 1 + 39 files changed, 2414 insertions(+), 13 deletions(-) create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/.eslintrc.js create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/.gitignore create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/.npmignore create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/LICENSE create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/NOTICE create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/jest.config.js create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/rosetta/default.ts-fixture create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-bucket.ts-fixture create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-delivery-stream.ts-fixture create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-destination.ts-fixture create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json create mode 100644 packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts delete mode 100644 packages/@aws-cdk/aws-kinesisfirehose/test/kinesisfirehose.test.ts diff --git a/packages/@aws-cdk/aws-ec2/lib/connections.ts b/packages/@aws-cdk/aws-ec2/lib/connections.ts index 0ecccea97fdb2..b68c04299cdf3 100644 --- a/packages/@aws-cdk/aws-ec2/lib/connections.ts +++ b/packages/@aws-cdk/aws-ec2/lib/connections.ts @@ -20,6 +20,9 @@ import { ISecurityGroup } from './security-group'; * An object that has a Connections object */ export interface IConnectable { + /** + * The network connections associated with this resource. + */ readonly connections: Connections; } diff --git a/packages/@aws-cdk/aws-ec2/package.json b/packages/@aws-cdk/aws-ec2/package.json index 985422e5da1fb..2f2dd7ab6e1c4 100644 --- a/packages/@aws-cdk/aws-ec2/package.json +++ b/packages/@aws-cdk/aws-ec2/package.json @@ -288,7 +288,6 @@ "props-default-doc:@aws-cdk/aws-ec2.AclPortRange.from", "props-default-doc:@aws-cdk/aws-ec2.AclPortRange.to", "docs-public-apis:@aws-cdk/aws-ec2.ConnectionRule", - "docs-public-apis:@aws-cdk/aws-ec2.IConnectable.connections", "docs-public-apis:@aws-cdk/aws-ec2.IInstance", "docs-public-apis:@aws-cdk/aws-ec2.IPrivateSubnet", "docs-public-apis:@aws-cdk/aws-ec2.IPublicSubnet", diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/.eslintrc.js b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.eslintrc.js new file mode 100644 index 0000000000000..61dd8dd001f63 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.eslintrc.js @@ -0,0 +1,3 @@ +const baseConfig = require('cdk-build-tools/config/eslintrc'); +baseConfig.parserOptions.project = __dirname + '/tsconfig.json'; +module.exports = baseConfig; diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/.gitignore b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.gitignore new file mode 100644 index 0000000000000..147448f7df4fe --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.gitignore @@ -0,0 +1,19 @@ +*.js +tsconfig.json +*.js.map +*.d.ts +*.generated.ts +dist +lib/generated/resources.ts +.jsii + +.LAST_BUILD +.nyc_output +coverage +nyc.config.js +.LAST_PACKAGE +*.snk +!.eslintrc.js +!jest.config.js + +junit.xml \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/.npmignore b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.npmignore new file mode 100644 index 0000000000000..aaabf1df59065 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/.npmignore @@ -0,0 +1,28 @@ +# Don't include original .ts files when doing `npm pack` +*.ts +!*.d.ts +coverage +.nyc_output +*.tgz + +dist +.LAST_PACKAGE +.LAST_BUILD +!*.js + +# Include .jsii +!.jsii + +*.snk + +*.tsbuildinfo + +tsconfig.json +.eslintrc.js +jest.config.js + +# exclude cdk artifacts +**/cdk.out +junit.xml +test/ +!*.lit.ts \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/LICENSE b/packages/@aws-cdk/aws-kinesisfirehose-destinations/LICENSE new file mode 100644 index 0000000000000..28e4bdcec77ec --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/NOTICE b/packages/@aws-cdk/aws-kinesisfirehose-destinations/NOTICE new file mode 100644 index 0000000000000..5fc3826926b5b --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/NOTICE @@ -0,0 +1,2 @@ +AWS Cloud Development Kit (AWS CDK) +Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md b/packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md new file mode 100644 index 0000000000000..03ef4657b3f78 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/README.md @@ -0,0 +1,22 @@ +# Amazon Kinesis Data Firehose Destinations Library + + +--- + +![cdk-constructs: Experimental](https://img.shields.io/badge/cdk--constructs-experimental-important.svg?style=for-the-badge) + +> The APIs of higher level constructs in this module are experimental and under active development. +> They are subject to non-backward compatible changes or removal in any future version. These are +> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be +> announced in the release notes. This means that while you may use them, you may need to update +> your source code when upgrading to a newer version of this package. + +--- + + + +This library provides constructs for adding destinations to a Amazon Kinesis Data Firehose +delivery stream. Destinations can be added by specifying the `destinations` prop when +defining a delivery stream. + +See [Amazon Kinesis Data Firehose module README](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-kinesisfirehose-readme.html) for usage examples. diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/jest.config.js b/packages/@aws-cdk/aws-kinesisfirehose-destinations/jest.config.js new file mode 100644 index 0000000000000..54e28beb9798b --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/jest.config.js @@ -0,0 +1,2 @@ +const baseConfig = require('cdk-build-tools/config/jest.config'); +module.exports = baseConfig; diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts new file mode 100644 index 0000000000000..3a97970d1ddbb --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/common.ts @@ -0,0 +1,32 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as logs from '@aws-cdk/aws-logs'; + +/** + * Generic properties for defining a delivery stream destination. + */ +export interface CommonDestinationProps { + /** + * If true, log errors when data transformation or data delivery fails. + * + * If `logGroup` is provided, this will be implicitly set to `true`. + * + * @default true - errors are logged. + */ + readonly logging?: boolean; + + /** + * The CloudWatch log group where log streams will be created to hold error logs. + * + * @default - if `logging` is set to `true`, a log group will be created for you. + */ + readonly logGroup?: logs.ILogGroup; + + /** + * The IAM role associated with this destination. + * + * Assumed by Kinesis Data Firehose to invoke processors and write to destinations + * + * @default - a role will be created with default permissions. + */ + readonly role?: iam.IRole; +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts new file mode 100644 index 0000000000000..7297f91a768c8 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/index.ts @@ -0,0 +1,2 @@ +export * from './common'; +export * from './s3-bucket'; diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts new file mode 100644 index 0000000000000..a2032c41914a0 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/private/helpers.ts @@ -0,0 +1,66 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as firehose from '@aws-cdk/aws-kinesisfirehose'; +import * as logs from '@aws-cdk/aws-logs'; +import * as cdk from '@aws-cdk/core'; +import { Construct, Node } from 'constructs'; + +export interface DestinationLoggingProps { + /** + * If true, log errors when data transformation or data delivery fails. + * + * If `logGroup` is provided, this will be implicitly set to `true`. + * + * @default true - errors are logged. + */ + readonly logging?: boolean; + + /** + * The CloudWatch log group where log streams will be created to hold error logs. + * + * @default - if `logging` is set to `true`, a log group will be created for you. + */ + readonly logGroup?: logs.ILogGroup; + + /** + * The IAM role associated with this destination. + */ + readonly role: iam.IRole; + + /** + * The ID of the stream that is created in the log group where logs will be placed. + * + * Must be unique within the log group, so should be different every time this function is called. + */ + readonly streamId: string; +} + +export interface DestinationLoggingOutput { + /** + * Logging options that will be injected into the destination configuration. + */ + readonly loggingOptions: firehose.CfnDeliveryStream.CloudWatchLoggingOptionsProperty; + + /** + * Resources that were created by the sub-config creator that must be deployed before the delivery stream is deployed. + */ + readonly dependables: cdk.IDependable[]; +} + +export function createLoggingOptions(scope: Construct, props: DestinationLoggingProps): DestinationLoggingOutput | undefined { + if (props.logging === false && props.logGroup) { + throw new Error('logging cannot be set to false when logGroup is provided'); + } + if (props.logging !== false || props.logGroup) { + const logGroup = props.logGroup ?? Node.of(scope).tryFindChild('LogGroup') as logs.ILogGroup ?? new logs.LogGroup(scope, 'LogGroup'); + const logGroupGrant = logGroup.grantWrite(props.role); + return { + loggingOptions: { + enabled: true, + logGroupName: logGroup.logGroupName, + logStreamName: logGroup.addStream(props.streamId).logStreamName, + }, + dependables: [logGroupGrant], + }; + } + return undefined; +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts new file mode 100644 index 0000000000000..ad3c0313ff061 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/lib/s3-bucket.ts @@ -0,0 +1,42 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as firehose from '@aws-cdk/aws-kinesisfirehose'; +import * as s3 from '@aws-cdk/aws-s3'; +import { Construct } from 'constructs'; +import { CommonDestinationProps } from './common'; +import { createLoggingOptions } from './private/helpers'; + +/** + * Props for defining an S3 destination of a Kinesis Data Firehose delivery stream. + */ +export interface S3BucketProps extends CommonDestinationProps { } + +/** + * An S3 bucket destination for data from a Kinesis Data Firehose delivery stream. + */ +export class S3Bucket implements firehose.IDestination { + constructor(private readonly bucket: s3.IBucket, private readonly props: S3BucketProps = {}) { } + + bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { + const role = this.props.role ?? new iam.Role(scope, 'S3 Destination Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), + }); + + const bucketGrant = this.bucket.grantReadWrite(role); + + const { loggingOptions, dependables: loggingDependables } = createLoggingOptions(scope, { + logging: this.props.logging, + logGroup: this.props.logGroup, + role, + streamId: 'S3Destination', + }) ?? {}; + + return { + extendedS3DestinationConfiguration: { + cloudWatchLoggingOptions: loggingOptions, + roleArn: role.roleArn, + bucketArn: this.bucket.bucketArn, + }, + dependables: [bucketGrant, ...(loggingDependables ?? [])], + }; + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json new file mode 100644 index 0000000000000..92cb7c8aa4332 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/package.json @@ -0,0 +1,119 @@ +{ + "name": "@aws-cdk/aws-kinesisfirehose-destinations", + "version": "0.0.0", + "description": "CDK Destinations Constructs for AWS Kinesis Firehose", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "jsii": { + "outdir": "dist", + "targets": { + "java": { + "package": "software.amazon.awscdk.services.kinesisfirehose.destinations", + "maven": { + "groupId": "software.amazon.awscdk", + "artifactId": "kinesisfirehose-destinations" + } + }, + "dotnet": { + "namespace": "Amazon.CDK.AWS.KinesisFirehose.Destinations", + "packageId": "Amazon.CDK.AWS.KinesisFirehose.Destinations", + "iconUrl": "https://raw.githubusercontent.com/aws/aws-cdk/master/logo/default-256-dark.png" + }, + "python": { + "distName": "aws-cdk.aws-kinesisfirehose-destinations", + "module": "aws_cdk.aws_kinesisfirehose_destinations", + "classifiers": [ + "Framework :: AWS CDK", + "Framework :: AWS CDK :: 1" + ] + } + }, + "projectReferences": true, + "metadata": { + "jsii": { + "rosetta": { + "strict": true + } + } + } + }, + "repository": { + "type": "git", + "url": "https://github.com/aws/aws-cdk.git", + "directory": "packages/@aws-cdk/aws-kinesisfirehose-destinations" + }, + "scripts": { + "build": "cdk-build", + "watch": "cdk-watch", + "lint": "cdk-lint", + "test": "cdk-test", + "integ": "cdk-integ", + "pkglint": "pkglint -f", + "package": "cdk-package", + "awslint": "cdk-awslint", + "build+test+package": "yarn build+test && yarn package", + "build+test": "yarn build && yarn test", + "compat": "cdk-compat", + "rosetta:extract": "yarn --silent jsii-rosetta extract", + "build+extract": "yarn build && yarn rosetta:extract", + "build+test+extract": "yarn build+test && yarn rosetta:extract" + }, + "keywords": [ + "aws", + "cdk", + "constructs", + "kinesisfirehose" + ], + "author": { + "name": "Amazon Web Services", + "url": "https://aws.amazon.com", + "organization": true + }, + "license": "Apache-2.0", + "devDependencies": { + "@types/jest": "^26.0.24", + "cdk-build-tools": "0.0.0", + "cdk-integ-tools": "0.0.0", + "cfn2ts": "0.0.0", + "jest": "^26.6.3", + "pkglint": "0.0.0", + "@aws-cdk/assert-internal": "0.0.0" + }, + "dependencies": { + "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", + "@aws-cdk/core": "0.0.0", + "constructs": "^3.3.69" + }, + "homepage": "https://github.com/aws/aws-cdk", + "peerDependencies": { + "@aws-cdk/aws-iam": "0.0.0", + "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", + "@aws-cdk/core": "0.0.0", + "constructs": "^3.3.69" + }, + "engines": { + "node": ">= 10.13.0 <13 || >=13.7.0" + }, + "stability": "experimental", + "maturity": "experimental", + "awslint": { + "exclude": [] + }, + "awscdkio": { + "announce": false + }, + "cdk-build": { + "jest": true, + "env": { + "AWSLINT_BASE_CONSTRUCT": true + } + }, + "publishConfig": { + "tag": "latest" + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture new file mode 100644 index 0000000000000..fe46e06908b34 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/rosetta/default.ts-fixture @@ -0,0 +1,11 @@ +// Fixture with packages imported, but nothing else +import { Construct } from '@aws-cdk/core'; +import { S3Bucket } from '@aws-cdk/aws-kinesisfirehose-destinations'; + +class Fixture extends Construct { + constructor(scope: Construct, id: string) { + super(scope, id); + + /// here + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json new file mode 100644 index 0000000000000..00bc62879e11e --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.expected.json @@ -0,0 +1,408 @@ +{ + "Resources": { + "Bucket83908E77": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "BucketPolicyE9A3008A": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "Bucket83908E77" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "BucketAutoDeleteObjectsCustomResourceBAFD23C2": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "Bucket83908E77" + } + }, + "DependsOn": [ + "BucketPolicyE9A3008A" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ] + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + } + ] + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3BucketF01ADF6B" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51" + } + ] + } + ] + } + ] + ] + } + }, + "Timeout": 900, + "MemorySize": 128, + "Handler": "__entrypoint__.handler", + "Role": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + }, + "Runtime": "nodejs12.x", + "Description": { + "Fn::Join": [ + "", + [ + "Lambda function for auto-deleting objects in ", + { + "Ref": "Bucket83908E77" + }, + " S3 bucket." + ] + ] + } + }, + "DependsOn": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + ] + }, + "LogGroupF5B46931": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "RetentionInDays": 731 + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "LogGroupS3Destination70CE1003": { + "Type": "AWS::Logs::LogStream", + "Properties": { + "LogGroupName": { + "Ref": "LogGroupF5B46931" + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "DeliveryStreamServiceRole964EEBCC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "DeliveryStreamS3DestinationRole500FC089": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:Abort*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "LogGroupF5B46931", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7", + "Roles": [ + { + "Ref": "DeliveryStreamS3DestinationRole500FC089" + } + ] + } + }, + "DeliveryStreamF6D5572D": { + "Type": "AWS::KinesisFirehose::DeliveryStream", + "Properties": { + "DeliveryStreamType": "DirectPut", + "ExtendedS3DestinationConfiguration": { + "BucketARN": { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "CloudWatchLoggingOptions": { + "Enabled": true, + "LogGroupName": { + "Ref": "LogGroupF5B46931" + }, + "LogStreamName": { + "Ref": "LogGroupS3Destination70CE1003" + } + }, + "RoleARN": { + "Fn::GetAtt": [ + "DeliveryStreamS3DestinationRole500FC089", + "Arn" + ] + } + } + }, + "DependsOn": [ + "DeliveryStreamS3DestinationRoleDefaultPolicy3015D8C7" + ] + } + }, + "Parameters": { + "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3BucketF01ADF6B": { + "Type": "String", + "Description": "S3 bucket for asset \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" + }, + "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1S3VersionKey6FC34F51": { + "Type": "String", + "Description": "S3 key for asset version \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" + }, + "AssetParameters1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1ArtifactHash9ECACDFD": { + "Type": "String", + "Description": "Artifact hash for asset \"1a8becf42c48697a059094af1e94aa6bc6df0512d30433db8c22618ca02dfca1\"" + } + }, + "Mappings": { + "awscdkawskinesisfirehoseCidrBlocks": { + "af-south-1": { + "FirehoseCidrBlock": "13.244.121.224/27" + }, + "ap-east-1": { + "FirehoseCidrBlock": "18.162.221.32/27" + }, + "ap-northeast-1": { + "FirehoseCidrBlock": "13.113.196.224/27" + }, + "ap-northeast-2": { + "FirehoseCidrBlock": "13.209.1.64/27" + }, + "ap-northeast-3": { + "FirehoseCidrBlock": "13.208.177.192/27" + }, + "ap-south-1": { + "FirehoseCidrBlock": "13.232.67.32/27" + }, + "ap-southeast-1": { + "FirehoseCidrBlock": "13.228.64.192/27" + }, + "ap-southeast-2": { + "FirehoseCidrBlock": "13.210.67.224/27" + }, + "ca-central-1": { + "FirehoseCidrBlock": "35.183.92.128/27" + }, + "cn-north-1": { + "FirehoseCidrBlock": "52.81.151.32/27" + }, + "cn-northwest-1": { + "FirehoseCidrBlock": "161.189.23.64/27" + }, + "eu-central-1": { + "FirehoseCidrBlock": "35.158.127.160/27" + }, + "eu-north-1": { + "FirehoseCidrBlock": "13.53.63.224/27" + }, + "eu-south-1": { + "FirehoseCidrBlock": "15.161.135.128/27" + }, + "eu-west-1": { + "FirehoseCidrBlock": "52.19.239.192/27" + }, + "eu-west-2": { + "FirehoseCidrBlock": "18.130.1.96/27" + }, + "eu-west-3": { + "FirehoseCidrBlock": "35.180.1.96/27" + }, + "me-south-1": { + "FirehoseCidrBlock": "15.185.91.0/27" + }, + "sa-east-1": { + "FirehoseCidrBlock": "18.228.1.128/27" + }, + "us-east-1": { + "FirehoseCidrBlock": "52.70.63.192/27" + }, + "us-east-2": { + "FirehoseCidrBlock": "13.58.135.96/27" + }, + "us-gov-east-1": { + "FirehoseCidrBlock": "18.253.138.96/27" + }, + "us-gov-west-1": { + "FirehoseCidrBlock": "52.61.204.160/27" + }, + "us-west-1": { + "FirehoseCidrBlock": "13.57.135.192/27" + }, + "us-west-2": { + "FirehoseCidrBlock": "52.89.255.224/27" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.ts new file mode 100644 index 0000000000000..222eaa6c0fb84 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/integ.s3-bucket.ts @@ -0,0 +1,28 @@ +#!/usr/bin/env node +import * as firehose from '@aws-cdk/aws-kinesisfirehose'; +import * as logs from '@aws-cdk/aws-logs'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import * as destinations from '../lib'; + +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-cdk-firehose-delivery-stream-s3-all-properties'); + +const bucket = new s3.Bucket(stack, 'Bucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, +}); + +const logGroup = new logs.LogGroup(stack, 'LogGroup', { + removalPolicy: cdk.RemovalPolicy.DESTROY, +}); + +new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [new destinations.S3Bucket(bucket, { + logging: true, + logGroup: logGroup, + })], +}); + +app.synth(); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts new file mode 100644 index 0000000000000..50e891b3091d1 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose-destinations/test/s3-bucket.test.ts @@ -0,0 +1,223 @@ +import '@aws-cdk/assert-internal/jest'; +import { ABSENT, MatchStyle, ResourcePart, anything, arrayWith } from '@aws-cdk/assert-internal'; +import * as iam from '@aws-cdk/aws-iam'; +import * as firehose from '@aws-cdk/aws-kinesisfirehose'; +import * as logs from '@aws-cdk/aws-logs'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import * as firehosedestinations from '../lib'; + +describe('S3 destination', () => { + let stack: cdk.Stack; + let bucket: s3.IBucket; + let destinationRole: iam.IRole; + + beforeEach(() => { + stack = new cdk.Stack(); + bucket = new s3.Bucket(stack, 'Bucket'); + destinationRole = new iam.Role(stack, 'Destination Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), + }); + }); + + it('provides defaults when no configuration is provided', () => { + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket, { role: destinationRole })], + }); + + expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + BucketARN: stack.resolve(bucket.bucketArn), + CloudWatchLoggingOptions: { + Enabled: true, + LogGroupName: anything(), + LogStreamName: anything(), + }, + RoleARN: stack.resolve(destinationRole.roleArn), + }, + }); + expect(stack).toHaveResource('AWS::Logs::LogGroup'); + expect(stack).toHaveResource('AWS::Logs::LogStream'); + }); + + it('creates a role when none is provided', () => { + + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket)], + }); + + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + RoleARN: { + 'Fn::GetAtt': [ + 'DeliveryStreamS3DestinationRoleD96B8345', + 'Arn', + ], + }, + }, + }); + expect(stack).toMatchTemplate({ + ['DeliveryStreamS3DestinationRoleD96B8345']: { + Type: 'AWS::IAM::Role', + }, + }, MatchStyle.SUPERSET); + }); + + it('grants read/write access to the bucket', () => { + const destination = new firehosedestinations.S3Bucket(bucket, { role: destinationRole }); + + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [destination], + }); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + Roles: [stack.resolve(destinationRole.roleName)], + PolicyDocument: { + Statement: [ + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + 's3:PutObject*', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + stack.resolve(bucket.bucketArn), + { 'Fn::Join': ['', [stack.resolve(bucket.bucketArn), '/*']] }, + ], + }, + ], + }, + }); + }); + + it('bucket and log group grants are depended on by delivery stream', () => { + const logGroup = logs.LogGroup.fromLogGroupName(stack, 'Log Group', 'evergreen'); + const destination = new firehosedestinations.S3Bucket(bucket, { role: destinationRole, logGroup }); + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [destination], + }); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyName: 'DestinationRoleDefaultPolicy1185C75D', + Roles: [stack.resolve(destinationRole.roleName)], + PolicyDocument: { + Statement: [ + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + 's3:PutObject*', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + stack.resolve(bucket.bucketArn), + { 'Fn::Join': ['', [stack.resolve(bucket.bucketArn), '/*']] }, + ], + }, + { + Action: [ + 'logs:CreateLogStream', + 'logs:PutLogEvents', + ], + Effect: 'Allow', + Resource: stack.resolve(logGroup.logGroupArn), + }, + ], + }, + }); + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + DependsOn: ['DestinationRoleDefaultPolicy1185C75D'], + }, ResourcePart.CompleteDefinition); + }); + + describe('logging', () => { + it('creates resources and configuration by default', () => { + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket)], + }); + + expect(stack).toHaveResource('AWS::Logs::LogGroup'); + expect(stack).toHaveResource('AWS::Logs::LogStream'); + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + CloudWatchLoggingOptions: { + Enabled: true, + LogGroupName: anything(), + LogStreamName: anything(), + }, + }, + }); + }); + + it('does not create resources or configuration if disabled', () => { + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket, { logging: false })], + }); + + expect(stack).not.toHaveResource('AWS::Logs::LogGroup'); + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + CloudWatchLoggingOptions: ABSENT, + }, + }); + }); + + it('uses provided log group', () => { + const logGroup = new logs.LogGroup(stack, 'Log Group'); + + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket, { logGroup })], + }); + + expect(stack).toCountResources('AWS::Logs::LogGroup', 1); + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + ExtendedS3DestinationConfiguration: { + CloudWatchLoggingOptions: { + Enabled: true, + LogGroupName: stack.resolve(logGroup.logGroupName), + LogStreamName: anything(), + }, + }, + }); + }); + + it('throws error if logging disabled but log group provided', () => { + const destination = new firehosedestinations.S3Bucket(bucket, { logging: false, logGroup: new logs.LogGroup(stack, 'Log Group') }); + + expect(() => new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [destination], + })).toThrowError('logging cannot be set to false when logGroup is provided'); + }); + + it('grants log group write permissions to destination role', () => { + const logGroup = new logs.LogGroup(stack, 'Log Group'); + + new firehose.DeliveryStream(stack, 'DeliveryStream', { + destinations: [new firehosedestinations.S3Bucket(bucket, { logGroup, role: destinationRole })], + }); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + Roles: [stack.resolve(destinationRole.roleName)], + PolicyDocument: { + Statement: arrayWith( + { + Action: [ + 'logs:CreateLogStream', + 'logs:PutLogEvents', + ], + Effect: 'Allow', + Resource: stack.resolve(logGroup.logGroupArn), + }, + ), + }, + }); + }); + }); +}); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/README.md b/packages/@aws-cdk/aws-kinesisfirehose/README.md index 9c4d9f96c6f36..5034ac4a0765f 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose/README.md @@ -9,8 +9,242 @@ > > [CFN Resources]: https://docs.aws.amazon.com/cdk/latest/guide/constructs.html#constructs_lib +![cdk-constructs: Experimental](https://img.shields.io/badge/cdk--constructs-experimental-important.svg?style=for-the-badge) + +> The APIs of higher level constructs in this module are experimental and under active development. +> They are subject to non-backward compatible changes or removal in any future version. These are +> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be +> announced in the release notes. This means that while you may use them, you may need to update +> your source code when upgrading to a newer version of this package. + --- -This module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) project. +[Amazon Kinesis Data Firehose](https://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html) +is a service for fully-managed delivery of real-time streaming data to storage services +such as Amazon S3, Amazon Redshift, Amazon Elasticsearch, Splunk, or any custom HTTP +endpoint or third-party services such as Datadog, Dynatrace, LogicMonitor, MongoDB, New +Relic, and Sumo Logic. + +Kinesis Data Firehose delivery streams are distinguished from Kinesis data streams in +their models of consumtpion. Whereas consumers read from a data stream by actively pulling +data from the stream, a delivery stream pushes data to its destination on a regular +cadence. This means that data streams are intended to have consumers that do on-demand +processing, like AWS Lambda or Amazon EC2. On the other hand, delivery streams are +intended to have destinations that are sources for offline processing and analytics, such +as Amazon S3 and Amazon Redshift. + +This module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) +project. It allows you to define Kinesis Data Firehose delivery streams. + +## Defining a Delivery Stream + +In order to define a Delivery Stream, you must specify a destination. An S3 bucket can be +used as a destination. More supported destinations are covered [below](#destinations). + +```ts +import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; +import * as s3 from '@aws-cdk/aws-s3'; + +const bucket = new s3.Bucket(this, 'Bucket'); +new DeliveryStream(this, 'Delivery Stream', { + destinations: [new destinations.S3Bucket(bucket)], +}); +``` + +The above example defines the following resources: + +- An S3 bucket +- A Kinesis Data Firehose delivery stream with Direct PUT as the source and CloudWatch + error logging turned on. +- An IAM role which gives the delivery stream permission to write to the S3 bucket. + +## Sources + +There are two main methods of sourcing input data: Kinesis Data Streams and via a "direct +put". This construct library currently only supports "direct put". See [#15500](https://github.com/aws/aws-cdk/issues/15500) to track the status of adding support for Kinesis Data Streams. + +See: [Sending Data to a Delivery Stream](https://docs.aws.amazon.com/firehose/latest/dev/basic-write.html) +in the *Kinesis Data Firehose Developer Guide*. + +### Direct Put + +Data must be provided via "direct put", ie., by using a `PutRecord` or `PutRecordBatch` API call. There are a number of ways of doing +so, such as: + +- Kinesis Agent: a standalone Java application that monitors and delivers files while + handling file rotation, checkpointing, and retries. See: [Writing to Kinesis Data Firehose Using Kinesis Agent](https://docs.aws.amazon.com/firehose/latest/dev/writing-with-agents.html) + in the *Kinesis Data Firehose Developer Guide*. +- AWS SDK: a general purpose solution that allows you to deliver data to a delivery stream + from anywhere using Java, .NET, Node.js, Python, or Ruby. See: [Writing to Kinesis Data Firehose Using the AWS SDK](https://docs.aws.amazon.com/firehose/latest/dev/writing-with-sdk.html) + in the *Kinesis Data Firehose Developer Guide*. +- CloudWatch Logs: subscribe to a log group and receive filtered log events directly into + a delivery stream. See: [logs-destinations](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-logs-destinations-readme.html). +- Eventbridge: add an event rule target to send events to a delivery stream based on the + rule filtering. See: [events-targets](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-events-targets-readme.html). +- SNS: add a subscription to send all notifications from the topic to a delivery + stream. See: [sns-subscriptions](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-sns-subscriptions-readme.html). +- IoT: add an action to an IoT rule to send various IoT information to a delivery stream + +## Destinations + +The following destinations are supported. See [kinesisfirehose-destinations](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-kinesisfirehose-destinations-readme.html) +for the implementations of these destinations. + +### S3 + +Defining a delivery stream with an S3 bucket destination: + +```ts +import * as s3 from '@aws-cdk/aws-s3'; +import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; + +const bucket = new s3.Bucket(this, 'Bucket'); + +const s3Destination = new destinations.S3Bucket(bucket); + +new DeliveryStream(this, 'Delivery Stream', { + destinations: [s3Destination], +}); +``` + +## Monitoring + +Kinesis Data Firehose is integrated with CloudWatch, so you can monitor the performance of +your delivery streams via logs and metrics. + +### Logs + +Kinesis Data Firehose will send logs to CloudWatch when data transformation or data +delivery fails. The CDK will enable logging by default and create a CloudWatch LogGroup +and LogStream for your Delivery Stream. + +You can provide a specific log group to specify where the CDK will create the log streams +where log events will be sent: + +```ts fixture=with-bucket +import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; +import * as logs from '@aws-cdk/aws-logs'; + +const logGroup = new logs.LogGroup(this, 'Log Group'); +const destination = new destinations.S3Bucket(bucket, { + logGroup: logGroup, +}); +new DeliveryStream(this, 'Delivery Stream', { + destinations: [destination], +}); +``` + +Logging can also be disabled: + +```ts fixture=with-bucket +import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; + +const destination = new destinations.S3Bucket(bucket, { + logging: false, +}); +new DeliveryStream(this, 'Delivery Stream', { + destinations: [destination], +}); +``` + +See: [Monitoring using CloudWatch Logs](https://docs.aws.amazon.com/firehose/latest/dev/monitoring-with-cloudwatch-logs.html) +in the *Kinesis Data Firehose Developer Guide*. + +## Specifying an IAM role + +The DeliveryStream class automatically creates IAM service roles with all the minimum +necessary permissions for Kinesis Data Firehose to access the resources referenced by your +delivery stream. One service role is created for the delivery stream that allows Kinesis +Data Firehose to read from a Kinesis data stream (if one is configured as the delivery +stream source) and for server-side encryption. Another service role is created for each +destination, which gives Kinesis Data Firehose write access to the destination resource, +as well as the ability to invoke data transformers and read schemas for record format +conversion. If you wish, you may specify your own IAM role for either the delivery stream +or the destination service role, or both. It must have the correct trust policy (it must +allow Kinesis Data Firehose to assume it) or delivery stream creation or data delivery +will fail. Other required permissions to destination resources, encryption keys, etc., +will be provided automatically. + +```ts fixture=with-bucket +import * as destinations from '@aws-cdk/aws-kinesisfirehose-destinations'; +import * as iam from '@aws-cdk/aws-iam'; + +// Create service roles for the delivery stream and destination. +// These can be used for other purposes and granted access to different resources. +// They must include the Kinesis Data Firehose service principal in their trust policies. +// Two separate roles are shown below, but the same role can be used for both purposes. +const deliveryStreamRole = new iam.Role(this, 'Delivery Stream Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), +}); +const destinationRole = new iam.Role(this, 'Destination Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), +}); + +// Specify the roles created above when defining the destination and delivery stream. +const destination = new destinations.S3Bucket(bucket, { role: destinationRole }); +new DeliveryStream(this, 'Delivery Stream', { + destinations: [destination], + role: deliveryStreamRole, +}); +``` + +See [Controlling Access](https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html) +in the *Kinesis Data Firehose Developer Guide*. + +## Granting application access to a delivery stream + +IAM roles, users or groups which need to be able to work with delivery streams should be +granted IAM permissions. + +Any object that implements the `IGrantable` interface (ie., has an associated principal) +can be granted permissions to a delivery stream by calling: + +- `grantPutRecords(principal)` - grants the principal the ability to put records onto the + delivery stream +- `grant(principal, ...actions)` - grants the principal permission to a custom set of + actions + +```ts fixture=with-delivery-stream +import * as iam from '@aws-cdk/aws-iam'; +const lambdaRole = new iam.Role(this, 'Role', { + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), +}); + +// Give the role permissions to write data to the delivery stream +deliveryStream.grantPutRecords(lambdaRole); +``` + +The following write permissions are provided to a service principal by the `grantPutRecords()` method: + +- `firehose:PutRecord` +- `firehose:PutRecordBatch` + +## Granting a delivery stream access to a resource + +Conversely to the above, Kinesis Data Firehose requires permissions in order for delivery +streams to interact with resources that you own. For example, if an S3 bucket is specified +as a destination of a delivery stream, the delivery stream must be granted permissions to +put and get objects from the bucket. When using the built-in AWS service destinations +found in the `@aws-cdk/aws-kinesisfirehose-destinations` module, the CDK grants the +permissions automatically. However, custom or third-party destinations may require custom +permissions. In this case, use the delivery stream as an `IGrantable`, as follows: + +```ts fixture=with-delivery-stream +import * as lambda from '@aws-cdk/aws-lambda'; + +const fn = new lambda.Function(this, 'Function', { + code: lambda.Code.fromInline('exports.handler = (event) => {}'), + runtime: lambda.Runtime.NODEJS_14_X, + handler: 'index.handler', +}); + +fn.grantInvoke(deliveryStream); +``` + +## Multiple destinations + +Though the delivery stream allows specifying an array of destinations, only one +destination per delivery stream is currently allowed. This limitation is enforced at CDK +synthesis time and will throw an error. diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts new file mode 100644 index 0000000000000..4968930808be6 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/delivery-stream.ts @@ -0,0 +1,266 @@ +import * as cloudwatch from '@aws-cdk/aws-cloudwatch'; +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as iam from '@aws-cdk/aws-iam'; +import * as cdk from '@aws-cdk/core'; +import { RegionInfo } from '@aws-cdk/region-info'; +import { Construct, Node } from 'constructs'; +import { IDestination } from './destination'; +import { CfnDeliveryStream } from './kinesisfirehose.generated'; + +const PUT_RECORD_ACTIONS = [ + 'firehose:PutRecord', + 'firehose:PutRecordBatch', +]; + +/** + * Represents a Kinesis Data Firehose delivery stream. + */ +export interface IDeliveryStream extends cdk.IResource, iam.IGrantable, ec2.IConnectable { + /** + * The ARN of the delivery stream. + * + * @attribute + */ + readonly deliveryStreamArn: string; + + /** + * The name of the delivery stream. + * + * @attribute + */ + readonly deliveryStreamName: string; + + /** + * Grant the `grantee` identity permissions to perform `actions`. + */ + grant(grantee: iam.IGrantable, ...actions: string[]): iam.Grant; + + /** + * Grant the `grantee` identity permissions to perform `firehose:PutRecord` and `firehose:PutRecordBatch` actions on this delivery stream. + */ + grantPutRecords(grantee: iam.IGrantable): iam.Grant; + + /** + * Return the given named metric for this delivery stream. + */ + metric(metricName: string, props?: cloudwatch.MetricOptions): cloudwatch.Metric; +} + +/** + * Base class for new and imported Kinesis Data Firehose delivery streams. + */ +abstract class DeliveryStreamBase extends cdk.Resource implements IDeliveryStream { + + public abstract readonly deliveryStreamName: string; + + public abstract readonly deliveryStreamArn: string; + + public abstract readonly grantPrincipal: iam.IPrincipal; + + /** + * Network connections between Kinesis Data Firehose and other resources, i.e. Redshift cluster. + */ + public readonly connections: ec2.Connections; + + constructor(scope: Construct, id: string, props: cdk.ResourceProps = {}) { + super(scope, id, props); + + this.connections = setConnections(this); + } + + public grant(grantee: iam.IGrantable, ...actions: string[]): iam.Grant { + return iam.Grant.addToPrincipal({ + resourceArns: [this.deliveryStreamArn], + grantee: grantee, + actions: actions, + }); + } + + public grantPutRecords(grantee: iam.IGrantable): iam.Grant { + return this.grant(grantee, ...PUT_RECORD_ACTIONS); + } + + public metric(metricName: string, props?: cloudwatch.MetricOptions): cloudwatch.Metric { + return new cloudwatch.Metric({ + namespace: 'AWS/Firehose', + metricName: metricName, + dimensions: { + DeliveryStreamName: this.deliveryStreamName, + }, + ...props, + }).attachTo(this); + } +} + +/** + * Properties for a new delivery stream. + */ +export interface DeliveryStreamProps { + /** + * The destinations that this delivery stream will deliver data to. + * + * Only a singleton array is supported at this time. + */ + readonly destinations: IDestination[]; + + /** + * A name for the delivery stream. + * + * @default - a name is generated by CloudFormation. + */ + readonly deliveryStreamName?: string; + + /** + * The IAM role associated with this delivery stream. + * + * Assumed by Kinesis Data Firehose to read from sources and encrypt data server-side. + * + * @default - a role will be created with default permissions. + */ + readonly role?: iam.IRole; +} + +/** + * A full specification of a delivery stream that can be used to import it fluently into the CDK application. + */ +export interface DeliveryStreamAttributes { + /** + * The ARN of the delivery stream. + * + * At least one of deliveryStreamArn and deliveryStreamName must be provided. + * + * @default - derived from `deliveryStreamName`. + */ + readonly deliveryStreamArn?: string; + + /** + * The name of the delivery stream + * + * At least one of deliveryStreamName and deliveryStreamArn must be provided. + * + * @default - derived from `deliveryStreamArn`. + */ + readonly deliveryStreamName?: string; + + /** + * The IAM role associated with this delivery stream. + * + * Assumed by Kinesis Data Firehose to read from sources and encrypt data server-side. + * + * @default - the imported stream cannot be granted access to other resources as an `iam.IGrantable`. + */ + readonly role?: iam.IRole; +} + +/** + * Create a Kinesis Data Firehose delivery stream + * + * @resource AWS::KinesisFirehose::DeliveryStream + */ +export class DeliveryStream extends DeliveryStreamBase { + /** + * Import an existing delivery stream from its name. + */ + static fromDeliveryStreamName(scope: Construct, id: string, deliveryStreamName: string): IDeliveryStream { + return this.fromDeliveryStreamAttributes(scope, id, { deliveryStreamName }); + } + + /** + * Import an existing delivery stream from its ARN. + */ + static fromDeliveryStreamArn(scope: Construct, id: string, deliveryStreamArn: string): IDeliveryStream { + return this.fromDeliveryStreamAttributes(scope, id, { deliveryStreamArn }); + } + + /** + * Import an existing delivery stream from its attributes. + */ + static fromDeliveryStreamAttributes(scope: Construct, id: string, attrs: DeliveryStreamAttributes): IDeliveryStream { + if (!attrs.deliveryStreamName && !attrs.deliveryStreamArn) { + throw new Error('Either deliveryStreamName or deliveryStreamArn must be provided in DeliveryStreamAttributes'); + } + const deliveryStreamName = attrs.deliveryStreamName ?? + cdk.Stack.of(scope).splitArn(attrs.deliveryStreamArn!, cdk.ArnFormat.SLASH_RESOURCE_NAME).resourceName; + + if (!deliveryStreamName) { + throw new Error(`No delivery stream name found in ARN: '${attrs.deliveryStreamArn}'`); + } + const deliveryStreamArn = attrs.deliveryStreamArn ?? cdk.Stack.of(scope).formatArn({ + service: 'firehose', + resource: 'deliverystream', + resourceName: attrs.deliveryStreamName, + arnFormat: cdk.ArnFormat.SLASH_RESOURCE_NAME, + }); + class Import extends DeliveryStreamBase { + public readonly deliveryStreamName = deliveryStreamName!; + public readonly deliveryStreamArn = deliveryStreamArn; + public readonly grantPrincipal = attrs.role ?? new iam.UnknownPrincipal({ resource: this }); + } + return new Import(scope, id); + } + + readonly deliveryStreamName: string; + + readonly deliveryStreamArn: string; + + readonly grantPrincipal: iam.IPrincipal; + + constructor(scope: Construct, id: string, props: DeliveryStreamProps) { + super(scope, id, { + physicalName: props.deliveryStreamName, + }); + + if (props.destinations.length !== 1) { + throw new Error(`Only one destination is allowed per delivery stream, given ${props.destinations.length}`); + } + + const role = props.role ?? new iam.Role(this, 'Service Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), + }); + this.grantPrincipal = role; + + const destinationConfig = props.destinations[0].bind(this, {}); + + const resource = new CfnDeliveryStream(this, 'Resource', { + deliveryStreamName: props.deliveryStreamName, + deliveryStreamType: 'DirectPut', + ...destinationConfig, + }); + destinationConfig.dependables?.forEach(dependable => resource.node.addDependency(dependable)); + + this.deliveryStreamArn = this.getResourceArnAttribute(resource.attrArn, { + service: 'kinesis', + resource: 'deliverystream', + resourceName: this.physicalName, + }); + this.deliveryStreamName = this.getResourceNameAttribute(resource.ref); + } +} + +function setConnections(scope: Construct) { + const stack = cdk.Stack.of(scope); + + const mappingId = '@aws-cdk/aws-kinesisfirehose.CidrBlocks'; + let cfnMapping = Node.of(stack).tryFindChild(mappingId) as cdk.CfnMapping; + + if (!cfnMapping) { + const mapping: {[region: string]: { FirehoseCidrBlock: string }} = {}; + RegionInfo.regions.forEach((regionInfo) => { + if (regionInfo.firehoseCidrBlock) { + mapping[regionInfo.name] = { + FirehoseCidrBlock: regionInfo.firehoseCidrBlock, + }; + } + }); + cfnMapping = new cdk.CfnMapping(stack, mappingId, { + mapping, + lazy: true, + }); + } + + const cidrBlock = cfnMapping.findInMap(stack.region, 'FirehoseCidrBlock'); + + return new ec2.Connections({ + peer: ec2.Peer.ipv4(cidrBlock), + }); +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts new file mode 100644 index 0000000000000..eb277babcdebb --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/destination.ts @@ -0,0 +1,40 @@ +import * as cdk from '@aws-cdk/core'; +import { Construct } from 'constructs'; +import { CfnDeliveryStream } from './kinesisfirehose.generated'; + +/** + * A Kinesis Data Firehose delivery stream destination configuration. + */ +export interface DestinationConfig { + /** + * S3 destination configuration properties. + * + * @default - S3 destination is not used. + */ + readonly extendedS3DestinationConfiguration?: CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty; + + /** + * Any resources that were created by the destination when binding it to the stack that must be deployed before the delivery stream is deployed. + * + * @default [] + */ + readonly dependables?: cdk.IDependable[]; +} + +/** + * Options when binding a destination to a delivery stream. + */ +export interface DestinationBindOptions { +} + +/** + * A Kinesis Data Firehose delivery stream destination. + */ +export interface IDestination { + /** + * Binds this destination to the Kinesis Data Firehose delivery stream. + * + * Implementers should use this method to bind resources to the stack and initialize values using the provided stream. + */ + bind(scope: Construct, options: DestinationBindOptions): DestinationConfig; +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/lib/index.ts b/packages/@aws-cdk/aws-kinesisfirehose/lib/index.ts index dd7beef14d159..3eddb6dec468e 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/lib/index.ts +++ b/packages/@aws-cdk/aws-kinesisfirehose/lib/index.ts @@ -1,2 +1,5 @@ +export * from './delivery-stream'; +export * from './destination'; + // AWS::KinesisFirehose CloudFormation Resources: export * from './kinesisfirehose.generated'; diff --git a/packages/@aws-cdk/aws-kinesisfirehose/package.json b/packages/@aws-cdk/aws-kinesisfirehose/package.json index fc949c6290e88..d6651f8d08c62 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose/package.json +++ b/packages/@aws-cdk/aws-kinesisfirehose/package.json @@ -73,26 +73,36 @@ }, "license": "Apache-2.0", "devDependencies": { + "@aws-cdk/assert-internal": "0.0.0", + "@aws-cdk/aws-s3": "0.0.0", "@types/jest": "^26.0.24", "cdk-build-tools": "0.0.0", + "cdk-integ-tools": "0.0.0", "cfn2ts": "0.0.0", - "pkglint": "0.0.0", - "@aws-cdk/assert-internal": "0.0.0" + "pkglint": "0.0.0" }, "dependencies": { + "@aws-cdk/aws-cloudwatch": "0.0.0", + "@aws-cdk/aws-ec2": "0.0.0", + "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/core": "0.0.0", + "@aws-cdk/region-info": "0.0.0", "constructs": "^3.3.69" }, "homepage": "https://github.com/aws/aws-cdk", "peerDependencies": { + "@aws-cdk/aws-cloudwatch": "0.0.0", + "@aws-cdk/aws-ec2": "0.0.0", + "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/core": "0.0.0", + "@aws-cdk/region-info": "0.0.0", "constructs": "^3.3.69" }, "engines": { "node": ">= 10.13.0 <13 || >=13.7.0" }, "stability": "experimental", - "maturity": "cfn-only", + "maturity": "experimental", "awscdkio": { "announce": false }, diff --git a/packages/@aws-cdk/aws-kinesisfirehose/rosetta/default.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/default.ts-fixture new file mode 100644 index 0000000000000..8a68efc25aa8e --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/default.ts-fixture @@ -0,0 +1,11 @@ +// Fixture with packages imported, but nothing else +import { Construct, Stack } from '@aws-cdk/core'; +import { DeliveryStream, DestinationBindOptions, DestinationConfig, IDestination } from '@aws-cdk/aws-kinesisfirehose'; + +class Fixture extends Stack { + constructor(scope: Construct, id: string) { + super(scope, id); + + /// here + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-bucket.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-bucket.ts-fixture new file mode 100644 index 0000000000000..d0851cff49639 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-bucket.ts-fixture @@ -0,0 +1,13 @@ +// Fixture with a bucket already created +import { Construct, Stack } from '@aws-cdk/core'; +import { DeliveryStream, DestinationBindOptions, DestinationConfig, IDestination } from '@aws-cdk/aws-kinesisfirehose'; +import * as s3 from '@aws-cdk/aws-s3'; +declare const bucket: s3.Bucket; + +class Fixture extends Stack { + constructor(scope: Construct, id: string) { + super(scope, id); + + /// here + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-delivery-stream.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-delivery-stream.ts-fixture new file mode 100644 index 0000000000000..c7b75b20d2c1b --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-delivery-stream.ts-fixture @@ -0,0 +1,12 @@ +// Fixture with a delivery stream already created +import { Construct, Stack } from '@aws-cdk/core'; +import { DeliveryStream, DestinationBindOptions, DestinationConfig, IDestination } from '@aws-cdk/aws-kinesisfirehose'; +declare const deliveryStream: DeliveryStream; + +class Fixture extends Stack { + constructor(scope: Construct, id: string) { + super(scope, id); + + /// here + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-destination.ts-fixture b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-destination.ts-fixture new file mode 100644 index 0000000000000..37d78bf7a43d3 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/rosetta/with-destination.ts-fixture @@ -0,0 +1,12 @@ +// Fixture with a destination already created +import { Construct, Stack } from '@aws-cdk/core'; +import { DeliveryStream, DestinationBindOptions, DestinationConfig, IDestination } from '@aws-cdk/aws-kinesisfirehose'; +declare const destination: IDestination; + +class Fixture extends Stack { + constructor(scope: Construct, id: string) { + super(scope, id); + + /// here + } +} diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts new file mode 100644 index 0000000000000..bb9c3ba744a2f --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/delivery-stream.test.ts @@ -0,0 +1,314 @@ +import '@aws-cdk/assert-internal/jest'; +import { ABSENT, ResourcePart, SynthUtils, anything } from '@aws-cdk/assert-internal'; +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as iam from '@aws-cdk/aws-iam'; +import * as cdk from '@aws-cdk/core'; +import { Construct, Node } from 'constructs'; +import * as firehose from '../lib'; + +describe('delivery stream', () => { + let stack: cdk.Stack; + let dependable: Construct; + let mockS3Destination: firehose.IDestination; + + const bucketArn = 'arn:aws:s3:::my-bucket'; + const roleArn = 'arn:aws:iam::111122223333:role/my-role'; + + beforeEach(() => { + stack = new cdk.Stack(); + mockS3Destination = { + bind(scope: Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { + dependable = new class extends cdk.Construct { + constructor(depScope: Construct, id: string) { + super(depScope, id); + new cdk.CfnResource(this, 'Resource', { type: 'CDK::Dummy' }); + } + }(scope, 'Dummy Dep'); + return { + extendedS3DestinationConfiguration: { + bucketArn: bucketArn, + roleArn: roleArn, + }, + dependables: [dependable], + }; + }, + }; + }); + + test('creates stream with default values', () => { + new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { + DeliveryStreamEncryptionConfigurationInput: ABSENT, + DeliveryStreamName: ABSENT, + DeliveryStreamType: 'DirectPut', + KinesisStreamSourceConfiguration: ABSENT, + ExtendedS3DestinationConfiguration: { + BucketARN: bucketArn, + RoleARN: roleArn, + }, + }); + }); + + test('provided role is set as grant principal', () => { + const role = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), + }); + + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + role: role, + }); + + expect(deliveryStream.grantPrincipal).toBe(role); + }); + + test('not providing role creates one', () => { + new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + expect(stack).toHaveResourceLike('AWS::IAM::Role', { + AssumeRolePolicyDocument: { + Statement: [ + { + Principal: { + Service: 'firehose.amazonaws.com', + }, + }, + ], + }, + }); + }); + + test('grant provides access to stream', () => { + const role = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + }); + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + deliveryStream.grant(role, 'firehose:PutRecord'); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: 'firehose:PutRecord', + Resource: stack.resolve(deliveryStream.deliveryStreamArn), + }, + ], + }, + Roles: [stack.resolve(role.roleName)], + }); + }); + + test('grantPutRecords provides PutRecord* access to stream', () => { + const role = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + }); + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + deliveryStream.grantPutRecords(role); + + expect(stack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'firehose:PutRecord', + 'firehose:PutRecordBatch', + ], + Resource: stack.resolve(deliveryStream.deliveryStreamArn), + }, + ], + }, + Roles: [stack.resolve(role.roleName)], + }); + }); + + test('dependables supplied from destination are depended on by just the CFN resource', () => { + const dependableId = stack.resolve((Node.of(dependable).defaultChild as cdk.CfnResource).logicalId); + + new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + expect(stack).toHaveResourceLike('AWS::KinesisFirehose::DeliveryStream', { + DependsOn: [dependableId], + }, ResourcePart.CompleteDefinition); + expect(stack).toHaveResourceLike('AWS::IAM::Role', { + DependsOn: ABSENT, + }, ResourcePart.CompleteDefinition); + }); + + test('supplying 0 or multiple destinations throws', () => { + expect(() => new firehose.DeliveryStream(stack, 'No Destinations', { + destinations: [], + })).toThrowError(/Only one destination is allowed per delivery stream/); + expect(() => new firehose.DeliveryStream(stack, 'Too Many Destinations', { + destinations: [mockS3Destination, mockS3Destination], + })).toThrowError(/Only one destination is allowed per delivery stream/); + }); + + describe('metric methods provide a Metric with configured and attached properties', () => { + beforeEach(() => { + stack = new cdk.Stack(undefined, undefined, { env: { account: '000000000000', region: 'us-west-1' } }); + }); + + test('metric', () => { + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + const metric = deliveryStream.metric('IncomingRecords'); + + expect(metric).toMatchObject({ + account: stack.account, + region: stack.region, + namespace: 'AWS/Firehose', + metricName: 'IncomingRecords', + dimensions: { + DeliveryStreamName: deliveryStream.deliveryStreamName, + }, + }); + }); + }); + + test('allows connections for Firehose IP addresses using map when region not specified', () => { + const vpc = new ec2.Vpc(stack, 'VPC'); + const securityGroup = new ec2.SecurityGroup(stack, 'Security Group', { vpc }); + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + securityGroup.connections.allowFrom(deliveryStream, ec2.Port.allTcp()); + + expect(stack).toHaveResourceLike('AWS::EC2::SecurityGroup', { + SecurityGroupIngress: [ + { + CidrIp: { + 'Fn::FindInMap': [ + anything(), + { + Ref: 'AWS::Region', + }, + 'FirehoseCidrBlock', + ], + }, + }, + ], + }); + }); + + test('allows connections for Firehose IP addresses using literal when region specified', () => { + stack = new cdk.Stack(undefined, undefined, { env: { region: 'us-west-1' } }); + const vpc = new ec2.Vpc(stack, 'VPC'); + const securityGroup = new ec2.SecurityGroup(stack, 'Security Group', { vpc }); + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + securityGroup.connections.allowFrom(deliveryStream, ec2.Port.allTcp()); + + expect(stack).toHaveResourceLike('AWS::EC2::SecurityGroup', { + SecurityGroupIngress: [ + { + CidrIp: '13.57.135.192/27', + }, + ], + }); + }); + + test('only adds one Firehose IP address mapping to stack even if multiple delivery streams defined', () => { + new firehose.DeliveryStream(stack, 'Delivery Stream 1', { + destinations: [mockS3Destination], + }); + new firehose.DeliveryStream(stack, 'Delivery Stream 2', { + destinations: [mockS3Destination], + }); + + expect(Object.keys(SynthUtils.toCloudFormation(stack).Mappings).length).toBe(1); + }); + + test('can add tags', () => { + const deliveryStream = new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], + }); + + cdk.Tags.of(deliveryStream).add('tagKey', 'tagValue'); + + expect(stack).toHaveResource('AWS::KinesisFirehose::DeliveryStream', { + Tags: [ + { + Key: 'tagKey', + Value: 'tagValue', + }, + ], + }); + }); + + describe('importing', () => { + test('from name', () => { + const deliveryStream = firehose.DeliveryStream.fromDeliveryStreamName(stack, 'DeliveryStream', 'mydeliverystream'); + + expect(deliveryStream.deliveryStreamName).toBe('mydeliverystream'); + expect(stack.resolve(deliveryStream.deliveryStreamArn)).toStrictEqual({ + 'Fn::Join': ['', ['arn:', stack.resolve(stack.partition), ':firehose:', stack.resolve(stack.region), ':', stack.resolve(stack.account), ':deliverystream/mydeliverystream']], + }); + expect(deliveryStream.grantPrincipal).toBeInstanceOf(iam.UnknownPrincipal); + }); + + test('from ARN', () => { + const deliveryStream = firehose.DeliveryStream.fromDeliveryStreamArn(stack, 'DeliveryStream', 'arn:aws:firehose:xx-west-1:111122223333:deliverystream/mydeliverystream'); + + expect(deliveryStream.deliveryStreamName).toBe('mydeliverystream'); + expect(deliveryStream.deliveryStreamArn).toBe('arn:aws:firehose:xx-west-1:111122223333:deliverystream/mydeliverystream'); + expect(deliveryStream.grantPrincipal).toBeInstanceOf(iam.UnknownPrincipal); + }); + + test('from attributes (just name)', () => { + const deliveryStream = firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', { deliveryStreamName: 'mydeliverystream' }); + + expect(deliveryStream.deliveryStreamName).toBe('mydeliverystream'); + expect(stack.resolve(deliveryStream.deliveryStreamArn)).toStrictEqual({ + 'Fn::Join': ['', ['arn:', stack.resolve(stack.partition), ':firehose:', stack.resolve(stack.region), ':', stack.resolve(stack.account), ':deliverystream/mydeliverystream']], + }); + expect(deliveryStream.grantPrincipal).toBeInstanceOf(iam.UnknownPrincipal); + }); + + test('from attributes (just ARN)', () => { + const deliveryStream = firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', { deliveryStreamArn: 'arn:aws:firehose:xx-west-1:111122223333:deliverystream/mydeliverystream' }); + + expect(deliveryStream.deliveryStreamName).toBe('mydeliverystream'); + expect(deliveryStream.deliveryStreamArn).toBe('arn:aws:firehose:xx-west-1:111122223333:deliverystream/mydeliverystream'); + expect(deliveryStream.grantPrincipal).toBeInstanceOf(iam.UnknownPrincipal); + }); + + test('from attributes (with role)', () => { + const role = iam.Role.fromRoleArn(stack, 'Delivery Stream Role', 'arn:aws:iam::111122223333:role/DeliveryStreamRole'); + const deliveryStream = firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', { deliveryStreamName: 'mydeliverystream', role }); + + expect(deliveryStream.deliveryStreamName).toBe('mydeliverystream'); + expect(stack.resolve(deliveryStream.deliveryStreamArn)).toStrictEqual({ + 'Fn::Join': ['', ['arn:', stack.resolve(stack.partition), ':firehose:', stack.resolve(stack.region), ':', stack.resolve(stack.account), ':deliverystream/mydeliverystream']], + }); + expect(deliveryStream.grantPrincipal).toBe(role); + }); + + test('throws when malformatted ARN', () => { + expect(() => firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', { deliveryStreamArn: 'arn:aws:firehose:xx-west-1:111122223333:deliverystream/' })) + .toThrowError("No delivery stream name found in ARN: 'arn:aws:firehose:xx-west-1:111122223333:deliverystream/'"); + }); + + test('throws when without name or ARN', () => { + expect(() => firehose.DeliveryStream.fromDeliveryStreamAttributes(stack, 'DeliveryStream', {})) + .toThrowError('Either deliveryStreamName or deliveryStreamArn must be provided in DeliveryStreamAttributes'); + }); + }); +}); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json new file mode 100644 index 0000000000000..f9e785a3def9e --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.expected.json @@ -0,0 +1,194 @@ +{ + "Resources": { + "Bucket83908E77": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "Role1ABCC5F0": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "RoleDefaultPolicy5FFB7DAB": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:Abort*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "RoleDefaultPolicy5FFB7DAB", + "Roles": [ + { + "Ref": "Role1ABCC5F0" + } + ] + } + }, + "DeliveryStreamServiceRole964EEBCC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "DeliveryStreamF6D5572D": { + "Type": "AWS::KinesisFirehose::DeliveryStream", + "Properties": { + "DeliveryStreamType": "DirectPut", + "ExtendedS3DestinationConfiguration": { + "BucketARN": { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "RoleARN": { + "Fn::GetAtt": [ + "Role1ABCC5F0", + "Arn" + ] + } + } + }, + "DependsOn": [ + "RoleDefaultPolicy5FFB7DAB" + ] + } + }, + "Mappings": { + "awscdkawskinesisfirehoseCidrBlocks": { + "af-south-1": { + "FirehoseCidrBlock": "13.244.121.224/27" + }, + "ap-east-1": { + "FirehoseCidrBlock": "18.162.221.32/27" + }, + "ap-northeast-1": { + "FirehoseCidrBlock": "13.113.196.224/27" + }, + "ap-northeast-2": { + "FirehoseCidrBlock": "13.209.1.64/27" + }, + "ap-northeast-3": { + "FirehoseCidrBlock": "13.208.177.192/27" + }, + "ap-south-1": { + "FirehoseCidrBlock": "13.232.67.32/27" + }, + "ap-southeast-1": { + "FirehoseCidrBlock": "13.228.64.192/27" + }, + "ap-southeast-2": { + "FirehoseCidrBlock": "13.210.67.224/27" + }, + "ca-central-1": { + "FirehoseCidrBlock": "35.183.92.128/27" + }, + "cn-north-1": { + "FirehoseCidrBlock": "52.81.151.32/27" + }, + "cn-northwest-1": { + "FirehoseCidrBlock": "161.189.23.64/27" + }, + "eu-central-1": { + "FirehoseCidrBlock": "35.158.127.160/27" + }, + "eu-north-1": { + "FirehoseCidrBlock": "13.53.63.224/27" + }, + "eu-south-1": { + "FirehoseCidrBlock": "15.161.135.128/27" + }, + "eu-west-1": { + "FirehoseCidrBlock": "52.19.239.192/27" + }, + "eu-west-2": { + "FirehoseCidrBlock": "18.130.1.96/27" + }, + "eu-west-3": { + "FirehoseCidrBlock": "35.180.1.96/27" + }, + "me-south-1": { + "FirehoseCidrBlock": "15.185.91.0/27" + }, + "sa-east-1": { + "FirehoseCidrBlock": "18.228.1.128/27" + }, + "us-east-1": { + "FirehoseCidrBlock": "52.70.63.192/27" + }, + "us-east-2": { + "FirehoseCidrBlock": "13.58.135.96/27" + }, + "us-gov-east-1": { + "FirehoseCidrBlock": "18.253.138.96/27" + }, + "us-gov-west-1": { + "FirehoseCidrBlock": "52.61.204.160/27" + }, + "us-west-1": { + "FirehoseCidrBlock": "13.57.135.192/27" + }, + "us-west-2": { + "FirehoseCidrBlock": "52.89.255.224/27" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts new file mode 100644 index 0000000000000..1adfd9bfc0221 --- /dev/null +++ b/packages/@aws-cdk/aws-kinesisfirehose/test/integ.delivery-stream.ts @@ -0,0 +1,37 @@ +#!/usr/bin/env node +import * as iam from '@aws-cdk/aws-iam'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import * as constructs from 'constructs'; +import * as firehose from '../lib'; + +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-cdk-firehose-delivery-stream'); + +const bucket = new s3.Bucket(stack, 'Bucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, +}); + +const role = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), +}); + +const mockS3Destination: firehose.IDestination = { + bind(_scope: constructs.Construct, _options: firehose.DestinationBindOptions): firehose.DestinationConfig { + const bucketGrant = bucket.grantReadWrite(role); + return { + extendedS3DestinationConfiguration: { + bucketArn: bucket.bucketArn, + roleArn: role.roleArn, + }, + dependables: [bucketGrant], + }; + }, +}; + +new firehose.DeliveryStream(stack, 'Delivery Stream', { + destinations: [mockS3Destination], +}); + +app.synth(); diff --git a/packages/@aws-cdk/aws-kinesisfirehose/test/kinesisfirehose.test.ts b/packages/@aws-cdk/aws-kinesisfirehose/test/kinesisfirehose.test.ts deleted file mode 100644 index c4505ad966984..0000000000000 --- a/packages/@aws-cdk/aws-kinesisfirehose/test/kinesisfirehose.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -import '@aws-cdk/assert-internal/jest'; -import {} from '../lib'; - -test('No tests are specified for this package', () => { - expect(true).toBe(true); -}); diff --git a/packages/@aws-cdk/region-info/build-tools/fact-tables.ts b/packages/@aws-cdk/region-info/build-tools/fact-tables.ts index c2ce689f3aaf3..28ec007b00c84 100644 --- a/packages/@aws-cdk/region-info/build-tools/fact-tables.ts +++ b/packages/@aws-cdk/region-info/build-tools/fact-tables.ts @@ -158,3 +158,32 @@ export const APPMESH_ECR_ACCOUNTS: { [region: string]: string } = { 'us-west-1': '840364872350', 'us-west-2': '840364872350', }; + +// https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html#using-iam-rs-vpc +export const FIREHOSE_CIDR_BLOCKS: { [region: string]: string } = { + 'af-south-1': '13.244.121.224', + 'ap-east-1': '18.162.221.32', + 'ap-northeast-1': '13.113.196.224', + 'ap-northeast-2': '13.209.1.64', + 'ap-northeast-3': '13.208.177.192', + 'ap-south-1': '13.232.67.32', + 'ap-southeast-1': '13.228.64.192', + 'ap-southeast-2': '13.210.67.224', + 'ca-central-1': '35.183.92.128', + 'cn-north-1': '52.81.151.32', + 'cn-northwest-1': '161.189.23.64', + 'eu-central-1': '35.158.127.160', + 'eu-north-1': '13.53.63.224', + 'eu-south-1': '15.161.135.128', + 'eu-west-1': '52.19.239.192', + 'eu-west-2': '18.130.1.96', + 'eu-west-3': '35.180.1.96', + 'me-south-1': '15.185.91.0', + 'sa-east-1': '18.228.1.128', + 'us-east-1': '52.70.63.192', + 'us-east-2': '13.58.135.96', + 'us-gov-east-1': '18.253.138.96', + 'us-gov-west-1': '52.61.204.160', + 'us-west-1': '13.57.135.192', + 'us-west-2': '52.89.255.224', +}; diff --git a/packages/@aws-cdk/region-info/build-tools/generate-static-data.ts b/packages/@aws-cdk/region-info/build-tools/generate-static-data.ts index d23704b6d0062..63455b72ef665 100644 --- a/packages/@aws-cdk/region-info/build-tools/generate-static-data.ts +++ b/packages/@aws-cdk/region-info/build-tools/generate-static-data.ts @@ -3,14 +3,15 @@ import * as fs from 'fs-extra'; import { Default } from '../lib/default'; import { AWS_REGIONS, AWS_SERVICES } from './aws-entities'; import { - APPMESH_ECR_ACCOUNTS, AWS_CDK_METADATA, AWS_OLDER_REGIONS, DLC_REPOSITORY_ACCOUNTS, ELBV2_ACCOUNTS, PARTITION_MAP, - ROUTE_53_BUCKET_WEBSITE_ZONE_IDS, + APPMESH_ECR_ACCOUNTS, AWS_CDK_METADATA, AWS_OLDER_REGIONS, DLC_REPOSITORY_ACCOUNTS, ELBV2_ACCOUNTS, FIREHOSE_CIDR_BLOCKS, + PARTITION_MAP, ROUTE_53_BUCKET_WEBSITE_ZONE_IDS, } from './fact-tables'; async function main(): Promise { checkRegions(APPMESH_ECR_ACCOUNTS); checkRegions(DLC_REPOSITORY_ACCOUNTS); checkRegions(ELBV2_ACCOUNTS); + checkRegions(FIREHOSE_CIDR_BLOCKS); checkRegions(ROUTE_53_BUCKET_WEBSITE_ZONE_IDS); const lines = [ @@ -61,6 +62,11 @@ async function main(): Promise { registerFact(region, 'APPMESH_ECR_ACCOUNT', APPMESH_ECR_ACCOUNTS[region]); + const firehoseCidrBlock = FIREHOSE_CIDR_BLOCKS[region]; + if (firehoseCidrBlock) { + registerFact(region, 'FIREHOSE_CIDR_BLOCK', `${FIREHOSE_CIDR_BLOCKS[region]}/27`); + } + const vpcEndpointServiceNamePrefix = `${domainSuffix.split('.').reverse().join('.')}.vpce`; registerFact(region, 'VPC_ENDPOINT_SERVICE_NAME_PREFIX', vpcEndpointServiceNamePrefix); diff --git a/packages/@aws-cdk/region-info/lib/fact.ts b/packages/@aws-cdk/region-info/lib/fact.ts index 3b5e57835cc7e..6ccef0e8b794f 100644 --- a/packages/@aws-cdk/region-info/lib/fact.ts +++ b/packages/@aws-cdk/region-info/lib/fact.ts @@ -152,6 +152,11 @@ export class FactName { */ public static readonly APPMESH_ECR_ACCOUNT = 'appMeshRepositoryAccount'; + /** + * The CIDR block used by Kinesis Data Firehose servers. + */ + public static readonly FIREHOSE_CIDR_BLOCK = 'firehoseCidrBlock'; + /** * The name of the regional service principal for a given service. * diff --git a/packages/@aws-cdk/region-info/lib/region-info.ts b/packages/@aws-cdk/region-info/lib/region-info.ts index 042b3cec9c177..9e28120a8da62 100644 --- a/packages/@aws-cdk/region-info/lib/region-info.ts +++ b/packages/@aws-cdk/region-info/lib/region-info.ts @@ -117,4 +117,11 @@ export class RegionInfo { public get appMeshRepositoryAccount(): string | undefined { return Fact.find(this.name, FactName.APPMESH_ECR_ACCOUNT); } + + /** + * The CIDR block used by Kinesis Data Firehose servers. + */ + public get firehoseCidrBlock(): string | undefined { + return Fact.find(this.name, FactName.FIREHOSE_CIDR_BLOCK); + } } diff --git a/packages/aws-cdk-lib/package.json b/packages/aws-cdk-lib/package.json index a200e97145db6..411d6eed9a312 100644 --- a/packages/aws-cdk-lib/package.json +++ b/packages/aws-cdk-lib/package.json @@ -239,6 +239,7 @@ "@aws-cdk/aws-kinesisanalytics": "0.0.0", "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-kinesisfirehose-destinations": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", diff --git a/packages/decdk/package.json b/packages/decdk/package.json index 0c38c44ff0625..73c670cec466d 100644 --- a/packages/decdk/package.json +++ b/packages/decdk/package.json @@ -146,6 +146,7 @@ "@aws-cdk/aws-kinesisanalytics": "0.0.0", "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-kinesisfirehose-destinations": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", diff --git a/packages/monocdk/package.json b/packages/monocdk/package.json index 33bbc89af3414..43227b31a4041 100644 --- a/packages/monocdk/package.json +++ b/packages/monocdk/package.json @@ -240,6 +240,7 @@ "@aws-cdk/aws-kinesisanalytics": "0.0.0", "@aws-cdk/aws-kinesisanalytics-flink": "0.0.0", "@aws-cdk/aws-kinesisfirehose": "0.0.0", + "@aws-cdk/aws-kinesisfirehose-destinations": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lakeformation": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", diff --git a/tools/pkglint/lib/rules.ts b/tools/pkglint/lib/rules.ts index a696b43bceabe..add3c58fee4b2 100644 --- a/tools/pkglint/lib/rules.ts +++ b/tools/pkglint/lib/rules.ts @@ -1634,6 +1634,7 @@ export class NoExperimentalDependents extends ValidationRule { ['@aws-cdk/aws-apigatewayv2-integrations', ['@aws-cdk/aws-apigatewayv2']], ['@aws-cdk/aws-apigatewayv2-authorizers', ['@aws-cdk/aws-apigatewayv2']], ['@aws-cdk/aws-events-targets', ['@aws-cdk/aws-kinesisfirehose']], + ['@aws-cdk/aws-kinesisfirehose-destinations', ['@aws-cdk/aws-kinesisfirehose']], ]); private readonly excludedModules = ['@aws-cdk/cloudformation-include']; From cb6e7c9c0c046a5c03bd1a4f1474c9ece1963604 Mon Sep 17 00:00:00 2001 From: Douglas Naphas Date: Thu, 22 Jul 2021 21:39:08 -0400 Subject: [PATCH 04/15] docs: fix link to example integration test in Contributing.md (#15729) Closes gh-15728. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 97bc876d86a0c..423fb1e897ff3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -251,7 +251,7 @@ The steps here are usually AWS CLI commands but they need not be. Examples: * [integ.destinations.ts](https://github.com/aws/aws-cdk/blob/master/packages/%40aws-cdk/aws-lambda-destinations/test/integ.destinations.ts#L7) -* [integ.token-authorizer.ts](https://github.com/aws/aws-cdk/blob/master/packages/%40aws-cdk/aws-apigateway/test/authorizers/integ.token-authorizer.ts#L6) +* [integ.token-authorizer.lit.ts](https://github.com/aws/aws-cdk/blob/master/packages/%40aws-cdk/aws-apigateway/test/authorizers/integ.token-authorizer.lit.ts#L7-L12) #### yarn watch (Optional) From e133bca61b95b71d51b509b646ff1720099ee31e Mon Sep 17 00:00:00 2001 From: marciocarmona Date: Thu, 22 Jul 2021 19:18:44 -0700 Subject: [PATCH 05/15] fix(stepfunctions): non-object arguments to recurseObject are incorrectly treated as objects (#14631) This doesn't actually fix the issue #12935 as currently the Json paths won't be resolved for Lambda steps where the `Resource` is the Lambda ARN and not `arn:aws:states:::lambda:invoke`, but it at least fixes the issue for Text inputs when `payloadResponseOnly: true` and will avoid the same error from happening again if the `recurseObject` is called with a value that's not an object. Ideally the `TaskInput.value` field should be changed to `{ [key: string]: any } | string` here to ensure the type check when sending the value to methods like `FieldUtils.renderObject`: https://github.com/aws/aws-cdk/blob/master/packages/@aws-cdk/aws-stepfunctions/lib/input.ts#L65 Or even better the `TaskInput` should be made generic like: ``` export class TaskInput { ... private constructor(public readonly type: T, public readonly value: ValueType[T]) {} } type ValueType = { [InputType.OBJECT]: { [key: string]: any }, [InputType.TEXT]: string } ``` However, any of the changes above wouldn't be backwards compatible and could break not only internal references in the `aws-cdk` but also on any customer packages using the CDK. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../aws-stepfunctions/lib/json-path.ts | 7 +++- .../aws-stepfunctions/test/fields.test.ts | 35 ++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-stepfunctions/lib/json-path.ts b/packages/@aws-cdk/aws-stepfunctions/lib/json-path.ts index 7a4a56c536a6b..b4602b5e887d0 100644 --- a/packages/@aws-cdk/aws-stepfunctions/lib/json-path.ts +++ b/packages/@aws-cdk/aws-stepfunctions/lib/json-path.ts @@ -82,7 +82,12 @@ interface FieldHandlers { } export function recurseObject(obj: object | undefined, handlers: FieldHandlers, visited: object[] = []): object | undefined { - if (obj === undefined) { return undefined; } + // If the argument received is not actually an object (string, number, boolean, undefined, ...) or null + // just return it as is as there's nothing to be rendered. This should only happen in the original call to + // recurseObject as any recursive calls to it are checking for typeof value === 'object' && value !== null + if (typeof obj !== 'object' || obj === null) { + return obj; + } // Avoiding infinite recursion if (visited.includes(obj)) { return {}; } diff --git a/packages/@aws-cdk/aws-stepfunctions/test/fields.test.ts b/packages/@aws-cdk/aws-stepfunctions/test/fields.test.ts index 15591ddeebd76..382ec424177f9 100644 --- a/packages/@aws-cdk/aws-stepfunctions/test/fields.test.ts +++ b/packages/@aws-cdk/aws-stepfunctions/test/fields.test.ts @@ -1,5 +1,5 @@ import '@aws-cdk/assert-internal/jest'; -import { FieldUtils, JsonPath } from '../lib'; +import { FieldUtils, JsonPath, TaskInput } from '../lib'; describe('Fields', () => { const jsonPathValidationErrorMsg = /exactly '\$', '\$\$', start with '\$.', start with '\$\$.' or start with '\$\['/; @@ -153,6 +153,39 @@ describe('Fields', () => { .toStrictEqual(['$.listField', '$.numField', '$.stringField']); }); + test('rendering a non-object value should just return itself', () => { + expect( + FieldUtils.renderObject(TaskInput.fromText('Hello World').value), + ).toEqual( + 'Hello World', + ); + expect( + FieldUtils.renderObject('Hello World' as any), + ).toEqual( + 'Hello World', + ); + expect( + FieldUtils.renderObject(null as any), + ).toEqual( + null, + ); + expect( + FieldUtils.renderObject(3.14 as any), + ).toEqual( + 3.14, + ); + expect( + FieldUtils.renderObject(true as any), + ).toEqual( + true, + ); + expect( + FieldUtils.renderObject(undefined), + ).toEqual( + undefined, + ); + }); + test('repeated object references at different tree paths should not be considered as recursions', () => { const repeatedObject = { field: JsonPath.stringAt('$.stringField'), From 7668400ec8d4e6ee042c05976f95e42147993375 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Fri, 23 Jul 2021 12:41:21 +0200 Subject: [PATCH 06/15] fix(pipelines): Secrets Manager permissions not added to asset projects (#15718) We used to use an immutable singleton role with `*` permissions for the assets projects, because if there were many different destinations in a pipeline, and each asset build had to publish to each destination, the policy could grow too long and exceed the maximum policy size. However, this also disabled the automatic policy wrangling that CodeBuild would do for us, like automatically adding permissions to bind to a VPC, and adding permissions to read Secrets Manager secrets. This especially becoming relevant since that now in the modern API, it's possible to modify build the environment in a way that normally automatically adds SecretsManager permission, but now won't (and it's not possible to fix either). Replace the immutable singleton role with a mutable singleton role, but in such a way that it won't add permissions statements for which it already has a `*` statement (to cut down on duplication), and have the CB project do the automatic VPC bind permissions again. Fixes #15628. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../@aws-cdk/aws-codebuild/lib/project.ts | 46 +- .../aws-codebuild/test/test.project.ts | 35 + .../lib/codepipeline/codepipeline.ts | 113 +- .../@aws-cdk/pipelines/lib/legacy/pipeline.ts | 85 +- .../@aws-cdk/pipelines/lib/legacy/stage.ts | 2 +- .../lib/private/asset-singleton-role.ts | 85 + .../pipelines/test/compliance/assets.test.ts | 117 +- .../integ.newpipeline-with-vpc.expected.json | 2401 +++++++++++++++++ .../test/integ.newpipeline-with-vpc.ts | 57 + 9 files changed, 2730 insertions(+), 211 deletions(-) create mode 100644 packages/@aws-cdk/pipelines/lib/private/asset-singleton-role.ts create mode 100644 packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.expected.json create mode 100644 packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.ts diff --git a/packages/@aws-cdk/aws-codebuild/lib/project.ts b/packages/@aws-cdk/aws-codebuild/lib/project.ts index 30f8be9051751..efb4896534c60 100644 --- a/packages/@aws-cdk/aws-codebuild/lib/project.ts +++ b/packages/@aws-cdk/aws-codebuild/lib/project.ts @@ -29,6 +29,8 @@ import { CODEPIPELINE_SOURCE_ARTIFACTS_TYPE, NO_SOURCE_TYPE } from './source-typ // eslint-disable-next-line import { Construct as CoreConstruct } from '@aws-cdk/core'; +const VPC_POLICY_SYM = Symbol.for('@aws-cdk/aws-codebuild.roleVpcPolicy'); + /** * The type returned from {@link IProject#enableBatchBuilds}. */ @@ -1437,23 +1439,33 @@ export class Project extends ProjectBase { }, })); - const policy = new iam.Policy(this, 'PolicyDocument', { - statements: [ - new iam.PolicyStatement({ - resources: ['*'], - actions: [ - 'ec2:CreateNetworkInterface', - 'ec2:DescribeNetworkInterfaces', - 'ec2:DeleteNetworkInterface', - 'ec2:DescribeSubnets', - 'ec2:DescribeSecurityGroups', - 'ec2:DescribeDhcpOptions', - 'ec2:DescribeVpcs', - ], - }), - ], - }); - this.role.attachInlinePolicy(policy); + // If the same Role is used for multiple Projects, always creating a new `iam.Policy` + // will attach the same policy multiple times, probably exceeding the maximum size of the + // Role policy. Make sure we only do it once for the same role. + // + // This deduplication could be a feature of the Role itself, but that feels risky and + // is hard to implement (what with Tokens and all). Safer to fix it locally for now. + let policy: iam.Policy | undefined = (this.role as any)[VPC_POLICY_SYM]; + if (!policy) { + policy = new iam.Policy(this, 'PolicyDocument', { + statements: [ + new iam.PolicyStatement({ + resources: ['*'], + actions: [ + 'ec2:CreateNetworkInterface', + 'ec2:DescribeNetworkInterfaces', + 'ec2:DeleteNetworkInterface', + 'ec2:DescribeSubnets', + 'ec2:DescribeSecurityGroups', + 'ec2:DescribeDhcpOptions', + 'ec2:DescribeVpcs', + ], + }), + ], + }); + this.role.attachInlinePolicy(policy); + (this.role as any)[VPC_POLICY_SYM] = policy; + } // add an explicit dependency between the EC2 Policy and this Project - // otherwise, creating the Project fails, as it requires these permissions diff --git a/packages/@aws-cdk/aws-codebuild/test/test.project.ts b/packages/@aws-cdk/aws-codebuild/test/test.project.ts index 2dc652e64f39d..038bb2e0e45cf 100644 --- a/packages/@aws-cdk/aws-codebuild/test/test.project.ts +++ b/packages/@aws-cdk/aws-codebuild/test/test.project.ts @@ -429,6 +429,41 @@ export = { test.done(); }, + 'if a role is shared between projects in a VPC, the VPC Policy is only attached once'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const vpc = new ec2.Vpc(stack, 'Vpc'); + const role = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('codebuild.amazonaws.com'), + }); + const source = codebuild.Source.gitHubEnterprise({ + httpsCloneUrl: 'https://mygithub-enterprise.com/myuser/myrepo', + }); + + // WHEN + new codebuild.Project(stack, 'Project1', { source, role, vpc, projectName: 'P1' }); + new codebuild.Project(stack, 'Project2', { source, role, vpc, projectName: 'P2' }); + + // THEN + // - 1 is for `ec2:CreateNetworkInterfacePermission`, deduplicated as they're part of a single policy + // - 1 is for `ec2:CreateNetworkInterface`, this is the separate Policy we're deduplicating + // We would have found 3 if the deduplication didn't work. + expect(stack).to(countResources('AWS::IAM::Policy', 2)); + + // THEN - both Projects have a DependsOn on the same policy + expect(stack).to(haveResourceLike('AWS::CodeBuild::Project', { + Properties: { Name: 'P1' }, + DependsOn: ['Project1PolicyDocumentF9761562'], + }, ResourcePart.CompleteDefinition)); + + expect(stack).to(haveResourceLike('AWS::CodeBuild::Project', { + Properties: { Name: 'P1' }, + DependsOn: ['Project1PolicyDocumentF9761562'], + }, ResourcePart.CompleteDefinition)); + + test.done(); + }, + 'can use an imported Role for a Project within a VPC'(test: Test) { const stack = new cdk.Stack(); diff --git a/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline.ts b/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline.ts index e3c10d29b0740..4e509faee2111 100644 --- a/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline.ts +++ b/packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline.ts @@ -4,13 +4,14 @@ import * as cp from '@aws-cdk/aws-codepipeline'; import * as cpa from '@aws-cdk/aws-codepipeline-actions'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; -import { Aws, Fn, IDependable, Lazy, PhysicalName, Stack } from '@aws-cdk/core'; +import { Aws, Fn, Lazy, PhysicalName, Stack } from '@aws-cdk/core'; import * as cxapi from '@aws-cdk/cx-api'; import { Construct, Node } from 'constructs'; import { AssetType, FileSet, IFileSetProducer, ManualApprovalStep, ShellStep, StackAsset, StackDeployment, Step } from '../blueprint'; import { DockerCredential, dockerCredentialsInstallCommands, DockerCredentialUsage } from '../docker-credentials'; import { GraphNode, GraphNodeCollection, isGraph, AGraphNode, PipelineGraph } from '../helpers-internal'; import { PipelineBase } from '../main'; +import { AssetSingletonRole } from '../private/asset-singleton-role'; import { appOf, assemblyBuilderOf, embeddedAsmPath, obtainScope } from '../private/construct-internals'; import { toPosixPath } from '../private/fs'; import { enumerate, flatten, maybeSuffix, noUndefined } from '../private/javascript'; @@ -254,11 +255,6 @@ export class CodePipeline extends PipelineBase { */ private readonly assetCodeBuildRoles: Record = {}; - /** - * Policies created for the build projects that they have to depend on - */ - private readonly assetAttachedPolicies: Record = {}; - /** * Per asset type, the target role ARNs that need to be assumed */ @@ -635,7 +631,7 @@ export class CodePipeline extends PipelineBase { } } - const assetBuildConfig = this.obtainAssetCodeBuildRole(assets[0].assetType); + const role = this.obtainAssetCodeBuildRole(assets[0].assetType); // The base commands that need to be run const script = new CodeBuildStep(node.id, { @@ -647,13 +643,12 @@ export class CodePipeline extends PipelineBase { buildEnvironment: { privileged: assets.some(asset => asset.assetType === AssetType.DOCKER_IMAGE), }, - role: assetBuildConfig.role, + role, }); // Customizations that are not accessible to regular users return CodeBuildFactory.fromCodeBuildStep(node.id, script, { additionalConstructLevel: false, - additionalDependable: assetBuildConfig.dependable, // If we use a single publisher, pass buildspec via file otherwise it'll // grow too big. @@ -775,18 +770,15 @@ export class CodePipeline extends PipelineBase { * Modeled after the CodePipeline role and 'CodePipelineActionRole' roles. * Generates one role per asset type to separate file and Docker/image-based permissions. */ - private obtainAssetCodeBuildRole(assetType: AssetType): AssetCodeBuildRole { + private obtainAssetCodeBuildRole(assetType: AssetType): iam.IRole { if (this.assetCodeBuildRoles[assetType]) { - return { - role: this.assetCodeBuildRoles[assetType], - dependable: this.assetAttachedPolicies[assetType], - }; + return this.assetCodeBuildRoles[assetType]; } const stack = Stack.of(this); const rolePrefix = assetType === AssetType.DOCKER_IMAGE ? 'Docker' : 'File'; - const assetRole = new iam.Role(this.assetsScope, `${rolePrefix}Role`, { + const assetRole = new AssetSingletonRole(this.assetsScope, `${rolePrefix}Role`, { roleName: PhysicalName.GENERATE_IF_NEEDED, assumedBy: new iam.CompositePrincipal( new iam.ServicePrincipal('codebuild.amazonaws.com'), @@ -794,45 +786,6 @@ export class CodePipeline extends PipelineBase { ), }); - // Logging permissions - const logGroupArn = stack.formatArn({ - service: 'logs', - resource: 'log-group', - sep: ':', - resourceName: '/aws/codebuild/*', - }); - assetRole.addToPolicy(new iam.PolicyStatement({ - resources: [logGroupArn], - actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], - })); - - // CodeBuild report groups - const codeBuildArn = stack.formatArn({ - service: 'codebuild', - resource: 'report-group', - resourceName: '*', - }); - assetRole.addToPolicy(new iam.PolicyStatement({ - actions: [ - 'codebuild:CreateReportGroup', - 'codebuild:CreateReport', - 'codebuild:UpdateReport', - 'codebuild:BatchPutTestCases', - 'codebuild:BatchPutCodeCoverages', - ], - resources: [codeBuildArn], - })); - - // CodeBuild start/stop - assetRole.addToPolicy(new iam.PolicyStatement({ - resources: ['*'], - actions: [ - 'codebuild:BatchGetBuilds', - 'codebuild:StartBuild', - 'codebuild:StopBuild', - ], - })); - // Publishing role access // The ARNs include raw AWS pseudo parameters (e.g., ${AWS::Partition}), which need to be substituted. // Lazy-evaluated so all asset publishing roles are included. @@ -846,51 +799,8 @@ export class CodePipeline extends PipelineBase { this.dockerCredentials.forEach(reg => reg.grantRead(assetRole, DockerCredentialUsage.ASSET_PUBLISHING)); } - // Artifact access - this.pipeline.artifactBucket.grantRead(assetRole); - - // VPC permissions required for CodeBuild - // Normally CodeBuild itself takes care of this but we're creating a singleton role so now - // we need to do this. - const assetCodeBuildOptions = this.codeBuildDefaultsFor(CodeBuildProjectType.ASSETS); - if (assetCodeBuildOptions?.vpc) { - const vpcPolicy = new iam.Policy(assetRole, 'VpcPolicy', { - statements: [ - new iam.PolicyStatement({ - resources: [`arn:${Aws.PARTITION}:ec2:${Aws.REGION}:${Aws.ACCOUNT_ID}:network-interface/*`], - actions: ['ec2:CreateNetworkInterfacePermission'], - conditions: { - StringEquals: { - 'ec2:Subnet': assetCodeBuildOptions.vpc - .selectSubnets(assetCodeBuildOptions.subnetSelection).subnetIds - .map(si => `arn:${Aws.PARTITION}:ec2:${Aws.REGION}:${Aws.ACCOUNT_ID}:subnet/${si}`), - 'ec2:AuthorizedService': 'codebuild.amazonaws.com', - }, - }, - }), - new iam.PolicyStatement({ - resources: ['*'], - actions: [ - 'ec2:CreateNetworkInterface', - 'ec2:DescribeNetworkInterfaces', - 'ec2:DeleteNetworkInterface', - 'ec2:DescribeSubnets', - 'ec2:DescribeSecurityGroups', - 'ec2:DescribeDhcpOptions', - 'ec2:DescribeVpcs', - ], - }), - ], - }); - assetRole.attachInlinePolicy(vpcPolicy); - this.assetAttachedPolicies[assetType] = vpcPolicy; - } - - this.assetCodeBuildRoles[assetType] = assetRole.withoutPolicyUpdates(); - return { - role: this.assetCodeBuildRoles[assetType], - dependable: this.assetAttachedPolicies[assetType], - }; + this.assetCodeBuildRoles[assetType] = assetRole; + return assetRole; } } @@ -903,11 +813,6 @@ function dockerUsageFromCodeBuild(cbt: CodeBuildProjectType): DockerCredentialUs } } -interface AssetCodeBuildRole { - readonly role: iam.IRole; - readonly dependable?: IDependable; -} - enum CodeBuildProjectType { SYNTH = 'SYNTH', ASSETS = 'ASSETS', diff --git a/packages/@aws-cdk/pipelines/lib/legacy/pipeline.ts b/packages/@aws-cdk/pipelines/lib/legacy/pipeline.ts index e100c8c4a90f0..b8d7769ad3fc0 100644 --- a/packages/@aws-cdk/pipelines/lib/legacy/pipeline.ts +++ b/packages/@aws-cdk/pipelines/lib/legacy/pipeline.ts @@ -3,11 +3,12 @@ import * as codebuild from '@aws-cdk/aws-codebuild'; import * as codepipeline from '@aws-cdk/aws-codepipeline'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; -import { Annotations, App, Aws, CfnOutput, Fn, Lazy, PhysicalName, Stack, Stage } from '@aws-cdk/core'; +import { Annotations, App, CfnOutput, Fn, Lazy, PhysicalName, Stack, Stage } from '@aws-cdk/core'; import { Construct } from 'constructs'; import { AssetType } from '../blueprint/asset-type'; import { dockerCredentialsInstallCommands, DockerCredential, DockerCredentialUsage } from '../docker-credentials'; import { ApplicationSecurityCheck } from '../private/application-security-check'; +import { AssetSingletonRole } from '../private/asset-singleton-role'; import { appOf, assemblyBuilderOf } from '../private/construct-internals'; import { DeployCdkStackAction, PublishAssetsAction, UpdatePipelineAction } from './actions'; import { AddStageOptions, AssetPublishingCommand, BaseStageOptions, CdkStage, StackOutput } from './stage'; @@ -580,50 +581,11 @@ class AssetPublishing extends CoreConstruct { if (this.assetRoles[assetType]) { return this.assetRoles[assetType]; } const rolePrefix = assetType === AssetType.DOCKER_IMAGE ? 'Docker' : 'File'; - const assetRole = new iam.Role(this, `${rolePrefix}Role`, { + const assetRole = new AssetSingletonRole(this, `${rolePrefix}Role`, { roleName: PhysicalName.GENERATE_IF_NEEDED, assumedBy: new iam.CompositePrincipal(new iam.ServicePrincipal('codebuild.amazonaws.com'), new iam.AccountPrincipal(Stack.of(this).account)), }); - // Logging permissions - const logGroupArn = Stack.of(this).formatArn({ - service: 'logs', - resource: 'log-group', - sep: ':', - resourceName: '/aws/codebuild/*', - }); - assetRole.addToPolicy(new iam.PolicyStatement({ - resources: [logGroupArn], - actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], - })); - - // CodeBuild report groups - const codeBuildArn = Stack.of(this).formatArn({ - service: 'codebuild', - resource: 'report-group', - resourceName: '*', - }); - assetRole.addToPolicy(new iam.PolicyStatement({ - actions: [ - 'codebuild:CreateReportGroup', - 'codebuild:CreateReport', - 'codebuild:UpdateReport', - 'codebuild:BatchPutTestCases', - 'codebuild:BatchPutCodeCoverages', - ], - resources: [codeBuildArn], - })); - - // CodeBuild start/stop - assetRole.addToPolicy(new iam.PolicyStatement({ - resources: ['*'], - actions: [ - 'codebuild:BatchGetBuilds', - 'codebuild:StartBuild', - 'codebuild:StopBuild', - ], - })); - // Publishing role access // The ARNs include raw AWS pseudo parameters (e.g., ${AWS::Partition}), which need to be substituted. // Lazy-evaluated so all asset publishing roles are included. @@ -637,46 +599,7 @@ class AssetPublishing extends CoreConstruct { this.dockerCredentials.forEach(reg => reg.grantRead(assetRole, DockerCredentialUsage.ASSET_PUBLISHING)); } - // Artifact access - this.pipeline.artifactBucket.grantRead(assetRole); - - // VPC permissions required for CodeBuild - // Normally CodeBuild itself takes care of this but we're creating a singleton role so now - // we need to do this. - if (this.props.vpc) { - const vpcPolicy = new iam.Policy(assetRole, 'VpcPolicy', { - statements: [ - new iam.PolicyStatement({ - resources: [`arn:${Aws.PARTITION}:ec2:${Aws.REGION}:${Aws.ACCOUNT_ID}:network-interface/*`], - actions: ['ec2:CreateNetworkInterfacePermission'], - conditions: { - StringEquals: { - 'ec2:Subnet': this.props.vpc - .selectSubnets(this.props.subnetSelection).subnetIds - .map(si => `arn:${Aws.PARTITION}:ec2:${Aws.REGION}:${Aws.ACCOUNT_ID}:subnet/${si}`), - 'ec2:AuthorizedService': 'codebuild.amazonaws.com', - }, - }, - }), - new iam.PolicyStatement({ - resources: ['*'], - actions: [ - 'ec2:CreateNetworkInterface', - 'ec2:DescribeNetworkInterfaces', - 'ec2:DeleteNetworkInterface', - 'ec2:DescribeSubnets', - 'ec2:DescribeSecurityGroups', - 'ec2:DescribeDhcpOptions', - 'ec2:DescribeVpcs', - ], - }), - ], - }); - assetRole.attachInlinePolicy(vpcPolicy); - this.assetAttachedPolicies[assetType] = vpcPolicy; - } - - this.assetRoles[assetType] = assetRole.withoutPolicyUpdates(); + this.assetRoles[assetType] = assetRole; return this.assetRoles[assetType]; } } diff --git a/packages/@aws-cdk/pipelines/lib/legacy/stage.ts b/packages/@aws-cdk/pipelines/lib/legacy/stage.ts index 3b4140fdba5ce..bfb997e908196 100644 --- a/packages/@aws-cdk/pipelines/lib/legacy/stage.ts +++ b/packages/@aws-cdk/pipelines/lib/legacy/stage.ts @@ -9,6 +9,7 @@ import { Construct, Node } from 'constructs'; import { AssetType } from '../blueprint/asset-type'; import { ApplicationSecurityCheck } from '../private/application-security-check'; import { AssetManifestReader, DockerImageManifestEntry, FileManifestEntry } from '../private/asset-manifest'; +import { pipelineSynth } from '../private/construct-internals'; import { topologicalSort } from '../private/toposort'; import { DeployCdkStackAction } from './actions'; import { CdkPipeline } from './pipeline'; @@ -16,7 +17,6 @@ import { CdkPipeline } from './pipeline'; // v2 - keep this import as a separate section to reduce merge conflict when forward merging with the v2 branch. // eslint-disable-next-line import { Construct as CoreConstruct } from '@aws-cdk/core'; -import { pipelineSynth } from '../private/construct-internals'; /** * Construction properties for a CdkStage diff --git a/packages/@aws-cdk/pipelines/lib/private/asset-singleton-role.ts b/packages/@aws-cdk/pipelines/lib/private/asset-singleton-role.ts new file mode 100644 index 0000000000000..5d52c1d5a47a9 --- /dev/null +++ b/packages/@aws-cdk/pipelines/lib/private/asset-singleton-role.ts @@ -0,0 +1,85 @@ +import * as iam from '@aws-cdk/aws-iam'; +import { PolicyStatement } from '@aws-cdk/aws-iam'; +import { ConcreteDependable, Stack } from '@aws-cdk/core'; +import { Construct } from 'constructs'; + +/** + * Role which will be reused across asset jobs + * + * Has some '*' resources to save IAM policy space, and will not + * actually add policies that look like policies that were already added. + */ +export class AssetSingletonRole extends iam.Role { + private _rejectDuplicates = false; + + constructor(scope: Construct, id: string, props: iam.RoleProps) { + super(scope, id, props); + + // Logging permissions + this.addToPolicy(new iam.PolicyStatement({ + resources: [Stack.of(this).formatArn({ + service: 'logs', + resource: 'log-group', + sep: ':', + resourceName: '/aws/codebuild/*', + })], + actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], + })); + + // CodeBuild report groups + this.addToPolicy(new iam.PolicyStatement({ + actions: [ + 'codebuild:CreateReportGroup', + 'codebuild:CreateReport', + 'codebuild:UpdateReport', + 'codebuild:BatchPutTestCases', + 'codebuild:BatchPutCodeCoverages', + ], + resources: [Stack.of(this).formatArn({ + service: 'codebuild', + resource: 'report-group', + resourceName: '*', + })], + })); + + // CodeBuild start/stop + this.addToPolicy(new iam.PolicyStatement({ + resources: ['*'], + actions: [ + 'codebuild:BatchGetBuilds', + 'codebuild:StartBuild', + 'codebuild:StopBuild', + ], + })); + + this._rejectDuplicates = true; + } + + public addToPrincipalPolicy(statement: PolicyStatement): iam.AddToPrincipalPolicyResult { + const json = statement.toStatementJson(); + const acts = JSON.stringify(json.Action); + + // These have already been added with wildcard resources on creation + const alreadyAdded = [ + '["logs:CreateLogGroup","logs:CreateLogStream","logs:PutLogEvents"]', + '["codebuild:CreateReportGroup","codebuild:CreateReport","codebuild:UpdateReport","codebuild:BatchPutTestCases","codebuild:BatchPutCodeCoverages"]', + '["codebuild:BatchGetBuilds","codebuild:StartBuild","codebuild:StopBuild"]', + ]; + + if (this._rejectDuplicates && alreadyAdded.includes(acts)) { + // Pretend we did it + return { statementAdded: true, policyDependable: new ConcreteDependable() }; + } + + // These are added in duplicate (specifically these come from + // Project#bindToCodePipeline) -- the original singleton asset role didn't + // have these, and they're not necessary either, so in order to not cause + // unnecessary diffs, recognize and drop them there as well. + if (acts === '["kms:Decrypt","kms:Encrypt","kms:ReEncrypt*","kms:GenerateDataKey*"]') { + // Pretend we did it + return { statementAdded: true, policyDependable: new ConcreteDependable() }; + } + + return super.addToPrincipalPolicy(statement); + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/pipelines/test/compliance/assets.test.ts b/packages/@aws-cdk/pipelines/test/compliance/assets.test.ts index 8a465309016ef..c1b72cf7ab316 100644 --- a/packages/@aws-cdk/pipelines/test/compliance/assets.test.ts +++ b/packages/@aws-cdk/pipelines/test/compliance/assets.test.ts @@ -670,7 +670,7 @@ describe('pipeline with VPC', () => { } }); - behavior('Asset publishing CodeBuild Projects have a dependency on attached policies to the role', (suite) => { + behavior('Asset publishing CodeBuild Projects have correct VPC permissions', (suite) => { suite.legacy(() => { const pipeline = new LegacyTestGitHubNpmPipeline(pipelineStack, 'Cdk', { vpc, @@ -690,17 +690,31 @@ describe('pipeline with VPC', () => { function THEN_codePipelineExpectation() { // Assets Project + expect(pipelineStack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Resource: '*', + Action: [ + 'ec2:CreateNetworkInterface', + 'ec2:DescribeNetworkInterfaces', + 'ec2:DeleteNetworkInterface', + 'ec2:DescribeSubnets', + 'ec2:DescribeSecurityGroups', + 'ec2:DescribeDhcpOptions', + 'ec2:DescribeVpcs', + ], + }, + ], + }, + Roles: [{ Ref: 'CdkAssetsDockerRole484B6DD3' }], + }); expect(pipelineStack).toHaveResourceLike('AWS::CodeBuild::Project', { Properties: { - ServiceRole: { - 'Fn::GetAtt': [ - 'CdkAssetsDockerRole484B6DD3', - 'Arn', - ], - }, + ServiceRole: { 'Fn::GetAtt': ['CdkAssetsDockerRole484B6DD3', 'Arn'] }, }, DependsOn: [ - 'CdkAssetsDockerRoleVpcPolicy86CA024B', + 'CdkAssetsDockerAsset1PolicyDocument8DA96A22', ], }, ResourcePart.CompleteDefinition); } @@ -939,3 +953,90 @@ function expectedAssetRolePolicy(assumeRolePattern: string | string[], attachedR Roles: [{ Ref: attachedRole }], }; } + + +behavior('necessary secrets manager permissions get added to asset roles', suite => { + // Not possible to configure this for legacy pipelines + suite.doesNotApply.legacy(); + + suite.modern(() => { + const pipeline = new ModernTestGitHubNpmPipeline(pipelineStack, 'Pipeline', { + assetPublishingCodeBuildDefaults: { + buildEnvironment: { + environmentVariables: { + FOOBAR: { + value: 'FoobarSecret', + type: cb.BuildEnvironmentVariableType.SECRETS_MANAGER, + }, + }, + }, + }, + }); + pipeline.addStage(new FileAssetApp(pipelineStack, 'MyApp')); + + THEN_codePipelineExpectation(); + }); + + function THEN_codePipelineExpectation() { + expect(pipelineStack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyDocument: { + Statement: arrayWith({ + Action: 'secretsmanager:GetSecretValue', + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { Ref: 'AWS::Partition' }, + ':secretsmanager:us-pipeline:123pipeline:secret:FoobarSecret-??????', + ], + ], + }, + }), + }, + Roles: [ + { Ref: 'PipelineAssetsFileRole59943A77' }, + ], + }); + } +}); + +behavior('adding environment variable to assets job adds SecretsManager permissions', suite => { + // No way to manipulate buildEnvironment in legacy API + suite.doesNotApply.legacy(); + + suite.modern(() => { + const pipeline = new ModernTestGitHubNpmPipeline(pipelineStack, 'Pipeline', { + assetPublishingCodeBuildDefaults: { + buildEnvironment: { + environmentVariables: { + FOOBAR: { + value: 'FoobarSecret', + type: cb.BuildEnvironmentVariableType.SECRETS_MANAGER, + }, + }, + }, + }, + }); + pipeline.addStage(new FileAssetApp(pipelineStack, 'MyApp')); + + expect(pipelineStack).toHaveResourceLike('AWS::IAM::Policy', { + PolicyDocument: { + Statement: arrayWith( + objectLike({ + Action: 'secretsmanager:GetSecretValue', + Effect: 'Allow', + Resource: { + 'Fn::Join': ['', [ + 'arn:', + { Ref: 'AWS::Partition' }, + ':secretsmanager:us-pipeline:123pipeline:secret:FoobarSecret-??????', + ]], + }, + }), + ), + }, + }); + }); +}); \ No newline at end of file diff --git a/packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.expected.json b/packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.expected.json new file mode 100644 index 0000000000000..848406b0ad02e --- /dev/null +++ b/packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.expected.json @@ -0,0 +1,2401 @@ +{ + "Resources": { + "Vpc8378EB38": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "InstanceTenancy": "default", + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc" + } + ] + } + }, + "VpcPublicSubnet1Subnet5C2D37C4": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.0.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet1" + } + ] + } + }, + "VpcPublicSubnet1RouteTable6C95E38E": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet1" + } + ] + } + }, + "VpcPublicSubnet1RouteTableAssociation97140677": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet1RouteTable6C95E38E" + }, + "SubnetId": { + "Ref": "VpcPublicSubnet1Subnet5C2D37C4" + } + } + }, + "VpcPublicSubnet1DefaultRoute3DA9E72A": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet1RouteTable6C95E38E" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "VpcIGWD7BA715C" + } + }, + "DependsOn": [ + "VpcVPCGWBF912B6E" + ] + }, + "VpcPublicSubnet1EIPD7E02669": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet1" + } + ] + } + }, + "VpcPublicSubnet1NATGateway4D7517AA": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "SubnetId": { + "Ref": "VpcPublicSubnet1Subnet5C2D37C4" + }, + "AllocationId": { + "Fn::GetAtt": [ + "VpcPublicSubnet1EIPD7E02669", + "AllocationId" + ] + }, + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet1" + } + ] + } + }, + "VpcPublicSubnet2Subnet691E08A3": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.32.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet2" + } + ] + } + }, + "VpcPublicSubnet2RouteTable94F7E489": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet2" + } + ] + } + }, + "VpcPublicSubnet2RouteTableAssociationDD5762D8": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet2RouteTable94F7E489" + }, + "SubnetId": { + "Ref": "VpcPublicSubnet2Subnet691E08A3" + } + } + }, + "VpcPublicSubnet2DefaultRoute97F91067": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet2RouteTable94F7E489" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "VpcIGWD7BA715C" + } + }, + "DependsOn": [ + "VpcVPCGWBF912B6E" + ] + }, + "VpcPublicSubnet2EIP3C605A87": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet2" + } + ] + } + }, + "VpcPublicSubnet2NATGateway9182C01D": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "SubnetId": { + "Ref": "VpcPublicSubnet2Subnet691E08A3" + }, + "AllocationId": { + "Fn::GetAtt": [ + "VpcPublicSubnet2EIP3C605A87", + "AllocationId" + ] + }, + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet2" + } + ] + } + }, + "VpcPublicSubnet3SubnetBE12F0B6": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.64.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet3" + } + ] + } + }, + "VpcPublicSubnet3RouteTable93458DBB": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet3" + } + ] + } + }, + "VpcPublicSubnet3RouteTableAssociation1F1EDF02": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet3RouteTable93458DBB" + }, + "SubnetId": { + "Ref": "VpcPublicSubnet3SubnetBE12F0B6" + } + } + }, + "VpcPublicSubnet3DefaultRoute4697774F": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPublicSubnet3RouteTable93458DBB" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "VpcIGWD7BA715C" + } + }, + "DependsOn": [ + "VpcVPCGWBF912B6E" + ] + }, + "VpcPublicSubnet3EIP3A666A23": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet3" + } + ] + } + }, + "VpcPublicSubnet3NATGateway7640CD1D": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "SubnetId": { + "Ref": "VpcPublicSubnet3SubnetBE12F0B6" + }, + "AllocationId": { + "Fn::GetAtt": [ + "VpcPublicSubnet3EIP3A666A23", + "AllocationId" + ] + }, + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PublicSubnet3" + } + ] + } + }, + "VpcPrivateSubnet1Subnet536B997A": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.96.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PrivateSubnet1" + } + ] + } + }, + "VpcPrivateSubnet1RouteTableB2C5B500": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PrivateSubnet1" + } + ] + } + }, + "VpcPrivateSubnet1RouteTableAssociation70C59FA6": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet1RouteTableB2C5B500" + }, + "SubnetId": { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + } + } + }, + "VpcPrivateSubnet1DefaultRouteBE02A9ED": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet1RouteTableB2C5B500" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "VpcPublicSubnet1NATGateway4D7517AA" + } + } + }, + "VpcPrivateSubnet2Subnet3788AAA1": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.128.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PrivateSubnet2" + } + ] + } + }, + "VpcPrivateSubnet2RouteTableA678073B": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PrivateSubnet2" + } + ] + } + }, + "VpcPrivateSubnet2RouteTableAssociationA89CAD56": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet2RouteTableA678073B" + }, + "SubnetId": { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + } + } + }, + "VpcPrivateSubnet2DefaultRoute060D2087": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet2RouteTableA678073B" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "VpcPublicSubnet2NATGateway9182C01D" + } + } + }, + "VpcPrivateSubnet3SubnetF258B56E": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.160.0/19", + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PrivateSubnet3" + } + ] + } + }, + "VpcPrivateSubnet3RouteTableD98824C7": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc/PrivateSubnet3" + } + ] + } + }, + "VpcPrivateSubnet3RouteTableAssociation16BDDC43": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet3RouteTableD98824C7" + }, + "SubnetId": { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + } + }, + "VpcPrivateSubnet3DefaultRoute94B74F0D": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "VpcPrivateSubnet3RouteTableD98824C7" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "VpcPublicSubnet3NATGateway7640CD1D" + } + } + }, + "VpcIGWD7BA715C": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "PipelineStack/Vpc" + } + ] + } + }, + "VpcVPCGWBF912B6E": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "Vpc8378EB38" + }, + "InternetGatewayId": { + "Ref": "VpcIGWD7BA715C" + } + } + }, + "PipelineArtifactsBucketAEA9A052": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + { + "ServerSideEncryptionByDefault": { + "SSEAlgorithm": "aws:kms" + } + } + ] + }, + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "PipelineArtifactsBucketPolicyF53CCC52": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "PipelineArtifactsBucketAEA9A052" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":role/cdk-hnb659fds-deploy-role-", + { + "Ref": "AWS::AccountId" + }, + "-", + { + "Ref": "AWS::Region" + } + ] + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "PipelineArtifactsBucketAEA9A052", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "PipelineArtifactsBucketAEA9A052", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "PipelineRoleB27FAA37": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "codepipeline.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "PipelineRoleDefaultPolicy7BDC1ABB": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:Abort*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "PipelineArtifactsBucketAEA9A052", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "PipelineArtifactsBucketAEA9A052", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "PipelineBuildSynthCodePipelineActionRole4E7A6C97", + "Arn" + ] + } + }, + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "PipelineUpdatePipelineSelfMutateCodePipelineActionRoleD6D4E5CF", + "Arn" + ] + } + }, + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "PipelineAssetsFileAsset1CodePipelineActionRoleC0EC649A", + "Arn" + ] + } + }, + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "PipelineAssetsFileAsset2CodePipelineActionRole06965A59", + "Arn" + ] + } + }, + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":role/cdk-hnb659fds-deploy-role-", + { + "Ref": "AWS::AccountId" + }, + "-", + { + "Ref": "AWS::Region" + } + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineRoleDefaultPolicy7BDC1ABB", + "Roles": [ + { + "Ref": "PipelineRoleB27FAA37" + } + ] + } + }, + "Pipeline9850B417": { + "Type": "AWS::CodePipeline::Pipeline", + "Properties": { + "RoleArn": { + "Fn::GetAtt": [ + "PipelineRoleB27FAA37", + "Arn" + ] + }, + "Stages": [ + { + "Actions": [ + { + "ActionTypeId": { + "Category": "Source", + "Owner": "ThirdParty", + "Provider": "GitHub", + "Version": "1" + }, + "Configuration": { + "Owner": "rix0rrr", + "Repo": "cdk-pipelines-demo", + "Branch": "main", + "OAuthToken": "{{resolve:secretsmanager:github-token:SecretString:::}}", + "PollForSourceChanges": false + }, + "Name": "rix0rrr_cdk-pipelines-demo", + "OutputArtifacts": [ + { + "Name": "rix0rrr_cdk-pipelines-demo_Source" + } + ], + "RunOrder": 1 + } + ], + "Name": "Source" + }, + { + "Actions": [ + { + "ActionTypeId": { + "Category": "Build", + "Owner": "AWS", + "Provider": "CodeBuild", + "Version": "1" + }, + "Configuration": { + "ProjectName": { + "Ref": "PipelineBuildSynthCdkBuildProject6BEFA8E6" + }, + "EnvironmentVariables": "[{\"name\":\"_PROJECT_CONFIG_HASH\",\"type\":\"PLAINTEXT\",\"value\":\"00ebacfb32b1bde8d3638577308e7b7144dfa3b0a58a83bc6ff38a3b1f26951c\"}]" + }, + "InputArtifacts": [ + { + "Name": "rix0rrr_cdk-pipelines-demo_Source" + } + ], + "Name": "Synth", + "OutputArtifacts": [ + { + "Name": "Synth_Output" + } + ], + "RoleArn": { + "Fn::GetAtt": [ + "PipelineBuildSynthCodePipelineActionRole4E7A6C97", + "Arn" + ] + }, + "RunOrder": 1 + } + ], + "Name": "Build" + }, + { + "Actions": [ + { + "ActionTypeId": { + "Category": "Build", + "Owner": "AWS", + "Provider": "CodeBuild", + "Version": "1" + }, + "Configuration": { + "ProjectName": { + "Ref": "PipelineUpdatePipelineSelfMutationDAA41400" + }, + "EnvironmentVariables": "[{\"name\":\"_PROJECT_CONFIG_HASH\",\"type\":\"PLAINTEXT\",\"value\":\"9eda7f97d24aac861052bb47a41b80eecdd56096bf9a88a27c88d94c463785c8\"}]" + }, + "InputArtifacts": [ + { + "Name": "Synth_Output" + } + ], + "Name": "SelfMutate", + "RoleArn": { + "Fn::GetAtt": [ + "PipelineUpdatePipelineSelfMutateCodePipelineActionRoleD6D4E5CF", + "Arn" + ] + }, + "RunOrder": 1 + } + ], + "Name": "UpdatePipeline" + }, + { + "Actions": [ + { + "ActionTypeId": { + "Category": "Build", + "Owner": "AWS", + "Provider": "CodeBuild", + "Version": "1" + }, + "Configuration": { + "ProjectName": { + "Ref": "PipelineAssetsFileAsset185A67CB4" + } + }, + "InputArtifacts": [ + { + "Name": "Synth_Output" + } + ], + "Name": "FileAsset1", + "RoleArn": { + "Fn::GetAtt": [ + "PipelineAssetsFileAsset1CodePipelineActionRoleC0EC649A", + "Arn" + ] + }, + "RunOrder": 1 + }, + { + "ActionTypeId": { + "Category": "Build", + "Owner": "AWS", + "Provider": "CodeBuild", + "Version": "1" + }, + "Configuration": { + "ProjectName": { + "Ref": "PipelineAssetsFileAsset24D2D639B" + } + }, + "InputArtifacts": [ + { + "Name": "Synth_Output" + } + ], + "Name": "FileAsset2", + "RoleArn": { + "Fn::GetAtt": [ + "PipelineAssetsFileAsset2CodePipelineActionRole06965A59", + "Arn" + ] + }, + "RunOrder": 1 + } + ], + "Name": "Assets" + }, + { + "Actions": [ + { + "ActionTypeId": { + "Category": "Deploy", + "Owner": "AWS", + "Provider": "CloudFormation", + "Version": "1" + }, + "Configuration": { + "StackName": "Beta-Stack1", + "Capabilities": "CAPABILITY_NAMED_IAM", + "RoleArn": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":role/cdk-hnb659fds-cfn-exec-role-", + { + "Ref": "AWS::AccountId" + }, + "-", + { + "Ref": "AWS::Region" + } + ] + ] + }, + "ActionMode": "CHANGE_SET_REPLACE", + "ChangeSetName": "PipelineChange", + "TemplatePath": "Synth_Output::assembly-PipelineStack-Beta/PipelineStackBetaStack1E6541489.template.json" + }, + "InputArtifacts": [ + { + "Name": "Synth_Output" + } + ], + "Name": "Prepare", + "RoleArn": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":role/cdk-hnb659fds-deploy-role-", + { + "Ref": "AWS::AccountId" + }, + "-", + { + "Ref": "AWS::Region" + } + ] + ] + }, + "RunOrder": 1 + }, + { + "ActionTypeId": { + "Category": "Deploy", + "Owner": "AWS", + "Provider": "CloudFormation", + "Version": "1" + }, + "Configuration": { + "StackName": "Beta-Stack1", + "ActionMode": "CHANGE_SET_EXECUTE", + "ChangeSetName": "PipelineChange" + }, + "Name": "Deploy", + "RoleArn": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":role/cdk-hnb659fds-deploy-role-", + { + "Ref": "AWS::AccountId" + }, + "-", + { + "Ref": "AWS::Region" + } + ] + ] + }, + "RunOrder": 2 + } + ], + "Name": "Beta" + } + ], + "ArtifactStore": { + "Location": { + "Ref": "PipelineArtifactsBucketAEA9A052" + }, + "Type": "S3" + }, + "RestartExecutionOnUpdate": true + }, + "DependsOn": [ + "PipelineRoleDefaultPolicy7BDC1ABB", + "PipelineRoleB27FAA37" + ] + }, + "PipelineSourcerix0rrrcdkpipelinesdemoWebhookResourceDB0C1BCA": { + "Type": "AWS::CodePipeline::Webhook", + "Properties": { + "Authentication": "GITHUB_HMAC", + "AuthenticationConfiguration": { + "SecretToken": "{{resolve:secretsmanager:github-token:SecretString:::}}" + }, + "Filters": [ + { + "JsonPath": "$.ref", + "MatchEquals": "refs/heads/{Branch}" + } + ], + "TargetAction": "rix0rrr_cdk-pipelines-demo", + "TargetPipeline": { + "Ref": "Pipeline9850B417" + }, + "TargetPipelineVersion": 1, + "RegisterWithThirdParty": true + } + }, + "PipelineBuildSynthCdkBuildProjectRole231EEA2A": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "codebuild.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "PipelineBuildSynthCdkBuildProjectRoleDefaultPolicyFB6C941C": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:CreateNetworkInterfacePermission", + "Condition": { + "StringEquals": { + "ec2:Subnet": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":subnet/", + { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":subnet/", + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":subnet/", + { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + ] + ] + } + ], + "ec2:AuthorizedService": "codebuild.amazonaws.com" + } + }, + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":network-interface/*" + ] + ] + } + }, + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:test-region:12345678:log-group:/aws/codebuild/", + { + "Ref": "PipelineBuildSynthCdkBuildProject6BEFA8E6" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:test-region:12345678:log-group:/aws/codebuild/", + { + "Ref": "PipelineBuildSynthCdkBuildProject6BEFA8E6" + }, + ":*" + ] + ] + } + ] + }, + { + "Action": [ + "codebuild:CreateReportGroup", + "codebuild:CreateReport", + "codebuild:UpdateReport", + "codebuild:BatchPutTestCases", + "codebuild:BatchPutCodeCoverages" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":codebuild:test-region:12345678:report-group/", + { + "Ref": "PipelineBuildSynthCdkBuildProject6BEFA8E6" + }, + "-*" + ] + ] + } + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:Abort*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "PipelineArtifactsBucketAEA9A052", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "PipelineArtifactsBucketAEA9A052", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineBuildSynthCdkBuildProjectRoleDefaultPolicyFB6C941C", + "Roles": [ + { + "Ref": "PipelineBuildSynthCdkBuildProjectRole231EEA2A" + } + ] + } + }, + "PipelineBuildSynthCdkBuildProjectSecurityGroup84F92459": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Automatic generated security group for CodeBuild PipelineStackPipelineBuildSynthCdkBuildProject225CEB2C", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "PipelineBuildSynthCdkBuildProject6BEFA8E6": { + "Type": "AWS::CodeBuild::Project", + "Properties": { + "Artifacts": { + "Type": "CODEPIPELINE" + }, + "Environment": { + "ComputeType": "BUILD_GENERAL1_SMALL", + "Image": "aws/codebuild/standard:5.0", + "ImagePullCredentialsType": "CODEBUILD", + "PrivilegedMode": false, + "Type": "LINUX_CONTAINER" + }, + "ServiceRole": { + "Fn::GetAtt": [ + "PipelineBuildSynthCdkBuildProjectRole231EEA2A", + "Arn" + ] + }, + "Source": { + "BuildSpec": "{\n \"version\": \"0.2\",\n \"phases\": {\n \"build\": {\n \"commands\": [\n \"npm ci\",\n \"npm run build\",\n \"npx cdk synth\"\n ]\n }\n },\n \"artifacts\": {\n \"base-directory\": \"cdk.out\",\n \"files\": \"**/*\"\n }\n}", + "Type": "CODEPIPELINE" + }, + "EncryptionKey": "alias/aws/s3", + "VpcConfig": { + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "PipelineBuildSynthCdkBuildProjectSecurityGroup84F92459", + "GroupId" + ] + } + ], + "Subnets": [ + { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + }, + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + }, + { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "DependsOn": [ + "PipelineBuildSynthCdkBuildProjectPolicyDocument4D16371A" + ] + }, + "PipelineBuildSynthCdkBuildProjectPolicyDocument4D16371A": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeDhcpOptions", + "ec2:DescribeVpcs" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineBuildSynthCdkBuildProjectPolicyDocument4D16371A", + "Roles": [ + { + "Ref": "PipelineBuildSynthCdkBuildProjectRole231EEA2A" + } + ] + } + }, + "PipelineBuildSynthCodePipelineActionRole4E7A6C97": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::12345678:root" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "PipelineBuildSynthCodePipelineActionRoleDefaultPolicy92C90290": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "codebuild:BatchGetBuilds", + "codebuild:StartBuild", + "codebuild:StopBuild" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "PipelineBuildSynthCdkBuildProject6BEFA8E6", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineBuildSynthCodePipelineActionRoleDefaultPolicy92C90290", + "Roles": [ + { + "Ref": "PipelineBuildSynthCodePipelineActionRole4E7A6C97" + } + ] + } + }, + "PipelineUpdatePipelineSelfMutateCodePipelineActionRoleD6D4E5CF": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::12345678:root" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "PipelineUpdatePipelineSelfMutateCodePipelineActionRoleDefaultPolicyE626265B": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "codebuild:BatchGetBuilds", + "codebuild:StartBuild", + "codebuild:StopBuild" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "PipelineUpdatePipelineSelfMutationDAA41400", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineUpdatePipelineSelfMutateCodePipelineActionRoleDefaultPolicyE626265B", + "Roles": [ + { + "Ref": "PipelineUpdatePipelineSelfMutateCodePipelineActionRoleD6D4E5CF" + } + ] + } + }, + "PipelineAssetsFileAsset1CodePipelineActionRoleC0EC649A": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::12345678:root" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "PipelineAssetsFileAsset1CodePipelineActionRoleDefaultPolicy5F0BE7E8": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "codebuild:BatchGetBuilds", + "codebuild:StartBuild", + "codebuild:StopBuild" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "PipelineAssetsFileAsset185A67CB4", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineAssetsFileAsset1CodePipelineActionRoleDefaultPolicy5F0BE7E8", + "Roles": [ + { + "Ref": "PipelineAssetsFileAsset1CodePipelineActionRoleC0EC649A" + } + ] + } + }, + "PipelineAssetsFileAsset2CodePipelineActionRole06965A59": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::12345678:root" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "PipelineAssetsFileAsset2CodePipelineActionRoleDefaultPolicy2399F4BC": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "codebuild:BatchGetBuilds", + "codebuild:StartBuild", + "codebuild:StopBuild" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "PipelineAssetsFileAsset24D2D639B", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineAssetsFileAsset2CodePipelineActionRoleDefaultPolicy2399F4BC", + "Roles": [ + { + "Ref": "PipelineAssetsFileAsset2CodePipelineActionRole06965A59" + } + ] + } + }, + "PipelineUpdatePipelineSelfMutationRole57E559E8": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "codebuild.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "PipelineUpdatePipelineSelfMutationRoleDefaultPolicyA225DA4E": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:CreateNetworkInterfacePermission", + "Condition": { + "StringEquals": { + "ec2:Subnet": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":subnet/", + { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":subnet/", + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":subnet/", + { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + ] + ] + } + ], + "ec2:AuthorizedService": "codebuild.amazonaws.com" + } + }, + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":network-interface/*" + ] + ] + } + }, + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:test-region:12345678:log-group:/aws/codebuild/", + { + "Ref": "PipelineUpdatePipelineSelfMutationDAA41400" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:test-region:12345678:log-group:/aws/codebuild/", + { + "Ref": "PipelineUpdatePipelineSelfMutationDAA41400" + }, + ":*" + ] + ] + } + ] + }, + { + "Action": [ + "codebuild:CreateReportGroup", + "codebuild:CreateReport", + "codebuild:UpdateReport", + "codebuild:BatchPutTestCases", + "codebuild:BatchPutCodeCoverages" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":codebuild:test-region:12345678:report-group/", + { + "Ref": "PipelineUpdatePipelineSelfMutationDAA41400" + }, + "-*" + ] + ] + } + }, + { + "Action": "sts:AssumeRole", + "Condition": { + "ForAnyValue:StringEquals": { + "iam:ResourceTag/aws-cdk:bootstrap-role": [ + "image-publishing", + "file-publishing", + "deploy" + ] + } + }, + "Effect": "Allow", + "Resource": "arn:*:iam::12345678:role/*" + }, + { + "Action": "cloudformation:DescribeStacks", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "s3:ListBucket", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "PipelineArtifactsBucketAEA9A052", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "PipelineArtifactsBucketAEA9A052", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineUpdatePipelineSelfMutationRoleDefaultPolicyA225DA4E", + "Roles": [ + { + "Ref": "PipelineUpdatePipelineSelfMutationRole57E559E8" + } + ] + } + }, + "PipelineUpdatePipelineSelfMutationSecurityGroup94164EDC": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Automatic generated security group for CodeBuild PipelineStackPipelineUpdatePipelineSelfMutationE51045FC", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "PipelineUpdatePipelineSelfMutationDAA41400": { + "Type": "AWS::CodeBuild::Project", + "Properties": { + "Artifacts": { + "Type": "CODEPIPELINE" + }, + "Environment": { + "ComputeType": "BUILD_GENERAL1_SMALL", + "Image": "aws/codebuild/standard:5.0", + "ImagePullCredentialsType": "CODEBUILD", + "PrivilegedMode": false, + "Type": "LINUX_CONTAINER" + }, + "ServiceRole": { + "Fn::GetAtt": [ + "PipelineUpdatePipelineSelfMutationRole57E559E8", + "Arn" + ] + }, + "Source": { + "BuildSpec": "{\n \"version\": \"0.2\",\n \"phases\": {\n \"install\": {\n \"commands\": [\n \"npm install -g aws-cdk\"\n ]\n },\n \"build\": {\n \"commands\": [\n \"cdk -a . deploy PipelineStack --require-approval=never --verbose\"\n ]\n }\n }\n}", + "Type": "CODEPIPELINE" + }, + "EncryptionKey": "alias/aws/s3", + "VpcConfig": { + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "PipelineUpdatePipelineSelfMutationSecurityGroup94164EDC", + "GroupId" + ] + } + ], + "Subnets": [ + { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + }, + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + }, + { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "DependsOn": [ + "PipelineUpdatePipelineSelfMutationPolicyDocumentD327DC74" + ] + }, + "PipelineUpdatePipelineSelfMutationPolicyDocumentD327DC74": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeDhcpOptions", + "ec2:DescribeVpcs" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineUpdatePipelineSelfMutationPolicyDocumentD327DC74", + "Roles": [ + { + "Ref": "PipelineUpdatePipelineSelfMutationRole57E559E8" + } + ] + } + }, + "PipelineAssetsFileRole59943A77": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "codebuild.amazonaws.com", + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::12345678:root" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "PipelineAssetsFileRoleDefaultPolicy14DB8755": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:test-region:12345678:log-group:/aws/codebuild/*" + ] + ] + } + }, + { + "Action": [ + "codebuild:CreateReportGroup", + "codebuild:CreateReport", + "codebuild:UpdateReport", + "codebuild:BatchPutTestCases", + "codebuild:BatchPutCodeCoverages" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":codebuild:test-region:12345678:report-group/*" + ] + ] + } + }, + { + "Action": [ + "codebuild:BatchGetBuilds", + "codebuild:StartBuild", + "codebuild:StopBuild" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Resource": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + ] + }, + { + "Action": "ec2:CreateNetworkInterfacePermission", + "Condition": { + "StringEquals": { + "ec2:Subnet": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":subnet/", + { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":subnet/", + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":subnet/", + { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + ] + ] + } + ], + "ec2:AuthorizedService": "codebuild.amazonaws.com" + } + }, + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ec2:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":network-interface/*" + ] + ] + } + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "PipelineArtifactsBucketAEA9A052", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "PipelineArtifactsBucketAEA9A052", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineAssetsFileRoleDefaultPolicy14DB8755", + "Roles": [ + { + "Ref": "PipelineAssetsFileRole59943A77" + } + ] + } + }, + "PipelineAssetsFileAsset1SecurityGroupF04F1AD4": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Automatic generated security group for CodeBuild PipelineStackPipelineAssetsFileAsset10191BEFB", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "PipelineAssetsFileAsset185A67CB4": { + "Type": "AWS::CodeBuild::Project", + "Properties": { + "Artifacts": { + "Type": "CODEPIPELINE" + }, + "Environment": { + "ComputeType": "BUILD_GENERAL1_SMALL", + "Image": "aws/codebuild/standard:5.0", + "ImagePullCredentialsType": "CODEBUILD", + "PrivilegedMode": false, + "Type": "LINUX_CONTAINER" + }, + "ServiceRole": { + "Fn::GetAtt": [ + "PipelineAssetsFileRole59943A77", + "Arn" + ] + }, + "Source": { + "BuildSpec": "{\n \"version\": \"0.2\",\n \"phases\": {\n \"install\": {\n \"commands\": [\n \"npm install -g cdk-assets\"\n ]\n },\n \"build\": {\n \"commands\": [\n \"cdk-assets --path \\\"assembly-PipelineStack-Beta/PipelineStackBetaStack1E6541489.assets.json\\\" --verbose publish \\\"8289faf53c7da377bb2b90615999171adef5e1d8f6b88810e5fef75e6ca09ba5:current_account-current_region\\\"\"\n ]\n }\n }\n}", + "Type": "CODEPIPELINE" + }, + "EncryptionKey": "alias/aws/s3", + "VpcConfig": { + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "PipelineAssetsFileAsset1SecurityGroupF04F1AD4", + "GroupId" + ] + } + ], + "Subnets": [ + { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + }, + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + }, + { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "DependsOn": [ + "PipelineAssetsFileAsset1PolicyDocument4681543E" + ] + }, + "PipelineAssetsFileAsset1PolicyDocument4681543E": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeDhcpOptions", + "ec2:DescribeVpcs" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "PipelineAssetsFileAsset1PolicyDocument4681543E", + "Roles": [ + { + "Ref": "PipelineAssetsFileRole59943A77" + } + ] + } + }, + "PipelineAssetsFileAsset2SecurityGroupA400C1A5": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Automatic generated security group for CodeBuild PipelineStackPipelineAssetsFileAsset24DB856A2", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "PipelineAssetsFileAsset24D2D639B": { + "Type": "AWS::CodeBuild::Project", + "Properties": { + "Artifacts": { + "Type": "CODEPIPELINE" + }, + "Environment": { + "ComputeType": "BUILD_GENERAL1_SMALL", + "Image": "aws/codebuild/standard:5.0", + "ImagePullCredentialsType": "CODEBUILD", + "PrivilegedMode": false, + "Type": "LINUX_CONTAINER" + }, + "ServiceRole": { + "Fn::GetAtt": [ + "PipelineAssetsFileRole59943A77", + "Arn" + ] + }, + "Source": { + "BuildSpec": "{\n \"version\": \"0.2\",\n \"phases\": {\n \"install\": {\n \"commands\": [\n \"npm install -g cdk-assets\"\n ]\n },\n \"build\": {\n \"commands\": [\n \"cdk-assets --path \\\"assembly-PipelineStack-Beta/PipelineStackBetaStack1E6541489.assets.json\\\" --verbose publish \\\"ac76997971c3f6ddf37120660003f1ced72b4fc58c498dfd99c78fa77e721e0e:current_account-current_region\\\"\"\n ]\n }\n }\n}", + "Type": "CODEPIPELINE" + }, + "EncryptionKey": "alias/aws/s3", + "VpcConfig": { + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "PipelineAssetsFileAsset2SecurityGroupA400C1A5", + "GroupId" + ] + } + ], + "Subnets": [ + { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + }, + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + }, + { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + ], + "VpcId": { + "Ref": "Vpc8378EB38" + } + } + }, + "DependsOn": [ + "PipelineAssetsFileAsset1PolicyDocument4681543E" + ] + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store." + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.ts b/packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.ts new file mode 100644 index 0000000000000..590757335081f --- /dev/null +++ b/packages/@aws-cdk/pipelines/test/integ.newpipeline-with-vpc.ts @@ -0,0 +1,57 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +/// !cdk-integ PipelineStack +import * as path from 'path'; +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as s3_assets from '@aws-cdk/aws-s3-assets'; +import * as sqs from '@aws-cdk/aws-sqs'; +import { App, Stack, StackProps, Stage, StageProps } from '@aws-cdk/core'; +import { Construct } from 'constructs'; +import * as pipelines from '../lib'; + +class PipelineStack extends Stack { + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id, props); + + const vpc = new ec2.Vpc(this, 'Vpc'); + + const pipeline = new pipelines.CodePipeline(this, 'Pipeline', { + codeBuildDefaults: { vpc }, + synth: new pipelines.ShellStep('Synth', { + input: pipelines.CodePipelineSource.gitHub('rix0rrr/cdk-pipelines-demo', 'main'), + commands: [ + 'npm ci', + 'npm run build', + 'npx cdk synth', + ], + }), + }); + + pipeline.addStage(new AppStage(this, 'Beta')); + } +} + +class AppStage extends Stage { + constructor(scope: Construct, id: string, props?: StageProps) { + super(scope, id, props); + + const stack = new Stack(this, 'Stack1'); + new s3_assets.Asset(stack, 'Asset', { + path: path.join(__dirname, 'testhelpers/assets/test-file-asset.txt'), + }); + new s3_assets.Asset(stack, 'Asset2', { + path: path.join(__dirname, 'testhelpers/assets/test-file-asset-two.txt'), + }); + + new sqs.Queue(stack, 'OtherQueue'); + } +} + +const app = new App({ + context: { + '@aws-cdk/core:newStyleStackSynthesis': '1', + }, +}); +new PipelineStack(app, 'PipelineStack', { + env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }, +}); +app.synth(); \ No newline at end of file From 774b9ed209558dbb94475ad66d854a907205028a Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Fri, 23 Jul 2021 13:22:16 +0200 Subject: [PATCH 07/15] docs(pipelines): fix table rendering in MarkDown (#15734) GitHub's MarkDown rendering doesn't recognize the `+` in a table column separator. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/pipelines/ORIGINAL_API.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/pipelines/ORIGINAL_API.md b/packages/@aws-cdk/pipelines/ORIGINAL_API.md index 14152b5b7f6d6..119e0037408e8 100644 --- a/packages/@aws-cdk/pipelines/ORIGINAL_API.md +++ b/packages/@aws-cdk/pipelines/ORIGINAL_API.md @@ -17,7 +17,7 @@ Replace `new CdkPipeline` with `new CodePipeline`. Some configuration properties have been changed: | Old API | New API | -|--------------------------------+------------------------------------------------------------------------------------------------| +|--------------------------------|------------------------------------------------------------------------------------------------| | `cloudAssemblyArtifact` | removed | | `sourceAction` | removed | | `synthAction` | `synth` | @@ -96,7 +96,7 @@ potentially `addWave().addStage()`. All stages inside a wave are deployed in parallel, which was not a capability of the original API. | Old API | New API | -|-------------------------------+-------------------------------------------------------------------------------------------------------------------------------| +|-------------------------------|-------------------------------------------------------------------------------------------------------------------------------| | `addApplicationStage()` | `addStage()` | | `addStage().addApplication()` | `addStage()`. Adding multiple CDK Stages into a single Pipeline stage is not supported, add multiple Pipeline stages instead. | From 81cbfec5ddf065aac442d925484a358ee8cd26a1 Mon Sep 17 00:00:00 2001 From: Julian Michel Date: Sat, 24 Jul 2021 00:22:22 +0200 Subject: [PATCH 08/15] fix(elasticsearch): advancedOptions in domain has no effect (#15330) Property `advancedOptions` in ElasticSearch domain did have no effect because the assignment was missing. * add assignment for advancedOptions to fix issue * test cases * describe function in readme Fixes #14067 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-elasticsearch/README.md | 15 ++++ .../@aws-cdk/aws-elasticsearch/lib/domain.ts | 2 + .../aws-elasticsearch/test/domain.test.ts | 28 ++++++++ .../test/integ.elasticsearch.expected.json | 68 +++++++++++++++---- .../test/integ.elasticsearch.ts | 7 +- 5 files changed, 106 insertions(+), 14 deletions(-) diff --git a/packages/@aws-cdk/aws-elasticsearch/README.md b/packages/@aws-cdk/aws-elasticsearch/README.md index 8585ffdd8bc55..7a002f3d2cfde 100644 --- a/packages/@aws-cdk/aws-elasticsearch/README.md +++ b/packages/@aws-cdk/aws-elasticsearch/README.md @@ -290,3 +290,18 @@ new Domain(stack, 'Domain', { It is also possible to specify a custom certificate instead of the auto-generated one. Additionally, an automatic CNAME-Record is created if a hosted zone is provided for the custom endpoint + +## Advanced options + +[Advanced options](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-createupdatedomains.html#es-createdomain-configure-advanced-options) can used to configure additional options. + +```ts +new Domain(stack, 'Domain', { + version: ElasticsearchVersion.V7_7, + advancedOptions: { + 'rest.action.multi.allow_explicit_index': 'false', + 'indices.fielddata.cache.size': '25', + 'indices.query.bool.max_clause_count': '2048', + }, +}); +``` diff --git a/packages/@aws-cdk/aws-elasticsearch/lib/domain.ts b/packages/@aws-cdk/aws-elasticsearch/lib/domain.ts index a29b530128c51..842071fa0ec68 100644 --- a/packages/@aws-cdk/aws-elasticsearch/lib/domain.ts +++ b/packages/@aws-cdk/aws-elasticsearch/lib/domain.ts @@ -410,6 +410,7 @@ export interface DomainProps { /** * Additional options to specify for the Amazon ES domain. * + * @see https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-createupdatedomains.html#es-createdomain-configure-advanced-options * @default - no advanced options are specified */ readonly advancedOptions?: { [key: string]: (string) }; @@ -1678,6 +1679,7 @@ export class Domain extends DomainBase implements IDomain, ec2.IConnectable { }, } : undefined, + advancedOptions: props.advancedOptions, }); this.domain.applyRemovalPolicy(props.removalPolicy); diff --git a/packages/@aws-cdk/aws-elasticsearch/test/domain.test.ts b/packages/@aws-cdk/aws-elasticsearch/test/domain.test.ts index 719341aa29408..6966882c55549 100644 --- a/packages/@aws-cdk/aws-elasticsearch/test/domain.test.ts +++ b/packages/@aws-cdk/aws-elasticsearch/test/domain.test.ts @@ -1633,6 +1633,34 @@ describe('unsigned basic auth', () => { }); }); +describe('advanced options', () => { + test('use advanced options', () => { + new Domain(stack, 'Domain', { + version: ElasticsearchVersion.V7_1, + advancedOptions: { + 'rest.action.multi.allow_explicit_index': 'true', + 'indices.fielddata.cache.size': '50', + }, + }); + + expect(stack).toHaveResourceLike('AWS::Elasticsearch::Domain', { + AdvancedOptions: { + 'rest.action.multi.allow_explicit_index': 'true', + 'indices.fielddata.cache.size': '50', + }, + }); + }); + + test('advanced options absent by default', () => { + new Domain(stack, 'Domain', { + version: ElasticsearchVersion.V7_1, + }); + + expect(stack).toHaveResourceLike('AWS::Elasticsearch::Domain', { + AdvancedOptions: assert.ABSENT, + }); + }); +}); function testGrant( expectedActions: string[], diff --git a/packages/@aws-cdk/aws-elasticsearch/test/integ.elasticsearch.expected.json b/packages/@aws-cdk/aws-elasticsearch/test/integ.elasticsearch.expected.json index 2f43d85055ad1..4299402202092 100644 --- a/packages/@aws-cdk/aws-elasticsearch/test/integ.elasticsearch.expected.json +++ b/packages/@aws-cdk/aws-elasticsearch/test/integ.elasticsearch.expected.json @@ -107,6 +107,11 @@ "Domain19FCBCB91": { "Type": "AWS::Elasticsearch::Domain", "Properties": { + "AdvancedOptions": { + "rest.action.multi.allow_explicit_index": "false", + "indices.fielddata.cache.size": "25", + "indices.query.bool.max_clause_count": "2048" + }, "CognitoOptions": { "Enabled": false }, @@ -206,7 +211,15 @@ { "Ref": "Domain19FCBCB91" }, - "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"*\\\"},\\\"Resource\\\":\\\"*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPath\":\"DomainConfig.ElasticsearchClusterConfig.AccessPolicies\",\"physicalResourceId\":{\"id\":\"", + "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root\\\"},\\\"Resource\\\":\\\"*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPath\":\"DomainConfig.ElasticsearchClusterConfig.AccessPolicies\",\"physicalResourceId\":{\"id\":\"", { "Ref": "Domain19FCBCB91" }, @@ -222,7 +235,15 @@ { "Ref": "Domain19FCBCB91" }, - "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"*\\\"},\\\"Resource\\\":\\\"*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPath\":\"DomainConfig.ElasticsearchClusterConfig.AccessPolicies\",\"physicalResourceId\":{\"id\":\"", + "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root\\\"},\\\"Resource\\\":\\\"*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPath\":\"DomainConfig.ElasticsearchClusterConfig.AccessPolicies\",\"physicalResourceId\":{\"id\":\"", { "Ref": "Domain19FCBCB91" }, @@ -275,7 +296,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters4600faecd25ab407ff0a9d16f935c93062aaea5d415e97046bb8befe6c8ec02cS3BucketD609D0D9" + "Ref": "AssetParameters5c61041c12314e1ad8e67a0107fa3733382a206a78cdc1576fffa7e93caca5b4S3BucketB17E5ABD" }, "S3Key": { "Fn::Join": [ @@ -288,7 +309,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters4600faecd25ab407ff0a9d16f935c93062aaea5d415e97046bb8befe6c8ec02cS3VersionKey77CF589B" + "Ref": "AssetParameters5c61041c12314e1ad8e67a0107fa3733382a206a78cdc1576fffa7e93caca5b4S3VersionKey77778F6A" } ] } @@ -301,7 +322,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters4600faecd25ab407ff0a9d16f935c93062aaea5d415e97046bb8befe6c8ec02cS3VersionKey77CF589B" + "Ref": "AssetParameters5c61041c12314e1ad8e67a0107fa3733382a206a78cdc1576fffa7e93caca5b4S3VersionKey77778F6A" } ] } @@ -432,6 +453,11 @@ "Domain2644FE48C": { "Type": "AWS::Elasticsearch::Domain", "Properties": { + "AdvancedOptions": { + "rest.action.multi.allow_explicit_index": "false", + "indices.fielddata.cache.size": "25", + "indices.query.bool.max_clause_count": "2048" + }, "CognitoOptions": { "Enabled": false }, @@ -531,7 +557,15 @@ { "Ref": "Domain2644FE48C" }, - "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"*\\\"},\\\"Resource\\\":\\\"*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPath\":\"DomainConfig.ElasticsearchClusterConfig.AccessPolicies\",\"physicalResourceId\":{\"id\":\"", + "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root\\\"},\\\"Resource\\\":\\\"*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPath\":\"DomainConfig.ElasticsearchClusterConfig.AccessPolicies\",\"physicalResourceId\":{\"id\":\"", { "Ref": "Domain2644FE48C" }, @@ -547,7 +581,15 @@ { "Ref": "Domain2644FE48C" }, - "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"*\\\"},\\\"Resource\\\":\\\"*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPath\":\"DomainConfig.ElasticsearchClusterConfig.AccessPolicies\",\"physicalResourceId\":{\"id\":\"", + "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root\\\"},\\\"Resource\\\":\\\"*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPath\":\"DomainConfig.ElasticsearchClusterConfig.AccessPolicies\",\"physicalResourceId\":{\"id\":\"", { "Ref": "Domain2644FE48C" }, @@ -566,17 +608,17 @@ } }, "Parameters": { - "AssetParameters4600faecd25ab407ff0a9d16f935c93062aaea5d415e97046bb8befe6c8ec02cS3BucketD609D0D9": { + "AssetParameters5c61041c12314e1ad8e67a0107fa3733382a206a78cdc1576fffa7e93caca5b4S3BucketB17E5ABD": { "Type": "String", - "Description": "S3 bucket for asset \"4600faecd25ab407ff0a9d16f935c93062aaea5d415e97046bb8befe6c8ec02c\"" + "Description": "S3 bucket for asset \"5c61041c12314e1ad8e67a0107fa3733382a206a78cdc1576fffa7e93caca5b4\"" }, - "AssetParameters4600faecd25ab407ff0a9d16f935c93062aaea5d415e97046bb8befe6c8ec02cS3VersionKey77CF589B": { + "AssetParameters5c61041c12314e1ad8e67a0107fa3733382a206a78cdc1576fffa7e93caca5b4S3VersionKey77778F6A": { "Type": "String", - "Description": "S3 key for asset version \"4600faecd25ab407ff0a9d16f935c93062aaea5d415e97046bb8befe6c8ec02c\"" + "Description": "S3 key for asset version \"5c61041c12314e1ad8e67a0107fa3733382a206a78cdc1576fffa7e93caca5b4\"" }, - "AssetParameters4600faecd25ab407ff0a9d16f935c93062aaea5d415e97046bb8befe6c8ec02cArtifactHash86CFA15D": { + "AssetParameters5c61041c12314e1ad8e67a0107fa3733382a206a78cdc1576fffa7e93caca5b4ArtifactHash580E429C": { "Type": "String", - "Description": "Artifact hash for asset \"4600faecd25ab407ff0a9d16f935c93062aaea5d415e97046bb8befe6c8ec02c\"" + "Description": "Artifact hash for asset \"5c61041c12314e1ad8e67a0107fa3733382a206a78cdc1576fffa7e93caca5b4\"" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-elasticsearch/test/integ.elasticsearch.ts b/packages/@aws-cdk/aws-elasticsearch/test/integ.elasticsearch.ts index d2851bd3d47b9..fb112d26390ee 100644 --- a/packages/@aws-cdk/aws-elasticsearch/test/integ.elasticsearch.ts +++ b/packages/@aws-cdk/aws-elasticsearch/test/integ.elasticsearch.ts @@ -24,12 +24,17 @@ class TestStack extends Stack { encryptionAtRest: { enabled: true, }, + advancedOptions: { + 'rest.action.multi.allow_explicit_index': 'false', + 'indices.fielddata.cache.size': '25', + 'indices.query.bool.max_clause_count': '2048', + }, // test the access policies custom resource works accessPolicies: [ new iam.PolicyStatement({ effect: iam.Effect.ALLOW, actions: ['es:ESHttp*'], - principals: [new iam.AnyPrincipal()], + principals: [new iam.AccountRootPrincipal()], resources: ['*'], }), ], From a8b1c471b7058bbf739a1d4f5b4860656ebd5432 Mon Sep 17 00:00:00 2001 From: Niranjan Jayakar Date: Mon, 26 Jul 2021 12:13:52 +0100 Subject: [PATCH 09/15] feat(assertions): retrieve matching resources from the template (#15642) Provide API `findResources()` that retrieves the matching resources from the template given its type and optional predicate. For complex assertions that cannot be modeled using the primitives provided by this module, this API allows an 'escape hatch' so that assertions can be written directly into the test case. This is being used in the `aws-cloudwatch` module, specifically to assert widgets in a CloudWatch Dashboard that are modeled as serialized JSON within a property in the resource. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/assertions/README.md | 6 +- packages/@aws-cdk/assertions/lib/index.ts | 2 +- .../{has-resource.ts => private/resource.ts} | 60 +++++++++++++----- .../lib/{assertions.ts => template.ts} | 13 +++- .../{assertions.test.ts => template.test.ts} | 62 +++++++++++++++++++ packages/@aws-cdk/aws-cloudwatch/package.json | 4 +- .../aws-cloudwatch/test/alarm.test.ts | 27 ++++---- .../test/composite-alarm.test.ts | 4 +- .../test/cross-environment.test.ts | 4 +- .../aws-cloudwatch/test/dashboard.test.ts | 44 ++++++------- .../aws-cloudwatch/test/metric-math.test.ts | 4 +- .../aws-cloudwatch/test/metrics.test.ts | 6 +- 12 files changed, 172 insertions(+), 64 deletions(-) rename packages/@aws-cdk/assertions/lib/{has-resource.ts => private/resource.ts} (53%) rename packages/@aws-cdk/assertions/lib/{assertions.ts => template.ts} (86%) rename packages/@aws-cdk/assertions/test/{assertions.test.ts => template.test.ts} (80%) diff --git a/packages/@aws-cdk/assertions/README.md b/packages/@aws-cdk/assertions/README.md index 5d61312a812cb..1a79f8294cf3b 100644 --- a/packages/@aws-cdk/assertions/README.md +++ b/packages/@aws-cdk/assertions/README.md @@ -62,7 +62,7 @@ in a template. assert.resourceCountIs('Foo::Bar', 2); ``` -## Resource Matching +## Resource Matching & Retrieval Beyond resource counting, the module also allows asserting that a resource with specific properties are present. @@ -88,6 +88,10 @@ assert.hasResource('Foo::Bar', { }); ``` +Beyond assertions, the module provides APIs to retrieve matching resources. +The `findResources()` API is complementary to the `hasResource()` API, except, +instead of asserting its presence, it returns the set of matching resources. + By default, the `hasResource()` and `hasResourceProperties()` APIs perform deep partial object matching. This behavior can be configured using matchers. See subsequent section on [special matchers](#special-matchers). diff --git a/packages/@aws-cdk/assertions/lib/index.ts b/packages/@aws-cdk/assertions/lib/index.ts index 937b59823762c..963039f921bc1 100644 --- a/packages/@aws-cdk/assertions/lib/index.ts +++ b/packages/@aws-cdk/assertions/lib/index.ts @@ -1,3 +1,3 @@ -export * from './assertions'; +export * from './template'; export * from './match'; export * from './matcher'; \ No newline at end of file diff --git a/packages/@aws-cdk/assertions/lib/has-resource.ts b/packages/@aws-cdk/assertions/lib/private/resource.ts similarity index 53% rename from packages/@aws-cdk/assertions/lib/has-resource.ts rename to packages/@aws-cdk/assertions/lib/private/resource.ts index e4b6bf3c1eb06..22a2b01734b70 100644 --- a/packages/@aws-cdk/assertions/lib/has-resource.ts +++ b/packages/@aws-cdk/assertions/lib/private/resource.ts @@ -1,6 +1,20 @@ -import { Match } from './match'; -import { Matcher, MatchResult } from './matcher'; -import { StackInspector } from './vendored/assert'; +import { Match } from '../match'; +import { Matcher, MatchResult } from '../matcher'; +import { StackInspector } from '../vendored/assert'; + +export function findResources(inspector: StackInspector, type: string, props: any = {}): { [key: string]: any }[] { + const matcher = Matcher.isMatcher(props) ? props : Match.objectLike(props); + let results: { [key: string]: any }[] = []; + + eachResourceWithType(inspector, type, (resource) => { + const result = matcher.test(resource); + if (!result.hasFailed()) { + results.push(resource); + } + }); + + return results; +} export function hasResource(inspector: StackInspector, type: string, props: any): string | void { const matcher = Matcher.isMatcher(props) ? props : Match.objectLike(props); @@ -8,19 +22,22 @@ export function hasResource(inspector: StackInspector, type: string, props: any) let closestResource: { [key: string]: any } | undefined = undefined; let count: number = 0; - for (const logicalId of Object.keys(inspector.value.Resources ?? {})) { - const resource: { [key: string]: any } = inspector.value.Resources[logicalId]; - if (resource.Type === type) { - count++; - const result = matcher.test(resource); - if (!result.hasFailed()) { - return; - } - if (closestResult === undefined || closestResult.failCount > result.failCount) { - closestResult = result; - closestResource = resource; - } + let match = false; + eachResourceWithType(inspector, type, (resource) => { + if (match) { return; } + count++; + const result = matcher.test(resource); + if (!result.hasFailed()) { + match = true; + } + if (closestResult === undefined || closestResult.failCount > result.failCount) { + closestResult = result; + closestResource = resource; } + }); + + if (match) { + return; } if (closestResult === undefined) { @@ -37,6 +54,19 @@ export function hasResource(inspector: StackInspector, type: string, props: any) ].join('\n'); } +function eachResourceWithType( + inspector: StackInspector, + type: string, + cb: (resource: {[key: string]: any}) => void): void { + + for (const logicalId of Object.keys(inspector.value.Resources ?? {})) { + const resource: { [key: string]: any } = inspector.value.Resources[logicalId]; + if (resource.Type === type) { + cb(resource); + } + } +} + function formatMessage(closestResult: MatchResult, closestResource: {}): string { return [ 'The closest result is:', diff --git a/packages/@aws-cdk/assertions/lib/assertions.ts b/packages/@aws-cdk/assertions/lib/template.ts similarity index 86% rename from packages/@aws-cdk/assertions/lib/assertions.ts rename to packages/@aws-cdk/assertions/lib/template.ts index aa7eaeb0bfd7d..c76f8150292a2 100644 --- a/packages/@aws-cdk/assertions/lib/assertions.ts +++ b/packages/@aws-cdk/assertions/lib/template.ts @@ -1,7 +1,7 @@ import { Stack, Stage } from '@aws-cdk/core'; -import { hasResource } from './has-resource'; import { Match } from './match'; import { Matcher } from './matcher'; +import { findResources, hasResource } from './private/resource'; import * as assert from './vendored/assert'; /** @@ -82,6 +82,17 @@ export class TemplateAssertions { } } + /** + * Get the set of matching resources of a given type and properties in the CloudFormation template. + * @param type the type to match in the CloudFormation template + * @param props by default, matches all resources with the given type. + * When a literal is provided, performs a partial match via `Match.objectLike()`. + * Use the `Match` APIs to configure a different behaviour. + */ + public findResources(type: string, props: any = {}): { [key: string]: any }[] { + return findResources(this.inspector, type, props); + } + /** * Assert that the CloudFormation template matches the given value * @param expected the expected CloudFormation template as key-value pairs. diff --git a/packages/@aws-cdk/assertions/test/assertions.test.ts b/packages/@aws-cdk/assertions/test/template.test.ts similarity index 80% rename from packages/@aws-cdk/assertions/test/assertions.test.ts rename to packages/@aws-cdk/assertions/test/template.test.ts index bd0cdb828f921..70b09b618446c 100644 --- a/packages/@aws-cdk/assertions/test/assertions.test.ts +++ b/packages/@aws-cdk/assertions/test/template.test.ts @@ -221,6 +221,68 @@ describe('TemplateAssertions', () => { })).toThrow(/No resource/); }); }); + + describe('getResources', () => { + test('matching resource type', () => { + const stack = new Stack(); + new CfnResource(stack, 'Foo', { + type: 'Foo::Bar', + properties: { baz: 'qux', fred: 'waldo' }, + }); + + const inspect = TemplateAssertions.fromStack(stack); + expect(inspect.findResources('Foo::Bar')).toEqual([{ + Type: 'Foo::Bar', + Properties: { baz: 'qux', fred: 'waldo' }, + }]); + }); + + test('no matching resource type', () => { + const stack = new Stack(); + new CfnResource(stack, 'Foo', { + type: 'Foo::Bar', + properties: { baz: 'qux', fred: 'waldo' }, + }); + + const inspect = TemplateAssertions.fromStack(stack); + expect(inspect.findResources('Foo::Baz')).toEqual([]); + }); + + test('matching resource props', () => { + const stack = new Stack(); + new CfnResource(stack, 'Foo', { + type: 'Foo::Bar', + properties: { baz: 'qux', fred: 'waldo' }, + }); + + const inspect = TemplateAssertions.fromStack(stack); + expect(inspect.findResources('Foo::Bar', { + Properties: { baz: 'qux' }, + }).length).toEqual(1); + }); + + test('no matching resource props', () => { + const stack = new Stack(); + new CfnResource(stack, 'Foo', { + type: 'Foo::Bar', + properties: { baz: 'qux', fred: 'waldo' }, + }); + + const inspect = TemplateAssertions.fromStack(stack); + expect(inspect.findResources('Foo::Bar', { + Properties: { baz: 'waldo' }, + })).toEqual([]); + }); + + test('multiple matching resources', () => { + const stack = new Stack(); + new CfnResource(stack, 'Foo', { type: 'Foo::Bar' }); + new CfnResource(stack, 'Bar', { type: 'Foo::Bar' }); + + const inspect = TemplateAssertions.fromStack(stack); + expect(inspect.findResources('Foo::Bar').length).toEqual(2); + }); + }); }); function expectToThrow(fn: () => void, msgs: (RegExp | string)[], done: jest.DoneCallback): void { diff --git a/packages/@aws-cdk/aws-cloudwatch/package.json b/packages/@aws-cdk/aws-cloudwatch/package.json index 246ad8a89ef37..683ca79bde154 100644 --- a/packages/@aws-cdk/aws-cloudwatch/package.json +++ b/packages/@aws-cdk/aws-cloudwatch/package.json @@ -73,13 +73,13 @@ }, "license": "Apache-2.0", "devDependencies": { + "@aws-cdk/assertions": "0.0.0", "@types/jest": "^26.0.24", "cdk-build-tools": "0.0.0", "cdk-integ-tools": "0.0.0", "cfn2ts": "0.0.0", "jest": "^26.6.3", - "pkglint": "0.0.0", - "@aws-cdk/assert-internal": "0.0.0" + "pkglint": "0.0.0" }, "dependencies": { "@aws-cdk/aws-iam": "0.0.0", diff --git a/packages/@aws-cdk/aws-cloudwatch/test/alarm.test.ts b/packages/@aws-cdk/aws-cloudwatch/test/alarm.test.ts index bfe3eca913b67..437dbc5df4ab8 100644 --- a/packages/@aws-cdk/aws-cloudwatch/test/alarm.test.ts +++ b/packages/@aws-cdk/aws-cloudwatch/test/alarm.test.ts @@ -1,5 +1,4 @@ -import '@aws-cdk/assert-internal/jest'; -import { ABSENT } from '@aws-cdk/assert-internal'; +import { Match, TemplateAssertions } from '@aws-cdk/assertions'; import { Duration, Stack } from '@aws-cdk/core'; import { Construct } from 'constructs'; import { Alarm, IAlarm, IAlarmAction, Metric, MathExpression, IMetric } from '../lib'; @@ -68,7 +67,7 @@ describe('Alarm', () => { }); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { ComparisonOperator: 'GreaterThanOrEqualToThreshold', EvaluationPeriods: 3, MetricName: 'Metric', @@ -94,7 +93,7 @@ describe('Alarm', () => { }); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { ComparisonOperator: 'GreaterThanOrEqualToThreshold', EvaluationPeriods: 3, MetricName: 'Metric', @@ -120,14 +119,14 @@ describe('Alarm', () => { }); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { ComparisonOperator: 'GreaterThanOrEqualToThreshold', EvaluationPeriods: 3, MetricName: 'Metric', Namespace: 'CDK/Test', Period: 300, Statistic: 'Maximum', - ExtendedStatistic: ABSENT, + ExtendedStatistic: Match.absentProperty(), Threshold: 1000, }); @@ -147,13 +146,13 @@ describe('Alarm', () => { }); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { ComparisonOperator: 'GreaterThanOrEqualToThreshold', EvaluationPeriods: 3, MetricName: 'Metric', Namespace: 'CDK/Test', Period: 300, - Statistic: ABSENT, + Statistic: Match.absentProperty(), ExtendedStatistic: 'p99', Threshold: 1000, }); @@ -174,7 +173,7 @@ describe('Alarm', () => { }); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { ComparisonOperator: 'GreaterThanOrEqualToThreshold', EvaluationPeriods: 3, DatapointsToAlarm: 2, @@ -204,7 +203,7 @@ describe('Alarm', () => { alarm.addOkAction(new TestAlarmAction('C')); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { AlarmActions: ['A'], InsufficientDataActions: ['B'], OKActions: ['C'], @@ -226,7 +225,7 @@ describe('Alarm', () => { }); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { ComparisonOperator: 'GreaterThanOrEqualToThreshold', EvaluationPeriods: 2, MetricName: 'Metric', @@ -251,7 +250,7 @@ describe('Alarm', () => { }); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { ExtendedStatistic: 'p99.9', }); @@ -270,8 +269,8 @@ describe('Alarm', () => { }); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Alarm', { - Statistic: ABSENT, + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { + Statistic: Match.absentProperty(), ExtendedStatistic: 'tm99.9999999999', }); diff --git a/packages/@aws-cdk/aws-cloudwatch/test/composite-alarm.test.ts b/packages/@aws-cdk/aws-cloudwatch/test/composite-alarm.test.ts index 054f1b21724ee..e77d33a546ca5 100644 --- a/packages/@aws-cdk/aws-cloudwatch/test/composite-alarm.test.ts +++ b/packages/@aws-cdk/aws-cloudwatch/test/composite-alarm.test.ts @@ -1,4 +1,4 @@ -import '@aws-cdk/assert-internal/jest'; +import { TemplateAssertions } from '@aws-cdk/assertions'; import { Stack } from '@aws-cdk/core'; import { Alarm, AlarmRule, AlarmState, CompositeAlarm, Metric } from '../lib'; @@ -59,7 +59,7 @@ describe('CompositeAlarm', () => { alarmRule, }); - expect(stack).toHaveResource('AWS::CloudWatch::CompositeAlarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::CompositeAlarm', { AlarmName: 'CompositeAlarm', AlarmRule: { 'Fn::Join': [ diff --git a/packages/@aws-cdk/aws-cloudwatch/test/cross-environment.test.ts b/packages/@aws-cdk/aws-cloudwatch/test/cross-environment.test.ts index 959ceafab54fc..fc00cda0753d1 100644 --- a/packages/@aws-cdk/aws-cloudwatch/test/cross-environment.test.ts +++ b/packages/@aws-cdk/aws-cloudwatch/test/cross-environment.test.ts @@ -1,4 +1,4 @@ -import '@aws-cdk/assert-internal/jest'; +import { TemplateAssertions } from '@aws-cdk/assertions'; import { Stack } from '@aws-cdk/core'; import { Alarm, GraphWidget, IWidget, Metric } from '../lib'; @@ -89,7 +89,7 @@ describe('cross environment', () => { }); // THEN - expect(stack1).toHaveResourceLike('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack1).hasResourceProperties('AWS::CloudWatch::Alarm', { MetricName: 'ACount', Namespace: 'Test', Period: 300, diff --git a/packages/@aws-cdk/aws-cloudwatch/test/dashboard.test.ts b/packages/@aws-cdk/aws-cloudwatch/test/dashboard.test.ts index 5501a47ba5c3a..cf3c4caa96042 100644 --- a/packages/@aws-cdk/aws-cloudwatch/test/dashboard.test.ts +++ b/packages/@aws-cdk/aws-cloudwatch/test/dashboard.test.ts @@ -1,5 +1,4 @@ -import '@aws-cdk/assert-internal/jest'; -import { isSuperObject } from '@aws-cdk/assert-internal'; +import { TemplateAssertions } from '@aws-cdk/assertions'; import { App, Stack } from '@aws-cdk/core'; import { Dashboard, GraphWidget, PeriodOverride, TextWidget } from '../lib'; @@ -27,11 +26,13 @@ describe('Dashboard', () => { })); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Dashboard', thatHasWidgets([ + const resources = TemplateAssertions.fromStack(stack).findResources('AWS::CloudWatch::Dashboard'); + expect(resources.length).toEqual(1); + hasWidgets(resources[0].Properties, [ { type: 'text', width: 10, height: 2, x: 0, y: 0, properties: { markdown: 'first' } }, { type: 'text', width: 1, height: 4, x: 0, y: 2, properties: { markdown: 'second' } }, { type: 'text', width: 4, height: 1, x: 0, y: 6, properties: { markdown: 'third' } }, - ])); + ]); }); @@ -61,11 +62,13 @@ describe('Dashboard', () => { ); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Dashboard', thatHasWidgets([ + const resources = TemplateAssertions.fromStack(stack).findResources('AWS::CloudWatch::Dashboard'); + expect(resources.length).toEqual(1); + hasWidgets(resources[0].Properties, [ { type: 'text', width: 10, height: 2, x: 0, y: 0, properties: { markdown: 'first' } }, { type: 'text', width: 1, height: 4, x: 10, y: 0, properties: { markdown: 'second' } }, { type: 'text', width: 4, height: 1, x: 11, y: 0, properties: { markdown: 'third' } }, - ])); + ]); }); @@ -81,7 +84,7 @@ describe('Dashboard', () => { ); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Dashboard', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Dashboard', { DashboardBody: { 'Fn::Join': ['', [ '{"widgets":[{"type":"metric","width":1,"height":1,"x":0,"y":0,"properties":{"view":"timeSeries","region":"', @@ -110,7 +113,7 @@ describe('Dashboard', () => { ); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Dashboard', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Dashboard', { DashboardBody: { 'Fn::Join': ['', [ '{"start":"-9H","end":"2018-12-17T06:00:00.000Z","periodOverride":"inherit",\ @@ -135,7 +138,7 @@ describe('Dashboard', () => { }); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Dashboard', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Dashboard', { DashboardName: 'MyCustomDashboardName', }); @@ -151,7 +154,7 @@ describe('Dashboard', () => { new Dashboard(stack, 'MyDashboard'); // THEN - expect(stack).toHaveResource('AWS::CloudWatch::Dashboard', {}); + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Dashboard', {}); }); @@ -178,15 +181,14 @@ describe('Dashboard', () => { /** * Returns a property predicate that checks that the given Dashboard has the indicated widgets */ -function thatHasWidgets(widgets: any): (props: any) => boolean { - return (props: any) => { - try { - const actualWidgets = JSON.parse(props.DashboardBody).widgets; - return isSuperObject(actualWidgets, widgets); - } catch (e) { - // eslint-disable-next-line no-console - console.error('Error parsing', props); - throw e; - } - }; +function hasWidgets(props: any, widgets: any[]) { + let actualWidgets: any[] = []; + try { + actualWidgets = JSON.parse(props.DashboardBody).widgets; + } catch (e) { + // eslint-disable-next-line no-console + console.error('Error parsing', props); + throw e; + } + expect(actualWidgets).toEqual(expect.arrayContaining(widgets)); } diff --git a/packages/@aws-cdk/aws-cloudwatch/test/metric-math.test.ts b/packages/@aws-cdk/aws-cloudwatch/test/metric-math.test.ts index b9379bbdc360c..52e7803575d05 100644 --- a/packages/@aws-cdk/aws-cloudwatch/test/metric-math.test.ts +++ b/packages/@aws-cdk/aws-cloudwatch/test/metric-math.test.ts @@ -1,4 +1,4 @@ -import '@aws-cdk/assert-internal/jest'; +import { TemplateAssertions } from '@aws-cdk/assertions'; import { Duration, Stack } from '@aws-cdk/core'; import { Alarm, GraphWidget, IWidget, MathExpression, Metric } from '../lib'; @@ -638,7 +638,7 @@ function graphMetricsAre(w: IWidget, metrics: any[]) { } function alarmMetricsAre(metrics: any[]) { - expect(stack).toHaveResourceLike('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { Metrics: metrics, }); } diff --git a/packages/@aws-cdk/aws-cloudwatch/test/metrics.test.ts b/packages/@aws-cdk/aws-cloudwatch/test/metrics.test.ts index ff1620f91ed50..f7b5c69f8018b 100644 --- a/packages/@aws-cdk/aws-cloudwatch/test/metrics.test.ts +++ b/packages/@aws-cdk/aws-cloudwatch/test/metrics.test.ts @@ -1,4 +1,4 @@ -import '@aws-cdk/assert-internal/jest'; +import { TemplateAssertions } from '@aws-cdk/assertions'; import * as iam from '@aws-cdk/aws-iam'; import * as cdk from '@aws-cdk/core'; import { Alarm, Metric } from '../lib'; @@ -15,7 +15,7 @@ describe('Metrics', () => { Metric.grantPutMetricData(role); // THEN - expect(stack).toHaveResource('AWS::IAM::Policy', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { PolicyDocument: { Version: '2012-10-17', Statement: [ @@ -188,7 +188,7 @@ describe('Metrics', () => { dimensionA: 'value1', dimensionB: 'value2', }); - expect(stack).toHaveResourceLike('AWS::CloudWatch::Alarm', { + TemplateAssertions.fromStack(stack).hasResourceProperties('AWS::CloudWatch::Alarm', { Namespace: 'Test', MetricName: 'Metric', Dimensions: [ From 807351857db014ccda09718f88a8c51732cd0f1a Mon Sep 17 00:00:00 2001 From: Wouter Klijn Date: Mon, 26 Jul 2021 17:26:18 +0200 Subject: [PATCH 10/15] chore(rds): add SQL Server version 14.00.3381.3.v1 (#15752) According to the [documentation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_SQLServer.html#SQLServer.Concepts.General.VersionSupport) on supported SQL Server versions for AWS RDS, `14.00.3381.3.v1` should be a supported engine version. It wasn't yet available in the CDK; this merge request should change that. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-rds/lib/instance-engine.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/@aws-cdk/aws-rds/lib/instance-engine.ts b/packages/@aws-cdk/aws-rds/lib/instance-engine.ts index 2921d828f37ba..7b1c8fad83eaf 100644 --- a/packages/@aws-cdk/aws-rds/lib/instance-engine.ts +++ b/packages/@aws-cdk/aws-rds/lib/instance-engine.ts @@ -1343,6 +1343,8 @@ export class SqlServerEngineVersion { public static readonly VER_14_00_3294_2_V1 = SqlServerEngineVersion.of('14.00.3294.2.v1', '14.00'); /** Version "14.00.3356.20.v1". */ public static readonly VER_14_00_3356_20_V1 = SqlServerEngineVersion.of('14.00.3356.20.v1', '14.00'); + /** Version "14.00.3381.3.v1". */ + public static readonly VER_14_00_3381_3_V1 = SqlServerEngineVersion.of('14.00.3381.3.v1', '14.00'); /** Version "15.00" (only a major version, without a specific minor version). */ public static readonly VER_15 = SqlServerEngineVersion.of('15.00', '15.00'); From d9285cb75745028ede8c36afcee34f7a53d27993 Mon Sep 17 00:00:00 2001 From: Michael Sambol Date: Mon, 26 Jul 2021 12:48:59 -0700 Subject: [PATCH 11/15] feat(stepfunctions): allow intrinsic functions for json path (#15320) `QueryString` can start with an [intrinsic function](https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-intrinsic-functions.html). Here is an example of a working `QueryString`: ``` "QueryString.$": "States.Format('select contact_id from interactions.conversation where case when \\'{}\\' is not null then bu=\\'{}\\' else 1=1 end and case when \\'{}\\' is not null then channel=\\'{}\\' else 1=1 end and case when {} is not null then year={} else 1=1 end and case when {} is not null then month={} else 1=1 end and case when {} is not null then day={} else 1=1 end limit 1;', States.JsonToString($.completed), $.partitions.bu, $.partitions.bu, $.partitions.channel, $.partitions.channel, $.partitions.year, $.partitions.year, $.partitions.month, $.partitions.month, $.partitions.day, $.partitions.day)" ``` --- .../aws-stepfunctions-tasks/README.md | 22 +++++++++++++++++++ .../@aws-cdk/aws-stepfunctions/lib/fields.ts | 3 ++- .../aws-stepfunctions/test/fields.test.ts | 7 +++++- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/README.md b/packages/@aws-cdk/aws-stepfunctions-tasks/README.md index bc73260059d4e..0ee1cbaba53f9 100644 --- a/packages/@aws-cdk/aws-stepfunctions-tasks/README.md +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/README.md @@ -205,6 +205,28 @@ const submitJob = new tasks.LambdaInvoke(this, 'Invoke Handler', { }); ``` +You can also use [intrinsic functions](https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-intrinsic-functions.html) with `JsonPath.stringAt()`. +Here is an example of starting an Athena query that is dynamically created using the task input: + +```ts +const startQueryExecutionJob = new tasks.AthenaStartQueryExecution(this, 'Athena Start Query', { + queryString: sfn.JsonPath.stringAt("States.Format('select contacts where year={};', $.year)"), + queryExecutionContext: { + databaseName: 'interactions', + }, + resultConfiguration: { + encryptionConfiguration: { + encryptionOption: tasks.EncryptionOption.S3_MANAGED, + }, + outputLocation: { + bucketName: 'mybucket', + objectKey: 'myprefix', + }, + }, + integrationPattern: sfn.IntegrationPattern.RUN_JOB, +}); +``` + Each service integration has its own set of parameters that can be supplied. ## Evaluate Expression diff --git a/packages/@aws-cdk/aws-stepfunctions/lib/fields.ts b/packages/@aws-cdk/aws-stepfunctions/lib/fields.ts index 9064fb93d2b6e..990a2542d4fea 100644 --- a/packages/@aws-cdk/aws-stepfunctions/lib/fields.ts +++ b/packages/@aws-cdk/aws-stepfunctions/lib/fields.ts @@ -219,8 +219,9 @@ function validateJsonPath(path: string) { && path !== '$$' && !path.startsWith('$$.') && !path.startsWith('$[') + && ['Format', 'StringToJson', 'JsonToString', 'Array'].every(fn => !path.startsWith(`States.${fn}`)) ) { - throw new Error(`JSON path values must be exactly '$', '$$', start with '$.', start with '$$.' or start with '$[' Received: ${path}`); + throw new Error(`JSON path values must be exactly '$', '$$', start with '$.', start with '$$.', start with '$[', or start with an intrinsic function: States.Format, States.StringToJson, States.JsonToString, or States.Array. Received: ${path}`); } } diff --git a/packages/@aws-cdk/aws-stepfunctions/test/fields.test.ts b/packages/@aws-cdk/aws-stepfunctions/test/fields.test.ts index 382ec424177f9..85549614a0538 100644 --- a/packages/@aws-cdk/aws-stepfunctions/test/fields.test.ts +++ b/packages/@aws-cdk/aws-stepfunctions/test/fields.test.ts @@ -2,7 +2,7 @@ import '@aws-cdk/assert-internal/jest'; import { FieldUtils, JsonPath, TaskInput } from '../lib'; describe('Fields', () => { - const jsonPathValidationErrorMsg = /exactly '\$', '\$\$', start with '\$.', start with '\$\$.' or start with '\$\['/; + const jsonPathValidationErrorMsg = /exactly '\$', '\$\$', start with '\$.', start with '\$\$.', start with '\$\[', or start with an intrinsic function: States.Format, States.StringToJson, States.JsonToString, or States.Array./; test('deep replace correctly handles fields in arrays', () => { expect( @@ -71,9 +71,14 @@ describe('Fields', () => { }), test('datafield path must be correct', () => { expect(JsonPath.stringAt('$')).toBeDefined(); + expect(JsonPath.stringAt('States.Format')).toBeDefined(); + expect(JsonPath.stringAt('States.StringToJson')).toBeDefined(); + expect(JsonPath.stringAt('States.JsonToString')).toBeDefined(); + expect(JsonPath.stringAt('States.Array')).toBeDefined(); expect(() => JsonPath.stringAt('$hello')).toThrowError(jsonPathValidationErrorMsg); expect(() => JsonPath.stringAt('hello')).toThrowError(jsonPathValidationErrorMsg); + expect(() => JsonPath.stringAt('States.FooBar')).toThrowError(jsonPathValidationErrorMsg); }), test('context path must be correct', () => { expect(JsonPath.stringAt('$$')).toBeDefined(); From 6f2384ddc180e944c9564a543351b8df2f75c1a7 Mon Sep 17 00:00:00 2001 From: Jackie Yuan <32397155+syeehyn@users.noreply.github.com> Date: Mon, 26 Jul 2021 16:27:38 -0400 Subject: [PATCH 12/15] fix(stepfunctions-tasks): instance type cannot be provided to SageMakerCreateTransformJob as input path (#15726) as referred in https://github.com/aws/aws-cdk/issues/11605 in SageMakerCreateTransformJob has the same kind of issue. similar solution can be found at https://github.com/aws/aws-cdk/pull/11749 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../lib/sagemaker/create-transform-job.ts | 3 +- .../sagemaker/create-transform-job.test.ts | 58 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/sagemaker/create-transform-job.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/sagemaker/create-transform-job.ts index e1b5aabfc61b6..cc108c45e4439 100644 --- a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/sagemaker/create-transform-job.ts +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/sagemaker/create-transform-job.ts @@ -227,7 +227,8 @@ export class SageMakerCreateTransformJob extends sfn.TaskStateBase { return { TransformResources: { InstanceCount: resources.instanceCount, - InstanceType: 'ml.' + resources.instanceType, + InstanceType: sfn.JsonPath.isEncodedJsonPath(resources.instanceType.toString()) + ? resources.instanceType.toString() : `ml.${resources.instanceType}`, ...(resources.volumeEncryptionKey ? { VolumeKmsKeyId: resources.volumeEncryptionKey.keyArn } : {}), }, }; diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/sagemaker/create-transform-job.test.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/test/sagemaker/create-transform-job.test.ts index cf545b3e1defb..16f499e53708c 100644 --- a/packages/@aws-cdk/aws-stepfunctions-tasks/test/sagemaker/create-transform-job.test.ts +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/sagemaker/create-transform-job.test.ts @@ -249,3 +249,61 @@ test('pass param to transform job', () => { }, }); }); +test('create transform job with instance type supplied as JsonPath', () => { + // WHEN + const task = new SageMakerCreateTransformJob(stack, 'TransformTask', { + transformJobName: 'MyTransformJob', + modelName: 'MyModelName', + transformInput: { + transformDataSource: { + s3DataSource: { + s3Uri: 's3://inputbucket/prefix', + }, + }, + }, + transformOutput: { + s3OutputPath: 's3://outputbucket/prefix', + }, + transformResources: { + instanceCount: 1, + instanceType: new ec2.InstanceType(sfn.JsonPath.stringAt('$.InstanceType')), + }, + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toEqual({ + Type: 'Task', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':states:::sagemaker:createTransformJob', + ], + ], + }, + End: true, + Parameters: { + TransformJobName: 'MyTransformJob', + ModelName: 'MyModelName', + TransformInput: { + DataSource: { + S3DataSource: { + S3Uri: 's3://inputbucket/prefix', + S3DataType: 'S3Prefix', + }, + }, + }, + TransformOutput: { + S3OutputPath: 's3://outputbucket/prefix', + }, + TransformResources: { + 'InstanceCount': 1, + 'InstanceType.$': '$.InstanceType', + }, + }, + }); +}); From 9451b33a65f4d83b5bb706ac03a89386e061d2d9 Mon Sep 17 00:00:00 2001 From: Otavio Macedo Date: Tue, 27 Jul 2021 09:25:55 +0100 Subject: [PATCH 13/15] chore(s3): additional documentation for addToResourcePolicy. (#15761) This is to clarify that in some cases the policy will not be added and the result should be checked. Closes #6548 and #7370. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-s3/README.md | 27 ++++++++++++++++++-- packages/@aws-cdk/aws-s3/lib/bucket.ts | 35 ++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-s3/README.md b/packages/@aws-cdk/aws-s3/README.md index 6a266560f65e4..8b60ec35eb766 100644 --- a/packages/@aws-cdk/aws-s3/README.md +++ b/packages/@aws-cdk/aws-s3/README.md @@ -92,13 +92,36 @@ A bucket policy will be automatically created for the bucket upon the first call ```ts const bucket = new Bucket(this, 'MyBucket'); -bucket.addToResourcePolicy(new iam.PolicyStatement({ +const result = bucket.addToResourcePolicy(new iam.PolicyStatement({ actions: ['s3:GetObject'], resources: [bucket.arnForObjects('file.txt')], principals: [new iam.AccountRootPrincipal()], })); ``` +If you try to add a policy statement to an existing bucket, this method will +not do anything: + +```ts +const bucket = Bucket.fromBucketName(this, 'existingBucket', 'bucket-name'); + +// Nothing will change here +const result = bucket.addToResourcePolicy(new iam.PolicyStatement({ + ... +})); +``` + +That's because it's not possible to tell whether the bucket +already has a policy attached, let alone to re-use that policy to add more +statements to it. We recommend that you always check the result of the call: + +```ts +const result = bucket.addToResourcePolicy(...) +if (!result.statementAdded) { + // Uh-oh! Someone probably made a mistake here. +} +``` + The bucket policy can be directly accessed after creation to add statements or adjust the removal policy. @@ -150,7 +173,7 @@ const bucket = Bucket.fromBucketAttributes(this, 'ImportedBucket', { }); // now you can just call methods on the bucket -bucket.grantReadWrite(user); +bucket.addEventNotification(EventType.OBJECT_CREATED, ...); ``` Alternatively, short-hand factories are available as `Bucket.fromBucketName` and diff --git a/packages/@aws-cdk/aws-s3/lib/bucket.ts b/packages/@aws-cdk/aws-s3/lib/bucket.ts index b4f549f5feef0..970710c9a3dc4 100644 --- a/packages/@aws-cdk/aws-s3/lib/bucket.ts +++ b/packages/@aws-cdk/aws-s3/lib/bucket.ts @@ -82,9 +82,23 @@ export interface IBucket extends IResource { /** * Adds a statement to the resource policy for a principal (i.e. - * account/role/service) to perform actions on this bucket and/or it's + * account/role/service) to perform actions on this bucket and/or its * contents. Use `bucketArn` and `arnForObjects(keys)` to obtain ARNs for * this bucket or objects. + * + * Note that the policy statement may or may not be added to the policy. + * For example, when an `IBucket` is created from an existing bucket, + * it's not possible to tell whether the bucket already has a policy + * attached, let alone to re-use that policy to add more statements to it. + * So it's safest to do nothing in these cases. + * + * @param permission the policy statement to be added to the bucket's + * policy. + * @returns metadata about the execution of this method. If the policy + * was not added, the value of `statementAdded` will be `false`. You + * should always check this value to make sure that the operation was + * actually carried out. Otherwise, synthesis and deploy will terminate + * silently, which may be confusing. */ addToResourcePolicy(permission: iam.PolicyStatement): iam.AddToResourcePolicyResult; @@ -546,9 +560,23 @@ export abstract class BucketBase extends Resource implements IBucket { /** * Adds a statement to the resource policy for a principal (i.e. - * account/role/service) to perform actions on this bucket and/or it's + * account/role/service) to perform actions on this bucket and/or its * contents. Use `bucketArn` and `arnForObjects(keys)` to obtain ARNs for * this bucket or objects. + * + * Note that the policy statement may or may not be added to the policy. + * For example, when an `IBucket` is created from an existing bucket, + * it's not possible to tell whether the bucket already has a policy + * attached, let alone to re-use that policy to add more statements to it. + * So it's safest to do nothing in these cases. + * + * @param permission the policy statement to be added to the bucket's + * policy. + * @returns metadata about the execution of this method. If the policy + * was not added, the value of `statementAdded` will be `false`. You + * should always check this value to make sure that the operation was + * actually carried out. Otherwise, synthesis and deploy will terminate + * silently, which may be confusing. */ public addToResourcePolicy(permission: iam.PolicyStatement): iam.AddToResourcePolicyResult { if (!this.policy && this.autoCreatePolicy) { @@ -720,6 +748,9 @@ export abstract class BucketBase extends Resource implements IBucket { * const grant = bucket.grantPublicAccess(); * grant.resourceStatement!.addCondition(‘IpAddress’, { “aws:SourceIp”: “54.240.143.0/24” }); * + * Note that if this `IBucket` refers to an existing bucket, possibly not + * managed by CloudFormation, this method will have no effect, since it's + * impossible to modify the policy of an existing bucket. * * @param keyPrefix the prefix of S3 object keys (e.g. `home/*`). Default is "*". * @param allowedActions the set of S3 actions to allow. Default is "s3:GetObject". From c6fa1318b45ed4d3db03766538865260862e50fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 13:54:25 +0200 Subject: [PATCH 14/15] chore(deps): bump actions/setup-node from 2.2.0 to 2.3.0 (#15759) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v2.2.0...v2.3.0) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/yarn-upgrade.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/yarn-upgrade.yml b/.github/workflows/yarn-upgrade.yml index bc42c35967a1d..918b691544287 100644 --- a/.github/workflows/yarn-upgrade.yml +++ b/.github/workflows/yarn-upgrade.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v2 - name: Set up Node - uses: actions/setup-node@v2.2.0 + uses: actions/setup-node@v2.3.0 with: node-version: 10 From 2cefe57835391155aa01962f01fe13fbf7b85c3a Mon Sep 17 00:00:00 2001 From: mokocm <52817048+mokocm@users.noreply.github.com> Date: Wed, 28 Jul 2021 00:50:28 +0900 Subject: [PATCH 15/15] chore(rds): add Aurora Postgres engine versions 10.16 and 11.11 (#15781) Amazon Aurora is available in 10.16 and 11.11, but I added it because it was not available in the CDK. https://aws.amazon.com/about-aws/whats-new/2021/06/amazon-aurora-supports-postgresql-12-6-11-11-10-16-and-9-6-21/ ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-rds/lib/cluster-engine.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/@aws-cdk/aws-rds/lib/cluster-engine.ts b/packages/@aws-cdk/aws-rds/lib/cluster-engine.ts index 2a9c3b378e1fd..f92e738b2f90b 100644 --- a/packages/@aws-cdk/aws-rds/lib/cluster-engine.ts +++ b/packages/@aws-cdk/aws-rds/lib/cluster-engine.ts @@ -447,6 +447,8 @@ export class AuroraPostgresEngineVersion { public static readonly VER_10_13 = AuroraPostgresEngineVersion.of('10.13', '10', { s3Import: true, s3Export: true }); /** Version "10.14". */ public static readonly VER_10_14 = AuroraPostgresEngineVersion.of('10.14', '10', { s3Import: true, s3Export: true }); + /** Version "10.16". */ + public static readonly VER_10_16 = AuroraPostgresEngineVersion.of('10.16', '10', { s3Import: true, s3Export: true }); /** Version "11.4". */ public static readonly VER_11_4 = AuroraPostgresEngineVersion.of('11.4', '11', { s3Import: true }); /** Version "11.6". */ @@ -457,6 +459,8 @@ export class AuroraPostgresEngineVersion { public static readonly VER_11_8 = AuroraPostgresEngineVersion.of('11.8', '11', { s3Import: true, s3Export: true }); /** Version "11.9". */ public static readonly VER_11_9 = AuroraPostgresEngineVersion.of('11.9', '11', { s3Import: true, s3Export: true }); + /** Version "11.11". */ + public static readonly VER_11_11 = AuroraPostgresEngineVersion.of('11.11', '11', { s3Import: true, s3Export: true }); /** Version "12.4". */ public static readonly VER_12_4 = AuroraPostgresEngineVersion.of('12.4', '12', { s3Import: true, s3Export: true }); /** Version "12.6". */