diff --git a/packages/@aws-cdk/assets/lib/asset.ts b/packages/@aws-cdk/assets/lib/asset.ts index a3fc81ed107e3..92e010de59983 100644 --- a/packages/@aws-cdk/assets/lib/asset.ts +++ b/packages/@aws-cdk/assets/lib/asset.ts @@ -67,6 +67,11 @@ export class Asset extends cdk.Construct { private readonly bucket: s3.BucketRef; + /** + * The S3 prefix where all different versions of this asset are stored + */ + private readonly s3Prefix: cdk.Token; + constructor(parent: cdk.Construct, id: string, props: GenericAssetProps) { super(parent, id); @@ -84,15 +89,15 @@ export class Asset extends cdk.Construct { description: `S3 bucket for asset "${this.path}"`, }); - const keyParam = new cdk.Parameter(this, 'S3ObjectKey', { + const keyParam = new cdk.Parameter(this, 'S3VersionKey', { type: 'String', - description: `S3 object for asset "${this.path}"` + description: `S3 key for asset version "${this.path}"` }); this.s3BucketName = bucketParam.value; - this.s3ObjectKey = keyParam.value; - - // grant the lambda's role read permissions on the code s3 object + this.s3Prefix = new cdk.FnSelect(0, new cdk.FnSplit(cxapi.ASSET_PREFIX_SEPARATOR, keyParam.value)); + const s3Filename = new cdk.FnSelect(1, new cdk.FnSplit(cxapi.ASSET_PREFIX_SEPARATOR, keyParam.value)); + this.s3ObjectKey = new cdk.FnConcat(this.s3Prefix, s3Filename); this.bucket = s3.BucketRef.import(parent, 'AssetBucket', { bucketName: this.s3BucketName @@ -105,9 +110,9 @@ export class Asset extends cdk.Construct { // for tooling to be able to package and upload a directory to the // s3 bucket and plug in the bucket name and key in the correct // parameters. - const asset: cxapi.AssetMetadataEntry = { path: this.assetPath, + id: this.uniqueId, packaging: props.packaging, s3BucketParameter: bucketParam.logicalId, s3KeyParameter: keyParam.logicalId, @@ -124,7 +129,11 @@ export class Asset extends cdk.Construct { * Grants read permissions to the principal on the asset's S3 object. */ public grantRead(principal?: iam.IPrincipal) { - this.bucket.grantRead(principal, this.s3ObjectKey); + // We give permissions on all files with the same prefix. Presumably + // different versions of the same file will have the same prefix + // and we don't want to accidentally revoke permission on old versions + // when deploying a new version. + this.bucket.grantRead(principal, `${this.s3Prefix}*`); } } diff --git a/packages/@aws-cdk/assets/test/integ.assets.directory.lit.expected.json b/packages/@aws-cdk/assets/test/integ.assets.directory.lit.expected.json index e3ca3797ddd0c..0b84b721fffae 100644 --- a/packages/@aws-cdk/assets/test/integ.assets.directory.lit.expected.json +++ b/packages/@aws-cdk/assets/test/integ.assets.directory.lit.expected.json @@ -4,9 +4,9 @@ "Type": "String", "Description": "S3 bucket for asset \"aws-cdk-asset-test/SampleAsset\"" }, - "SampleAssetS3ObjectKey6F5D200B": { + "SampleAssetS3VersionKey3E106D34": { "Type": "String", - "Description": "S3 object for asset \"aws-cdk-asset-test/SampleAsset\"" + "Description": "S3 key for asset version \"aws-cdk-asset-test/SampleAsset\"" } }, "Resources": { @@ -76,7 +76,25 @@ }, "/", { - "Ref": "SampleAssetS3ObjectKey6F5D200B" + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "SampleAssetS3VersionKey3E106D34" + } + ] + } + ] + }, + "*" + ] + ] } ] ] diff --git a/packages/@aws-cdk/assets/test/integ.assets.file.lit.expected.json b/packages/@aws-cdk/assets/test/integ.assets.file.lit.expected.json index 99d5cee84363e..c62111674435a 100644 --- a/packages/@aws-cdk/assets/test/integ.assets.file.lit.expected.json +++ b/packages/@aws-cdk/assets/test/integ.assets.file.lit.expected.json @@ -4,9 +4,9 @@ "Type": "String", "Description": "S3 bucket for asset \"aws-cdk-asset-file-test/SampleAsset\"" }, - "SampleAssetS3ObjectKey6F5D200B": { + "SampleAssetS3VersionKey3E106D34": { "Type": "String", - "Description": "S3 object for asset \"aws-cdk-asset-file-test/SampleAsset\"" + "Description": "S3 key for asset version \"aws-cdk-asset-file-test/SampleAsset\"" } }, "Resources": { @@ -76,7 +76,25 @@ }, "/", { - "Ref": "SampleAssetS3ObjectKey6F5D200B" + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "SampleAssetS3VersionKey3E106D34" + } + ] + } + ] + }, + "*" + ] + ] } ] ] diff --git a/packages/@aws-cdk/assets/test/integ.assets.permissions.lit.expected.json b/packages/@aws-cdk/assets/test/integ.assets.permissions.lit.expected.json index e7540be27bb73..c39bebcc10145 100644 --- a/packages/@aws-cdk/assets/test/integ.assets.permissions.lit.expected.json +++ b/packages/@aws-cdk/assets/test/integ.assets.permissions.lit.expected.json @@ -4,9 +4,9 @@ "Type": "String", "Description": "S3 bucket for asset \"aws-cdk-asset-refs/MyFile\"" }, - "MyFileS3ObjectKey4641930D": { + "MyFileS3VersionKey568C3C9F": { "Type": "String", - "Description": "S3 object for asset \"aws-cdk-asset-refs/MyFile\"" + "Description": "S3 key for asset version \"aws-cdk-asset-refs/MyFile\"" } }, "Resources": { @@ -76,7 +76,25 @@ }, "/", { - "Ref": "MyFileS3ObjectKey4641930D" + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "MyFileS3VersionKey568C3C9F" + } + ] + } + ] + }, + "*" + ] + ] } ] ] diff --git a/packages/@aws-cdk/assets/test/integ.assets.refs.lit.expected.json b/packages/@aws-cdk/assets/test/integ.assets.refs.lit.expected.json index 659b218855cc3..6f239e31e2a97 100644 --- a/packages/@aws-cdk/assets/test/integ.assets.refs.lit.expected.json +++ b/packages/@aws-cdk/assets/test/integ.assets.refs.lit.expected.json @@ -4,9 +4,9 @@ "Type": "String", "Description": "S3 bucket for asset \"aws-cdk-asset-refs/SampleAsset\"" }, - "SampleAssetS3ObjectKey6F5D200B": { + "SampleAssetS3VersionKey3E106D34": { "Type": "String", - "Description": "S3 object for asset \"aws-cdk-asset-refs/SampleAsset\"" + "Description": "S3 key for asset version \"aws-cdk-asset-refs/SampleAsset\"" } }, "Outputs": { @@ -20,7 +20,37 @@ }, "S3ObjectKey": { "Value": { - "Ref": "SampleAssetS3ObjectKey6F5D200B" + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "SampleAssetS3VersionKey3E106D34" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "SampleAssetS3VersionKey3E106D34" + } + ] + } + ] + } + ] + ] }, "Export": { "Name": "aws-cdk-asset-refs:S3ObjectKey" @@ -46,7 +76,30 @@ }, "/", { - "Ref": "SampleAssetS3ObjectKey6F5D200B" + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "SampleAssetS3VersionKey3E106D34" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "SampleAssetS3VersionKey3E106D34" + } + ] + } + ] } ] ] @@ -123,7 +176,25 @@ }, "/", { - "Ref": "SampleAssetS3ObjectKey6F5D200B" + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "SampleAssetS3VersionKey3E106D34" + } + ] + } + ] + }, + "*" + ] + ] } ] ] diff --git a/packages/@aws-cdk/assets/test/test.asset.ts b/packages/@aws-cdk/assets/test/test.asset.ts index eee3edd0c7686..e78d02c18fa34 100644 --- a/packages/@aws-cdk/assets/test/test.asset.ts +++ b/packages/@aws-cdk/assets/test/test.asset.ts @@ -1,4 +1,4 @@ -import { expect } from '@aws-cdk/assert'; +import { expect, haveResource } from '@aws-cdk/assert'; import iam = require('@aws-cdk/aws-iam'); import cdk = require('@aws-cdk/cdk'); import { Test } from 'nodeunit'; @@ -19,24 +19,16 @@ export = { test.ok(entry, 'found metadata entry'); test.deepEqual(entry!.data, { path: dirPath, + id: 'MyAsset', packaging: 'zip', s3BucketParameter: 'MyAssetS3Bucket68C9B344', - s3KeyParameter: 'MyAssetS3ObjectKeyC07605E4' + s3KeyParameter: 'MyAssetS3VersionKey68E1A45D', }); - // verify that now the template contains two parameters for this asset - expect(stack).toMatch({ - Parameters: { - MyAssetS3Bucket68C9B344: { - Type: "String", - Description: 'S3 bucket for asset "MyAsset"' - }, - MyAssetS3ObjectKeyC07605E4: { - Type: "String", - Description: 'S3 object for asset "MyAsset"' - } - } - }); + // verify that now the template contains parameters for this asset + const template = stack.toCloudFormation(); + test.equal(template.Parameters.MyAssetS3Bucket68C9B344.Type, 'String'); + test.equal(template.Parameters.MyAssetS3VersionKey68E1A45D.Type, 'String'); test.done(); }, @@ -50,22 +42,15 @@ export = { test.deepEqual(entry!.data, { path: filePath, packaging: 'file', + id: 'MyAsset', s3BucketParameter: 'MyAssetS3Bucket68C9B344', - s3KeyParameter: 'MyAssetS3ObjectKeyC07605E4' + s3KeyParameter: 'MyAssetS3VersionKey68E1A45D', }); - expect(stack).toMatch({ - Parameters: { - MyAssetS3Bucket68C9B344: { - Type: "String", - Description: 'S3 bucket for asset "MyAsset"' - }, - MyAssetS3ObjectKeyC07605E4: { - Type: "String", - Description: 'S3 object for asset "MyAsset"' - } - } - }); + // verify that now the template contains parameters for this asset + const template = stack.toCloudFormation(); + test.equal(template.Parameters.MyAssetS3Bucket68C9B344.Type, 'String'); + test.equal(template.Parameters.MyAssetS3VersionKey68E1A45D.Type, 'String'); test.done(); }, @@ -82,188 +67,28 @@ export = { asset.grantRead(group); - expect(stack).toMatch({ - Resources: { - MyUserDC45028B: { - Type: "AWS::IAM::User" - }, - MyUserDefaultPolicy7B897426: { - Type: "AWS::IAM::Policy", - Properties: { - PolicyDocument: { - Statement: [ - { - Action: [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*" - ], - Effect: "Allow", - Resource: [ - { - "Fn::Join": [ - "", - [ - "arn", - ":", - { - Ref: "AWS::Partition" - }, - ":", - "s3", - ":", - "", - ":", - "", - ":", - { - Ref: "MyAssetS3Bucket68C9B344" - } - ] - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::Join": [ - "", - [ - "arn", - ":", - { - Ref: "AWS::Partition" - }, - ":", - "s3", - ":", - "", - ":", - "", - ":", - { - Ref: "MyAssetS3Bucket68C9B344" - } - ] - ] - }, - "/", - { - Ref: "MyAssetS3ObjectKeyC07605E4" - } - ] - ] - } - ] - } - ], - Version: "2012-10-17" - }, - PolicyName: "MyUserDefaultPolicy7B897426", - Users: [ - { - Ref: "MyUserDC45028B" - } + expect(stack).to(haveResource('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: ["s3:GetObject*", "s3:GetBucket*", "s3:List*"], + Resource: [ + {"Fn::Join": ["", ["arn", ":", {Ref: "AWS::Partition"}, ":", "s3", ":", "", ":", "", ":", {Ref: "MyAssetS3Bucket68C9B344"}]]}, + {"Fn::Join": [ "", [ + {"Fn::Join": ["", [ "arn", ":", {Ref: "AWS::Partition"}, ":", "s3", ":", "", ":", "", ":", {Ref: "MyAssetS3Bucket68C9B344"}]]}, + "/", + {"Fn::Join": ["", [ + {"Fn::Select": [ + 0, + {"Fn::Split": [ "||", { Ref: "MyAssetS3VersionKey68E1A45D"}]} + ]}, + "*" + ]]} + ]]} ] - } - }, - MyGroupCBA54B1B: { - Type: "AWS::IAM::Group" - }, - MyGroupDefaultPolicy72C41231: { - Type: "AWS::IAM::Policy", - Properties: { - Groups: [ - { - Ref: "MyGroupCBA54B1B" - } - ], - PolicyDocument: { - Statement: [ - { - Action: [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*" - ], - Effect: "Allow", - Resource: [ - { - "Fn::Join": [ - "", - [ - "arn", - ":", - { - Ref: "AWS::Partition" - }, - ":", - "s3", - ":", - "", - ":", - "", - ":", - { - Ref: "MyAssetS3Bucket68C9B344" - } - ] - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::Join": [ - "", - [ - "arn", - ":", - { - Ref: "AWS::Partition" - }, - ":", - "s3", - ":", - "", - ":", - "", - ":", - { - Ref: "MyAssetS3Bucket68C9B344" - } - ] - ] - }, - "/", - { - Ref: "MyAssetS3ObjectKeyC07605E4" - } - ] - ] - } - ] - } - ], - Version: "2012-10-17" - }, - PolicyName: "MyGroupDefaultPolicy72C41231" - } - } - }, - Parameters: { - MyAssetS3Bucket68C9B344: { - Type: "String", - Description: "S3 bucket for asset \"MyAsset\"" - }, - MyAssetS3ObjectKeyC07605E4: { - Type: "String", - Description: "S3 object for asset \"MyAsset\"" } - } - }); + ] + }})); test.done(); }, @@ -274,4 +99,4 @@ export = { })); test.done(); } -}; +}; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda/test/integ.assets.file.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.assets.file.expected.json index 07396ecaa6393..8da50c070ca2b 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.assets.file.expected.json +++ b/packages/@aws-cdk/aws-lambda/test/integ.assets.file.expected.json @@ -104,7 +104,25 @@ }, "/", { - "Ref": "MyLambdaCodeS3ObjectKeyA7272AC7" + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "MyLambdaCodeS3VersionKey47762537" + } + ] + } + ] + }, + "*" + ] + ] } ] ] @@ -130,7 +148,37 @@ "Ref": "MyLambdaCodeS3BucketC82A5870" }, "S3Key": { - "Ref": "MyLambdaCodeS3ObjectKeyA7272AC7" + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "MyLambdaCodeS3VersionKey47762537" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "MyLambdaCodeS3VersionKey47762537" + } + ] + } + ] + } + ] + ] } }, "Handler": "index.main", @@ -153,9 +201,9 @@ "Type": "String", "Description": "S3 bucket for asset \"lambda-test-assets-file/MyLambda/Code\"" }, - "MyLambdaCodeS3ObjectKeyA7272AC7": { + "MyLambdaCodeS3VersionKey47762537": { "Type": "String", - "Description": "S3 object for asset \"lambda-test-assets-file/MyLambda/Code\"" + "Description": "S3 key for asset version \"lambda-test-assets-file/MyLambda/Code\"" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda/test/integ.assets.lit.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.assets.lit.expected.json index 97e93682df598..8c3e4e874b294 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.assets.lit.expected.json +++ b/packages/@aws-cdk/aws-lambda/test/integ.assets.lit.expected.json @@ -104,7 +104,25 @@ }, "/", { - "Ref": "MyLambdaCodeS3ObjectKeyA7272AC7" + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "MyLambdaCodeS3VersionKey47762537" + } + ] + } + ] + }, + "*" + ] + ] } ] ] @@ -130,7 +148,37 @@ "Ref": "MyLambdaCodeS3BucketC82A5870" }, "S3Key": { - "Ref": "MyLambdaCodeS3ObjectKeyA7272AC7" + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "MyLambdaCodeS3VersionKey47762537" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "MyLambdaCodeS3VersionKey47762537" + } + ] + } + ] + } + ] + ] } }, "Handler": "index.main", @@ -153,9 +201,9 @@ "Type": "String", "Description": "S3 bucket for asset \"lambda-test-assets/MyLambda/Code\"" }, - "MyLambdaCodeS3ObjectKeyA7272AC7": { + "MyLambdaCodeS3VersionKey47762537": { "Type": "String", - "Description": "S3 object for asset \"lambda-test-assets/MyLambda/Code\"" + "Description": "S3 key for asset version \"lambda-test-assets/MyLambda/Code\"" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda/test/test.lambda.ts b/packages/@aws-cdk/aws-lambda/test/test.lambda.ts index 60af9d7dd575b..9c12f4e482579 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.lambda.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.lambda.ts @@ -307,9 +307,10 @@ export = { "S3Bucket": { "Ref": "MyLambdaCodeS3BucketC82A5870" }, - "S3Key": { - "Ref": "MyLambdaCodeS3ObjectKeyA7272AC7" - } + "S3Key": { "Fn::Join": [ "", [ + {"Fn::Select": [0, {"Fn::Split": ["||", {"Ref": "MyLambdaCodeS3VersionKey47762537"}]}]}, + {"Fn::Select": [1, {"Fn::Split": ["||", {"Ref": "MyLambdaCodeS3VersionKey47762537"}]}]}, + ]]} }, "Handler": "index.handler", "Role": { diff --git a/packages/@aws-cdk/cx-api/lib/cxapi.ts b/packages/@aws-cdk/cx-api/lib/cxapi.ts index 06b5e453caa4a..2f22a853c03e0 100644 --- a/packages/@aws-cdk/cx-api/lib/cxapi.ts +++ b/packages/@aws-cdk/cx-api/lib/cxapi.ts @@ -109,9 +109,29 @@ export const DEFAULT_REGION_CONTEXT_KEY = 'aws:cdk:toolkit:default-region'; export const ASSET_METADATA = 'aws:cdk:asset'; export interface AssetMetadataEntry { + /** + * Path on disk to the asset + */ path: string; + + /** + * Logical identifier for the asset + */ + id: string; + + /** + * Requested packaging style + */ packaging: 'zip' | 'file'; + + /** + * Name of parameter where S3 bucket should be passed in + */ s3BucketParameter: string; + + /** + * Name of parameter where S3 key should be passed in + */ s3KeyParameter: string; } @@ -129,3 +149,15 @@ export const WARNING_METADATA_KEY = 'aws:cdk:warning'; * Metadata key used to print ERROR-level messages by the toolkit when an app is syntheized. */ export const ERROR_METADATA_KEY = 'aws:cdk:error'; + +/** + * Separator string that separates the prefix separator from the object key separator. + * + * Asset keys will look like: + * + * /assets/MyConstruct12345678/||abcdef12345.zip + * + * This allows us to encode both the prefix and the full location in a single + * CloudFormation Template Parameter. + */ +export const ASSET_PREFIX_SEPARATOR = '||'; \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/toolkit-info.ts b/packages/aws-cdk/lib/api/toolkit-info.ts index ee16e0f13c9f2..f171d1deed3d6 100644 --- a/packages/aws-cdk/lib/api/toolkit-info.ts +++ b/packages/aws-cdk/lib/api/toolkit-info.ts @@ -8,6 +8,18 @@ import { BUCKET_DOMAIN_NAME_OUTPUT, BUCKET_NAME_OUTPUT } from './bootstrap-envi import { waitForStack } from './util/cloudformation'; import { SDK } from './util/sdk'; +export interface UploadProps { + s3KeyPrefix?: string, + s3KeySuffix?: string, + contentType?: string, +} + +export interface Uploaded { + filename: string; + key: string; + changed: boolean; +} + export class ToolkitInfo { constructor(private readonly props: { sdk: SDK, @@ -29,11 +41,7 @@ export class ToolkitInfo { * Uses md5 hash to render the full key and skips upload if an object * already exists by this key. */ - public async uploadIfChanged(data: any, props: { - s3KeyPrefix?: string, - s3KeySuffix?: string, - contentType?: string, - }): Promise<{ key: string, changed: boolean }> { + public async uploadIfChanged(data: any, props: UploadProps): Promise { const s3 = await this.props.sdk.s3(this.props.environment, Mode.ForWriting); const s3KeyPrefix = props.s3KeyPrefix || ''; @@ -42,13 +50,14 @@ export class ToolkitInfo { const bucket = this.props.bucketName; const hash = md5hash(data); - const key = `${s3KeyPrefix}${hash}${s3KeySuffix}`; + const filename = `${hash}${s3KeySuffix}`; + const key = `${s3KeyPrefix}${filename}`; const url = `s3://${bucket}/${key}`; debug(`${url}: checking if already exists`); if (await objectExists(s3, bucket, key)) { debug(`${url}: found (skipping upload)`); - return { key, changed: false }; + return { filename, key, changed: false }; } debug(`${url}: uploading`); @@ -61,7 +70,7 @@ export class ToolkitInfo { debug(`${url}: upload complete`); - return { key, changed: true }; + return { filename, key, changed: true }; } } diff --git a/packages/aws-cdk/lib/api/util/sdk.ts b/packages/aws-cdk/lib/api/util/sdk.ts index 49e24cb84658a..a6520d5ee8d08 100644 --- a/packages/aws-cdk/lib/api/util/sdk.ts +++ b/packages/aws-cdk/lib/api/util/sdk.ts @@ -19,22 +19,25 @@ import { SharedIniFile } from './sdk_ini_file'; * to the requested account. */ export class SDK { - private defaultAccountFetched = false; - private defaultAccountId?: string = undefined; private readonly userAgent: string; - private readonly accountCache = new AccountAccessKeyCache(); - private defaultCredentialProvider?: AWS.CredentialProviderChain; + private readonly defaultAwsAccount: DefaultAWSAccount; + private readonly credentialProviderCache: CredentialProviderCache; constructor(private readonly profile: string | undefined) { // Find the package.json from the main toolkit const pkg = (require.main as any).require('../package.json'); this.userAgent = `${pkg.name}/${pkg.version}`; + + const defaultCredentialProvider = makeCLICompatibleCredentialProvider(profile); + + this.defaultAwsAccount = new DefaultAWSAccount(defaultCredentialProvider); + this.credentialProviderCache = new CredentialProviderCache(this.defaultAwsAccount, defaultCredentialProvider); } public async cloudFormation(environment: Environment, mode: Mode): Promise { return new AWS.CloudFormation({ region: environment.region, - credentialProvider: await this.getCredentialProvider(environment.account, mode), + credentialProvider: await this.credentialProviderCache.get(environment.account, mode), customUserAgent: this.userAgent }); } @@ -42,7 +45,7 @@ export class SDK { public async ec2(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise { return new AWS.EC2({ region, - credentialProvider: await this.getCredentialProvider(awsAccountId, mode), + credentialProvider: await this.credentialProviderCache.get(awsAccountId, mode), customUserAgent: this.userAgent }); } @@ -50,7 +53,7 @@ export class SDK { public async ssm(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise { return new AWS.SSM({ region, - credentialProvider: await this.getCredentialProvider(awsAccountId, mode), + credentialProvider: await this.credentialProviderCache.get(awsAccountId, mode), customUserAgent: this.userAgent }); } @@ -58,7 +61,7 @@ export class SDK { public async s3(environment: Environment, mode: Mode): Promise { return new AWS.S3({ region: environment.region, - credentialProvider: await this.getCredentialProvider(environment.account, mode), + credentialProvider: await this.credentialProviderCache.get(environment.account, mode), customUserAgent: this.userAgent }); } @@ -67,7 +70,86 @@ export class SDK { return await getCLICompatibleDefaultRegion(this.profile); } - public async defaultAccount(): Promise { + public defaultAccount(): Promise { + return this.defaultAwsAccount.get(); + } +} + +/** + * Cache for credential providers. + * + * Given an account and an operating mode (read or write) will return an + * appropriate credential provider for credentials for the given account. The + * credential provider will be cached so that multiple AWS clients for the same + * environment will not make multiple network calls to obtain credentials. + * + * Will use default credentials if they are for the right account; otherwise, + * all loaded credential provider plugins will be tried to obtain credentials + * for the given account. + */ +class CredentialProviderCache { + private readonly cache: {[key: string]: AWS.CredentialProviderChain} = {}; + + public constructor( + private readonly defaultAwsAccount: DefaultAWSAccount, + private readonly defaultCredentialProvider: Promise) { + } + + public async get(awsAccountId: string | undefined, mode: Mode): Promise { + const key = `${awsAccountId}-${mode}`; + if (!(key in this.cache)) { + this.cache[key] = await this.getCredentialProvider(awsAccountId, mode); + } + return this.cache[key]; + } + + private async getCredentialProvider(awsAccountId: string | undefined, mode: Mode): Promise { + // If requested account is undefined or equal to default account, use default credentials provider. + // (Note that we ignore the mode in this case, if you preloaded credentials they better be correct!) + const defaultAccount = await this.defaultAwsAccount.get(); + if (!awsAccountId || awsAccountId === defaultAccount) { + debug(`Using default AWS SDK credentials for account ${awsAccountId}`); + return this.defaultCredentialProvider; + } + + const triedSources: CredentialProviderSource[] = []; + // Otherwise, inspect the various credential sources we have + for (const source of PluginHost.instance.credentialProviderSources) { + if (!(await source.isAvailable())) { + debug('Credentials source %s is not available, ignoring it.', source.name); + continue; + } + triedSources.push(source); + if (!(await source.canProvideCredentials(awsAccountId))) { continue; } + debug(`Using ${source.name} credentials for account ${awsAccountId}`); + return await source.getProvider(awsAccountId, mode); + } + const sourceNames = ['default credentials'].concat(triedSources.map(s => s.name)).join(', '); + throw new Error(`Need to perform AWS calls for account ${awsAccountId}, but no credentials found. Tried: ${sourceNames}.`); + } +} + +/** + * Class to retrieve the account for default credentials and cache it. + * + * Uses the default credentials provider to obtain credentials (if available), + * and uses those credentials to call STS to request the current account ID. + * + * The credentials => accountId lookup is cached on disk, since it's + * guaranteed that igven access key will always remain for the same account. + */ +class DefaultAWSAccount { + private defaultAccountFetched = false; + private defaultAccountId?: string = undefined; + private readonly accountCache = new AccountAccessKeyCache(); + + constructor(private readonly defaultCredentialsProvider: Promise) { + } + + /** + * Return the default account + */ + public async get(): Promise { if (!this.defaultAccountFetched) { this.defaultAccountId = await this.lookupDefaultAccount(); this.defaultAccountFetched = true; @@ -75,13 +157,12 @@ export class SDK { return this.defaultAccountId; } - private async lookupDefaultAccount() { + private async lookupDefaultAccount(): Promise { try { debug('Resolving default credentials'); - if (!this.defaultCredentialProvider) { - this.defaultCredentialProvider = await makeCLICompatibleCredentialProvider(this.profile); - } - const creds = await this.defaultCredentialProvider.resolvePromise(); + const credentialProvider = await this.defaultCredentialsProvider; + const creds = await credentialProvider.resolvePromise(); + const accessKeyId = creds.accessKeyId; if (!accessKeyId) { throw new Error('Unable to resolve AWS credentials (setup with "aws configure")'); @@ -106,35 +187,6 @@ export class SDK { return undefined; } } - - private async getCredentialProvider(awsAccountId: string | undefined, mode: Mode): Promise { - // If requested account is undefined or equal to default account, use default credentials provider. - const defaultAccount = await this.defaultAccount(); - if (!awsAccountId || awsAccountId === defaultAccount) { - debug(`Using default AWS SDK credentials for account ${awsAccountId}`); - return undefined; - } - - const triedSources: CredentialProviderSource[] = []; - - // Otherwise, inspect the various credential sources we have - for (const source of PluginHost.instance.credentialProviderSources) { - if (!(await source.isAvailable())) { - debug('Credentials source %s is not available, ignoring it.', source.name); - continue; - } - triedSources.push(source); - - if (!(await source.canProvideCredentials(awsAccountId))) { continue; } - debug(`Using ${source.name} credentials for account ${awsAccountId}`); - - return await source.getProvider(awsAccountId, mode); - } - - const sourceNames = ['default credentials'].concat(triedSources.map(s => s.name)).join(', '); - - throw new Error(`Need to perform AWS calls for account ${awsAccountId}, but no credentials found. Tried: ${sourceNames}.`); - } } /** diff --git a/packages/aws-cdk/lib/assets.ts b/packages/aws-cdk/lib/assets.ts index fcda72433f870..93fa8e3c6dd83 100644 --- a/packages/aws-cdk/lib/assets.ts +++ b/packages/aws-cdk/lib/assets.ts @@ -1,6 +1,7 @@ -import { ASSET_METADATA, AssetMetadataEntry, StackMetadata, SynthesizedStack } from '@aws-cdk/cx-api'; +import { ASSET_METADATA, ASSET_PREFIX_SEPARATOR, AssetMetadataEntry, StackMetadata, SynthesizedStack } from '@aws-cdk/cx-api'; import { CloudFormation } from 'aws-sdk'; import fs = require('fs-extra'); +import os = require('os'); import path = require('path'); import { ToolkitInfo } from './api/toolkit-info'; import { zipDirectory } from './archive'; @@ -41,7 +42,7 @@ async function prepareAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo) async function prepareZipAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo): Promise { debug('Preparing zip asset from directory:', asset.path); - const staging = await fs.mkdtemp('/tmp/cdk-assets'); + const staging = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-assets')); try { const archiveFile = path.join(staging, 'archive.zip'); await zipDirectory(asset.path, archiveFile); @@ -69,8 +70,10 @@ async function prepareFileAsset( const data = await fs.readFile(filePath); - const { key, changed } = await toolkitInfo.uploadIfChanged(data, { - s3KeyPrefix: 'assets/', + const s3KeyPrefix = `assets/${asset.id}/`; + + const { filename, key, changed } = await toolkitInfo.uploadIfChanged(data, { + s3KeyPrefix, s3KeySuffix: path.extname(filePath), contentType }); @@ -84,7 +87,7 @@ async function prepareFileAsset( return [ { ParameterKey: asset.s3BucketParameter, ParameterValue: toolkitInfo.bucketName }, - { ParameterKey: asset.s3KeyParameter, ParameterValue: key } + { ParameterKey: asset.s3KeyParameter, ParameterValue: `${s3KeyPrefix}${ASSET_PREFIX_SEPARATOR}${filename}` }, ]; } diff --git a/packages/aws-cdk/test/test.archive.ts b/packages/aws-cdk/test/test.archive.ts index 48bdb9849ae99..4baf24c45a7e4 100644 --- a/packages/aws-cdk/test/test.archive.ts +++ b/packages/aws-cdk/test/test.archive.ts @@ -1,6 +1,7 @@ import { exec as _exec } from 'child_process'; import fs = require('fs-extra'); import { Test } from 'nodeunit'; +import os = require('os'); import path = require('path'); import { promisify } from 'util'; import { md5hash, zipDirectory } from '../lib/archive'; @@ -8,10 +9,10 @@ const exec = promisify(_exec); export = { async 'zipDirectory can take a directory and produce a zip from it'(test: Test) { - const stagingDir = await fs.mkdtemp('/tmp/test.archive'); + const stagingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'test.archive')); const zipFile = path.join(stagingDir, 'output.zip'); const originalDir = path.join(__dirname, 'test-archive'); - const extractDir = await fs.mkdtemp('/tmp/test.archive.extract'); + const extractDir = await fs.mkdtemp(path.join(os.tmpdir(), 'test.archive.extract')); await zipDirectory(originalDir, zipFile); // unzip and verify that the resulting tree is the same @@ -29,7 +30,7 @@ export = { }, async 'md5 hash of a zip stays consistent across invocations'(test: Test) { - const stagingDir = await fs.mkdtemp('/tmp/test.archive'); + const stagingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'test.archive')); const zipFile1 = path.join(stagingDir, 'output.zip'); const zipFile2 = path.join(stagingDir, 'output.zip'); const originalDir = path.join(__dirname, 'test-archive'); diff --git a/packages/aws-cdk/test/test.assets.ts b/packages/aws-cdk/test/test.assets.ts new file mode 100644 index 0000000000000..a270bde14592a --- /dev/null +++ b/packages/aws-cdk/test/test.assets.ts @@ -0,0 +1,59 @@ +import { AssetMetadataEntry } from '@aws-cdk/cx-api'; +import { Test } from 'nodeunit'; +import { Uploaded, UploadProps } from '../lib'; +import { prepareAssets } from '../lib/assets'; + +export = { + async 'prepare assets'(test: Test) { + // GIVEN + const stack = { + name: 'SomeStack', + metadata: { + '/SomeStack/SomeResource': [{ + type: 'aws:cdk:asset', + data: { + path: __filename, + id: 'SomeStackSomeResource4567', + packaging: 'file', + s3BucketParameter: 'BucketParameter', + s3KeyParameter: 'KeyParameter' + } as AssetMetadataEntry, + trace: [] + }] + }, + template: { + Resources: { + SomeResource: { + Type: 'AWS::Something::Something' + } + } + } + }; + const toolkit = new FakeToolkit(); + + // WHEN + const params = await prepareAssets(stack, toolkit as any); + + // THEN + test.deepEqual(params, [ + { ParameterKey: 'BucketParameter', ParameterValue: 'bucket' }, + { ParameterKey: 'KeyParameter', ParameterValue: 'assets/SomeStackSomeResource4567/||12345.js' }, + ]); + + test.done(); + } +}; + +class FakeToolkit { + public bucketUrl: string = 'https://bucket'; + public bucketName: string = 'bucket'; + + public async uploadIfChanged(_data: any, props: UploadProps): Promise { + const filename = `12345${props.s3KeySuffix}`; + return { + filename, + changed: true, + key: `${props.s3KeyPrefix}${filename}` + }; + } +} \ No newline at end of file