diff --git a/packages/@aws-cdk/cdk-assets/.gitignore b/packages/@aws-cdk/cdk-assets/.gitignore new file mode 100644 index 0000000000000..6f03998f7c880 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/.gitignore @@ -0,0 +1,13 @@ +*.js +*.js.map +*.d.ts +node_modules +dist +tsconfig.json +tslint.json + +.LAST_BUILD +.nyc_output +coverage + +.jsii diff --git a/packages/@aws-cdk/cdk-assets/.npmignore b/packages/@aws-cdk/cdk-assets/.npmignore new file mode 100644 index 0000000000000..414172bb772ec --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/.npmignore @@ -0,0 +1,6 @@ +# Don't include original .ts files when doing `npm pack` +*.ts +!*.d.ts +coverage +.nyc_output +*.tgz diff --git a/packages/@aws-cdk/cdk-assets/README.md b/packages/@aws-cdk/cdk-assets/README.md new file mode 100644 index 0000000000000..681a4d2e45dec --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/README.md @@ -0,0 +1,61 @@ +## AWS CDK Assets + +Assets are local files or directories which are needed by a CDK app. A common +example is a directory which contains the handler code for a Lambda function, +but assets can represent any artifact that is needed for the app's operation. + +When deploying a CDK app that includes constructs with assets, the CDK toolkit +will first upload all the assets to S3, and only then deploy the stacks. The S3 +locations of the uploaded assets will be passed in as CloudFormation Parameters +to the relevant stacks. + +The following JavaScript example defines an directory asset which is archived as +a .zip file and uploaded to S3 during deployment. + +[Example of a ZipDirectoryAsset](./test/integ.assets.directory.lit.ts) + +The following JavaScript example defines a file asset, which is uploaded as-is +to an S3 bucket during deployment. + +[Example of a FileAsset](./test/integ.assets.file.lit.ts) + +## Attributes + +`Asset` constructs expose the following deploy-time attributes: + + * `s3BucketName` - the name of the assets S3 bucket. + * `s3ObjectKey` - the S3 object key of the asset file (whether it's a file or a zip archive) + * `s3Url` - the S3 URL of the asset (i.e. https://s3.us-east-1.amazonaws.com/mybucket/mykey.zip) + +In the following example, the various asset attributes are exported as stack outputs: + +[Example of referencing an asset](./test/integ.assets.refs.lit.ts) + +## Permissions + +IAM roles, users or groups which need to be able to read assets in runtime will should be +granted IAM permissions. To do that use the `asset.grantRead(principal)` method: + +The following examples grants an IAM group read permissions on an asset: + +[Example of granting read access to an asset](./test/integ.assets.permissions.lit.ts) + +## How does it work? + +When an asset is defined in a construct, a construct metadata entry +`aws:cdk:asset` is emitted with instructions on where to find the asset and what +type of packaging to perform (`zip` or `file`). Furthermore, the synthesized +CloudFormation template will also include two CloudFormation parameters: one for +the asset's bucket and one for the asset S3 key. Those parameters are used to +reference the deploy-time values of the asset (using `{ Ref: "Param" }`). + +Then, when the stack is deployed, the toolkit will package the asset (i.e. zip +the directory), calculate an MD5 hash of the contents and will render an S3 key +for this asset within the toolkit's asset store. If the file doesn't exist in +the asset store, it is uploaded during deployment. + +> The toolkit's asset store is an S3 bucket created by the toolkit for each + environment the toolkit operates in (environment = account + region). + +Now, when the toolkit deploys the stack, it will set the relevant CloudFormation +Parameters to point to the actual bucket and key for each asset. diff --git a/packages/@aws-cdk/cdk-assets/lib/asset.ts b/packages/@aws-cdk/cdk-assets/lib/asset.ts new file mode 100644 index 0000000000000..260953680f867 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/lib/asset.ts @@ -0,0 +1,196 @@ +import cdk = require('@aws-cdk/core'); +import cxapi = require('@aws-cdk/cx-api'); +import iam = require('@aws-cdk/iam'); +import s3 = require('@aws-cdk/s3'); +import * as fs from 'fs'; +import * as path from 'path'; + +/** + * Defines the way an asset is packaged before it is uploaded to S3. + */ +export enum AssetPackaging { + /** + * Path refers to a directory on disk, the contents of the directory is + * archived into a .zip. + */ + ZipDirectory = 'zip', + + /** + * Path refers to a single file on disk. The file is uploaded as-is. + */ + File = 'file', +} + +export interface GenericAssetProps { + /** + * The disk location of the asset. + */ + path: string; + + /** + * The packaging type for this asset. + */ + packaging: AssetPackaging; + + /** + * A list of principals that should be able to read this asset from S3. + * You can use `asset.grantRead(principal)` to grant read permissions later. + */ + readers?: iam.IPrincipal[]; +} + +/** + * An asset represents a local file or directory, which is automatically uploaded to S3 + * and then can be referenced within a CDK application. + */ +export class Asset extends cdk.Construct { + /** + * Attribute that represents the name of the bucket this asset exists in. + */ + public readonly s3BucketName: s3.BucketName; + + /** + * Attribute which represents the S3 object key of this asset. + */ + public readonly s3ObjectKey: s3.ObjectKey; + + /** + * Attribute which represents the S3 URL of this asset. + * @example https://s3.us-west-1.amazonaws.com/bucket/key + */ + public readonly s3Url: s3.S3Url; + + /** + * Resolved full-path location of this asset. + */ + public readonly assetPath: string; + + private readonly bucket: s3.BucketRef; + + constructor(parent: cdk.Construct, id: string, props: GenericAssetProps) { + super(parent, id); + + // resolve full path + this.assetPath = path.resolve(props.path); + + validateAssetOnDisk(this.assetPath, props.packaging); + + // add parameters for s3 bucket and s3 key. those will be set by + // the toolkit or by CI/CD when the stack is deployed and will include + // the name of the bucket and the S3 key where the code lives. + + const bucketParam = new cdk.Parameter(this, 'S3Bucket', { + type: 'String', + description: `S3 bucket for asset "${this.path}"`, + }); + + const keyParam = new cdk.Parameter(this, 'S3ObjectKey', { + type: 'String', + description: `S3 object for asset "${this.path}"` + }); + + this.s3BucketName = bucketParam.value; + this.s3ObjectKey = keyParam.value; + + // grant the lambda's role read permissions on the code s3 object + + this.bucket = s3.BucketRef.import(parent, 'AssetBucket', { + bucketName: this.s3BucketName + }); + + // form the s3 URL of the object key + this.s3Url = this.bucket.urlForObject(this.s3ObjectKey); + + // attach metadata to the lambda function which includes information + // for tooling to be able to package and upload a directory to the + // s3 bucket and plug in the bucket name and key in the correct + // parameters. + + const asset: cxapi.AssetMetadataEntry = { + path: this.assetPath, + packaging: props.packaging, + s3BucketParameter: bucketParam.logicalId, + s3KeyParameter: keyParam.logicalId, + }; + + this.addMetadata(cxapi.ASSET_METADATA, asset); + + for (const reader of (props.readers || [])) { + this.grantRead(reader); + } + } + + /** + * Grants read permissions to the principal on the asset's S3 object. + */ + public grantRead(principal?: iam.IPrincipal) { + this.bucket.grantRead(principal, this.s3ObjectKey); + } +} + +export interface FileAssetProps { + /** + * File path. + */ + path: string; + + /** + * A list of principals that should be able to read this file asset from S3. + * You can use `asset.grantRead(principal)` to grant read permissions later. + */ + readers?: iam.IPrincipal[]; +} + +/** + * An asset that represents a file on disk. + */ +export class FileAsset extends Asset { + constructor(parent: cdk.Construct, id: string, props: FileAssetProps) { + super(parent, id, { packaging: AssetPackaging.File, ...props }); + } +} + +export interface ZipDirectoryAssetProps { + /** + * Path of the directory. + */ + path: string; + + /** + * A list of principals that should be able to read this ZIP file from S3. + * You can use `asset.grantRead(principal)` to grant read permissions later. + */ + readers?: iam.IPrincipal[]; +} + +/** + * An asset that represents a ZIP archive of a directory on disk. + */ +export class ZipDirectoryAsset extends Asset { + constructor(parent: cdk.Construct, id: string, props: ZipDirectoryAssetProps) { + super(parent, id, { packaging: AssetPackaging.ZipDirectory, ...props }); + } +} + +function validateAssetOnDisk(assetPath: string, packaging: AssetPackaging) { + if (!fs.existsSync(assetPath)) { + throw new Error(`Cannot find asset at ${assetPath}`); + } + + switch (packaging) { + case AssetPackaging.ZipDirectory: + if (!fs.statSync(assetPath).isDirectory()) { + throw new Error(`${assetPath} is expected to be a directory when asset packaging is 'zip'`); + } + break; + + case AssetPackaging.File: + if (!fs.statSync(assetPath).isFile()) { + throw new Error(`${assetPath} is expected to be a regular file when asset packaging is 'file'`); + } + break; + + default: + throw new Error(`Unsupported asset packaging format: ${packaging}`); + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/cdk-assets/lib/index.ts b/packages/@aws-cdk/cdk-assets/lib/index.ts new file mode 100644 index 0000000000000..946a5dd1a6851 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/lib/index.ts @@ -0,0 +1 @@ +export * from './asset'; \ No newline at end of file diff --git a/packages/@aws-cdk/cdk-assets/package.json b/packages/@aws-cdk/cdk-assets/package.json new file mode 100644 index 0000000000000..27be41f8a1443 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/package.json @@ -0,0 +1,50 @@ +{ + "name": "@aws-cdk/cdk-assets", + "version": "0.7.3-beta", + "description": "Integration of CDK apps with local assets", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "jsii": { + "outdir": "dist", + "names": { + "java": "com.amazonaws.cdk.cdkassets", + "dotnet": "Amazon.Cdk.CdkAssets" + } + }, + "repository": { + "type": "git", + "url": "git://github.com/awslabs/aws-cdk" + }, + "scripts": { + "build": "cdk-build", + "watch": "cdk-watch", + "lint": "cdk-lint", + "test": "cdk-test", + "integ": "cdk-integ", + "pkglint": "pkglint -f" + }, + "keywords": [ + "aws", + "cdk", + "constructs", + "assets" + ], + "author": { + "name": "Amazon Web Services", + "url": "https://aws.amazon.com" + }, + "license": "LicenseRef-LICENSE", + "devDependencies": { + "@aws-cdk/assert": "^0.7.3-beta", + "aws-cdk": "^0.7.3-beta", + "pkglint": "^0.7.3-beta", + "cdk-build-tools": "^0.7.3-beta", + "cdk-integ-tools": "^0.7.3-beta" + }, + "dependencies": { + "@aws-cdk/s3": "^0.7.3-beta", + "@aws-cdk/core": "^0.7.3-beta", + "@aws-cdk/iam": "^0.7.3-beta", + "@aws-cdk/cx-api": "^0.7.3-beta" + } +} diff --git a/packages/@aws-cdk/cdk-assets/test/file-asset.txt b/packages/@aws-cdk/cdk-assets/test/file-asset.txt new file mode 100644 index 0000000000000..87cda36f4da3e --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/file-asset.txt @@ -0,0 +1 @@ +Hello, this is a just a file! \ No newline at end of file diff --git a/packages/@aws-cdk/cdk-assets/test/integ.assets.directory.lit.expected.json b/packages/@aws-cdk/cdk-assets/test/integ.assets.directory.lit.expected.json new file mode 100644 index 0000000000000..e3ca3797ddd0c --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/integ.assets.directory.lit.expected.json @@ -0,0 +1,98 @@ +{ + "Parameters": { + "SampleAssetS3BucketE6B2908E": { + "Type": "String", + "Description": "S3 bucket for asset \"aws-cdk-asset-test/SampleAsset\"" + }, + "SampleAssetS3ObjectKey6F5D200B": { + "Type": "String", + "Description": "S3 object for asset \"aws-cdk-asset-test/SampleAsset\"" + } + }, + "Resources": { + "MyUserDC45028B": { + "Type": "AWS::IAM::User" + }, + "MyUserDefaultPolicy7B897426": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + "Ref": "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + "Ref": "SampleAssetS3BucketE6B2908E" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + "Ref": "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + "Ref": "SampleAssetS3BucketE6B2908E" + } + ] + ] + }, + "/", + { + "Ref": "SampleAssetS3ObjectKey6F5D200B" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyUserDefaultPolicy7B897426", + "Users": [ + { + "Ref": "MyUserDC45028B" + } + ] + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/cdk-assets/test/integ.assets.directory.lit.ts b/packages/@aws-cdk/cdk-assets/test/integ.assets.directory.lit.ts new file mode 100644 index 0000000000000..ab3b7b642af50 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/integ.assets.directory.lit.ts @@ -0,0 +1,23 @@ +import cdk = require('@aws-cdk/core'); +import iam = require('@aws-cdk/iam'); +import path = require('path'); +import assets = require('../lib'); + +class TestStack extends cdk.Stack { + constructor(parent: cdk.App, name: string, props?: cdk.StackProps) { + super(parent, name, props); + + /// !show + const asset = new assets.ZipDirectoryAsset(this, 'SampleAsset', { + path: path.join(__dirname, 'sample-asset-directory') + }); + /// !hide + + const user = new iam.User(this, 'MyUser'); + asset.grantRead(user); + } +} + +const app = new cdk.App(process.argv); +new TestStack(app, 'aws-cdk-asset-test'); +process.stdout.write(app.run()); diff --git a/packages/@aws-cdk/cdk-assets/test/integ.assets.file.lit.expected.json b/packages/@aws-cdk/cdk-assets/test/integ.assets.file.lit.expected.json new file mode 100644 index 0000000000000..99d5cee84363e --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/integ.assets.file.lit.expected.json @@ -0,0 +1,98 @@ +{ + "Parameters": { + "SampleAssetS3BucketE6B2908E": { + "Type": "String", + "Description": "S3 bucket for asset \"aws-cdk-asset-file-test/SampleAsset\"" + }, + "SampleAssetS3ObjectKey6F5D200B": { + "Type": "String", + "Description": "S3 object for asset \"aws-cdk-asset-file-test/SampleAsset\"" + } + }, + "Resources": { + "MyUserDC45028B": { + "Type": "AWS::IAM::User" + }, + "MyUserDefaultPolicy7B897426": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + "Ref": "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + "Ref": "SampleAssetS3BucketE6B2908E" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + "Ref": "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + "Ref": "SampleAssetS3BucketE6B2908E" + } + ] + ] + }, + "/", + { + "Ref": "SampleAssetS3ObjectKey6F5D200B" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyUserDefaultPolicy7B897426", + "Users": [ + { + "Ref": "MyUserDC45028B" + } + ] + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/cdk-assets/test/integ.assets.file.lit.ts b/packages/@aws-cdk/cdk-assets/test/integ.assets.file.lit.ts new file mode 100644 index 0000000000000..16b95cb108b83 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/integ.assets.file.lit.ts @@ -0,0 +1,23 @@ +import cdk = require('@aws-cdk/core'); +import iam = require('@aws-cdk/iam'); +import path = require('path'); +import assets = require('../lib'); + +class TestStack extends cdk.Stack { + constructor(parent: cdk.App, name: string, props?: cdk.StackProps) { + super(parent, name, props); + + /// !show + const asset = new assets.FileAsset(this, 'SampleAsset', { + path: path.join(__dirname, 'file-asset.txt') + }); + /// !hide + + const user = new iam.User(this, 'MyUser'); + asset.grantRead(user); + } +} + +const app = new cdk.App(process.argv); +new TestStack(app, 'aws-cdk-asset-file-test'); +process.stdout.write(app.run()); diff --git a/packages/@aws-cdk/cdk-assets/test/integ.assets.permissions.lit.expected.json b/packages/@aws-cdk/cdk-assets/test/integ.assets.permissions.lit.expected.json new file mode 100644 index 0000000000000..e7540be27bb73 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/integ.assets.permissions.lit.expected.json @@ -0,0 +1,98 @@ +{ + "Parameters": { + "MyFileS3BucketACE13C36": { + "Type": "String", + "Description": "S3 bucket for asset \"aws-cdk-asset-refs/MyFile\"" + }, + "MyFileS3ObjectKey4641930D": { + "Type": "String", + "Description": "S3 object for asset \"aws-cdk-asset-refs/MyFile\"" + } + }, + "Resources": { + "MyUserGroupDA7A39B2": { + "Type": "AWS::IAM::Group" + }, + "MyUserGroupDefaultPolicy50C5D742": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + "Ref": "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + "Ref": "MyFileS3BucketACE13C36" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + "Ref": "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + "Ref": "MyFileS3BucketACE13C36" + } + ] + ] + }, + "/", + { + "Ref": "MyFileS3ObjectKey4641930D" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyUserGroupDefaultPolicy50C5D742", + "Groups": [ + { + "Ref": "MyUserGroupDA7A39B2" + } + ] + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/cdk-assets/test/integ.assets.permissions.lit.ts b/packages/@aws-cdk/cdk-assets/test/integ.assets.permissions.lit.ts new file mode 100644 index 0000000000000..7de0186fdb08a --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/integ.assets.permissions.lit.ts @@ -0,0 +1,23 @@ +import cdk = require('@aws-cdk/core'); +import iam = require('@aws-cdk/iam'); +import path = require('path'); +import assets = require('../lib'); + +class TestStack extends cdk.Stack { + constructor(parent: cdk.App, name: string, props?: cdk.StackProps) { + super(parent, name, props); + + const asset = new assets.FileAsset(this, 'MyFile', { + path: path.join(__dirname, 'file-asset.txt') + }); + + /// !show + const group = new iam.Group(this, 'MyUserGroup'); + asset.grantRead(group); + /// !hide + } +} + +const app = new cdk.App(process.argv); +new TestStack(app, 'aws-cdk-asset-refs'); +process.stdout.write(app.run()); diff --git a/packages/@aws-cdk/cdk-assets/test/integ.assets.refs.lit.expected.json b/packages/@aws-cdk/cdk-assets/test/integ.assets.refs.lit.expected.json new file mode 100644 index 0000000000000..659b218855cc3 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/integ.assets.refs.lit.expected.json @@ -0,0 +1,145 @@ +{ + "Parameters": { + "SampleAssetS3BucketE6B2908E": { + "Type": "String", + "Description": "S3 bucket for asset \"aws-cdk-asset-refs/SampleAsset\"" + }, + "SampleAssetS3ObjectKey6F5D200B": { + "Type": "String", + "Description": "S3 object for asset \"aws-cdk-asset-refs/SampleAsset\"" + } + }, + "Outputs": { + "S3BucketName": { + "Value": { + "Ref": "SampleAssetS3BucketE6B2908E" + }, + "Export": { + "Name": "aws-cdk-asset-refs:S3BucketName" + } + }, + "S3ObjectKey": { + "Value": { + "Ref": "SampleAssetS3ObjectKey6F5D200B" + }, + "Export": { + "Name": "aws-cdk-asset-refs:S3ObjectKey" + } + }, + "S3URL": { + "Value": { + "Fn::Join": [ + "", + [ + "https://", + "s3.", + { + "Ref": "AWS::Region" + }, + ".", + { + "Ref": "AWS::URLSuffix" + }, + "/", + { + "Ref": "SampleAssetS3BucketE6B2908E" + }, + "/", + { + "Ref": "SampleAssetS3ObjectKey6F5D200B" + } + ] + ] + }, + "Export": { + "Name": "aws-cdk-asset-refs:S3URL" + } + } + }, + "Resources": { + "MyUserDC45028B": { + "Type": "AWS::IAM::User" + }, + "MyUserDefaultPolicy7B897426": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + "Ref": "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + "Ref": "SampleAssetS3BucketE6B2908E" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + "Ref": "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + "Ref": "SampleAssetS3BucketE6B2908E" + } + ] + ] + }, + "/", + { + "Ref": "SampleAssetS3ObjectKey6F5D200B" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyUserDefaultPolicy7B897426", + "Users": [ + { + "Ref": "MyUserDC45028B" + } + ] + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/cdk-assets/test/integ.assets.refs.lit.ts b/packages/@aws-cdk/cdk-assets/test/integ.assets.refs.lit.ts new file mode 100644 index 0000000000000..769ececac6fe5 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/integ.assets.refs.lit.ts @@ -0,0 +1,27 @@ +import cdk = require('@aws-cdk/core'); +import iam = require('@aws-cdk/iam'); +import path = require('path'); +import assets = require('../lib'); + +class TestStack extends cdk.Stack { + constructor(parent: cdk.App, name: string, props?: cdk.StackProps) { + super(parent, name, props); + + /// !show + const asset = new assets.ZipDirectoryAsset(this, 'SampleAsset', { + path: path.join(__dirname, 'sample-asset-directory') + }); + + new cdk.Output(this, 'S3BucketName', { value: asset.s3BucketName }); + new cdk.Output(this, 'S3ObjectKey', { value: asset.s3ObjectKey }); + new cdk.Output(this, 'S3URL', { value: asset.s3Url }); + /// !hide + + // we need at least one resource + asset.grantRead(new iam.User(this, 'MyUser')); + } +} + +const app = new cdk.App(process.argv); +new TestStack(app, 'aws-cdk-asset-refs'); +process.stdout.write(app.run()); diff --git a/packages/@aws-cdk/cdk-assets/test/sample-asset-directory/sample-asset-file.txt b/packages/@aws-cdk/cdk-assets/test/sample-asset-directory/sample-asset-file.txt new file mode 100644 index 0000000000000..a21cdbf7d0691 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/sample-asset-directory/sample-asset-file.txt @@ -0,0 +1 @@ +// hello dear asset \ No newline at end of file diff --git a/packages/@aws-cdk/cdk-assets/test/test.asset.ts b/packages/@aws-cdk/cdk-assets/test/test.asset.ts new file mode 100644 index 0000000000000..990abc66800c1 --- /dev/null +++ b/packages/@aws-cdk/cdk-assets/test/test.asset.ts @@ -0,0 +1,277 @@ +import { expect } from '@aws-cdk/assert'; +import { Stack } from '@aws-cdk/core'; +import { Group, User } from '@aws-cdk/iam'; +import { Test } from 'nodeunit'; +import * as path from 'path'; +import { FileAsset, ZipDirectoryAsset } from '../lib/asset'; + +export = { + 'simple use case'(test: Test) { + const stack = new Stack(); + const dirPath = path.join(__dirname, 'sample-asset-directory'); + const asset = new ZipDirectoryAsset(stack, 'MyAsset', { + path: dirPath + }); + + // verify that metadata contains an "aws:cdk:asset" entry with + // the correct information + const entry = asset.metadata.find(m => m.type === 'aws:cdk:asset'); + test.ok(entry, 'found metadata entry'); + test.deepEqual(entry!.data, { + path: dirPath, + packaging: 'zip', + s3BucketParameter: 'MyAssetS3Bucket68C9B344', + s3KeyParameter: 'MyAssetS3ObjectKeyC07605E4' + }); + + // verify that now the template contains two parameters for this asset + expect(stack).toMatch({ + Parameters: { + MyAssetS3Bucket68C9B344: { + Type: "String", + Description: 'S3 bucket for asset "MyAsset"' + }, + MyAssetS3ObjectKeyC07605E4: { + Type: "String", + Description: 'S3 object for asset "MyAsset"' + } + } + }); + + test.done(); + }, + + '"file" assets'(test: Test) { + const stack = new Stack(); + const filePath = path.join(__dirname, 'file-asset.txt'); + const asset = new FileAsset(stack, 'MyAsset', { path: filePath }); + const entry = asset.metadata.find(m => m.type === 'aws:cdk:asset'); + test.ok(entry, 'found metadata entry'); + test.deepEqual(entry!.data, { + path: filePath, + packaging: 'file', + s3BucketParameter: 'MyAssetS3Bucket68C9B344', + s3KeyParameter: 'MyAssetS3ObjectKeyC07605E4' + }); + + expect(stack).toMatch({ + Parameters: { + MyAssetS3Bucket68C9B344: { + Type: "String", + Description: 'S3 bucket for asset "MyAsset"' + }, + MyAssetS3ObjectKeyC07605E4: { + Type: "String", + Description: 'S3 object for asset "MyAsset"' + } + } + }); + + test.done(); + }, + + '"readers" or "grantRead" can be used to grant read permissions on the asset to a principal'(test: Test) { + const stack = new Stack(); + const user = new User(stack, 'MyUser'); + const group = new Group(stack, 'MyGroup'); + + const asset = new ZipDirectoryAsset(stack, 'MyAsset', { + path: path.join(__dirname, 'sample-asset-directory'), + readers: [ user ] + }); + + asset.grantRead(group); + + expect(stack).toMatch({ + Resources: { + MyUserDC45028B: { + Type: "AWS::IAM::User" + }, + MyUserDefaultPolicy7B897426: { + Type: "AWS::IAM::Policy", + Properties: { + PolicyDocument: { + Statement: [ + { + Action: [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + Effect: "Allow", + Resource: [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + Ref: "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + Ref: "MyAssetS3Bucket68C9B344" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + Ref: "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + Ref: "MyAssetS3Bucket68C9B344" + } + ] + ] + }, + "/", + { + Ref: "MyAssetS3ObjectKeyC07605E4" + } + ] + ] + } + ] + } + ], + Version: "2012-10-17" + }, + PolicyName: "MyUserDefaultPolicy7B897426", + Users: [ + { + Ref: "MyUserDC45028B" + } + ] + } + }, + MyGroupCBA54B1B: { + Type: "AWS::IAM::Group" + }, + MyGroupDefaultPolicy72C41231: { + Type: "AWS::IAM::Policy", + Properties: { + Groups: [ + { + Ref: "MyGroupCBA54B1B" + } + ], + PolicyDocument: { + Statement: [ + { + Action: [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + Effect: "Allow", + Resource: [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + Ref: "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + Ref: "MyAssetS3Bucket68C9B344" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + Ref: "AWS::Partition" + }, + ":", + "s3", + ":", + "", + ":", + "", + ":", + { + Ref: "MyAssetS3Bucket68C9B344" + } + ] + ] + }, + "/", + { + Ref: "MyAssetS3ObjectKeyC07605E4" + } + ] + ] + } + ] + } + ], + Version: "2012-10-17" + }, + PolicyName: "MyGroupDefaultPolicy72C41231" + } + } + }, + Parameters: { + MyAssetS3Bucket68C9B344: { + Type: "String", + Description: "S3 bucket for asset \"MyAsset\"" + }, + MyAssetS3ObjectKeyC07605E4: { + Type: "String", + Description: "S3 object for asset \"MyAsset\"" + } + } + }); + + test.done(); + }, + 'fails if directory not found'(test: Test) { + const stack = new Stack(); + test.throws(() => new ZipDirectoryAsset(stack, 'MyDirectory', { + path: '/path/not/found/' + Math.random() * 999999 + })); + test.done(); + } +}; diff --git a/packages/@aws-cdk/cx-api/lib/cxapi.ts b/packages/@aws-cdk/cx-api/lib/cxapi.ts index acb5ce231a336..9e813f3bd437c 100644 --- a/packages/@aws-cdk/cx-api/lib/cxapi.ts +++ b/packages/@aws-cdk/cx-api/lib/cxapi.ts @@ -95,3 +95,11 @@ export const DEFAULT_ACCOUNT_CONTEXT_KEY = 'aws:cdk:toolkit:default-account'; * Context parameter for the default AWS region to use if a stack's environment is not set. */ export const DEFAULT_REGION_CONTEXT_KEY = 'aws:cdk:toolkit:default-region'; + +export const ASSET_METADATA = 'aws:cdk:asset'; +export interface AssetMetadataEntry { + path: string; + packaging: 'zip' | 'file'; + s3BucketParameter: string; + s3KeyParameter: string; +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/deploy-stack.ts b/packages/aws-cdk/lib/api/deploy-stack.ts index 0b99db4f0cb5a..4e5d78027ddad 100644 --- a/packages/aws-cdk/lib/api/deploy-stack.ts +++ b/packages/aws-cdk/lib/api/deploy-stack.ts @@ -1,9 +1,9 @@ import { StackInfo, SynthesizedStack } from '@aws-cdk/cx-api'; import { CloudFormation } from 'aws-sdk'; import * as colors from 'colors/safe'; -import * as crypto from 'crypto'; import * as uuid from 'uuid'; import * as YAML from 'yamljs'; +import { prepareAssets } from '../assets'; import { debug, error } from '../logging'; import { Mode } from './aws-auth/credentials'; import { ToolkitInfo } from './toolkit-info'; @@ -31,12 +31,14 @@ export async function deployStack(stack: SynthesizedStack, throw new Error(`The stack ${stack.name} does not have an environment`); } + const params = await prepareAssets(stack, toolkitInfo); + deployName = deployName || stack.name; const executionId = uuid.v4(); const cfn = await sdk.cloudFormation(stack.environment, Mode.ForWriting); - const bodyParameter = await makeBodyParameter(stack, sdk, toolkitInfo); + const bodyParameter = await makeBodyParameter(stack, toolkitInfo); if (!await stackExists(cfn, deployName)) { await createEmptyStack(cfn, deployName, quiet); @@ -52,6 +54,7 @@ export async function deployStack(stack: SynthesizedStack, Description: `CDK Changeset for execution ${executionId}`, TemplateBody: bodyParameter.TemplateBody, TemplateURL: bodyParameter.TemplateURL, + Parameters: params, Capabilities: [ 'CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM' ] }).promise(); debug('Initiated creation of changeset: %s; waiting for it to finish creating...', changeSet.Id); @@ -110,20 +113,17 @@ async function createEmptyStack(cfn: CloudFormation, stackName: string, quiet: b * @param sdk an AWS SDK to use when interacting with S3 * @param toolkitInfo information about the toolkit stack */ -async function makeBodyParameter(stack: SynthesizedStack, sdk: SDK, toolkitInfo?: ToolkitInfo): Promise { +async function makeBodyParameter(stack: SynthesizedStack, toolkitInfo?: ToolkitInfo): Promise { const templateJson = YAML.stringify(stack.template, 16, 4); if (toolkitInfo) { - const hash = crypto.createHash('sha256').update(templateJson).digest('hex'); - const key = `cdk/${stack.name}/${hash}.yml`; - const s3 = await sdk.s3(stack.environment!, Mode.ForWriting); - await s3.putObject({ - Bucket: toolkitInfo.bucketName, - Key: key, - Body: templateJson, - ContentType: 'application/x-yaml' - }).promise(); - debug('Stored template in S3 at s3://%s/%s', toolkitInfo.bucketName, key); - return { TemplateURL: `https://${toolkitInfo.bucketName}.s3.amazonaws.com/${key}` }; + const s3KeyPrefix = `cdk/${stack.name}/`; + const s3KeySuffix = '.yml'; + const { key } = await toolkitInfo.uploadIfChanged(templateJson, { + s3KeyPrefix, s3KeySuffix, contentType: 'application/x-yaml' + }); + const templateURL = `${toolkitInfo.bucketUrl}/${key}`; + debug('Stored template in S3 at:', templateURL); + return { TemplateURL: templateURL }; } else if (templateJson.length > 51_200) { error('The template for stack %s is %d bytes long, a CDK Toolkit stack is required for deployment of templates larger than 51,200 bytes. ' + 'A CDK Toolkit stack can be created using %s', diff --git a/packages/aws-cdk/lib/api/toolkit-info.ts b/packages/aws-cdk/lib/api/toolkit-info.ts index 91e2927fa8517..57bbdbe55e6d4 100644 --- a/packages/aws-cdk/lib/api/toolkit-info.ts +++ b/packages/aws-cdk/lib/api/toolkit-info.ts @@ -1,15 +1,82 @@ import { Environment } from '@aws-cdk/cx-api'; -import { CloudFormation } from 'aws-sdk'; +import { CloudFormation, S3 } from 'aws-sdk'; import * as colors from 'colors/safe'; +import { md5hash } from '../archive'; import { debug } from '../logging'; import { Mode } from './aws-auth/credentials'; import { BUCKET_DOMAIN_NAME_OUTPUT, BUCKET_NAME_OUTPUT } from './bootstrap-environment'; import { waitForStack } from './util/cloudformation'; import { SDK } from './util/sdk'; -export interface ToolkitInfo { - bucketName: string - bucketEndpoint: string +export class ToolkitInfo { + constructor(private readonly props: { + sdk: SDK, + bucketName: string, + bucketEndpoint: string, + environment: Environment + }) { } + + public get bucketUrl() { + return `https://${this.props.bucketEndpoint}`; + } + + public get bucketName() { + return this.props.bucketName; + } + + /** + * Uploads a data blob to S3 under the specified key prefix. + * Uses md5 hash to render the full key and skips upload if an object + * already exists by this key. + */ + public async uploadIfChanged(data: any, props: { + s3KeyPrefix?: string, + s3KeySuffix?: string, + contentType?: string, + }): Promise<{ key: string, changed: boolean }> { + const s3 = await this.props.sdk.s3(this.props.environment, Mode.ForWriting); + + const s3KeyPrefix = props.s3KeyPrefix || ''; + const s3KeySuffix = props.s3KeySuffix || ''; + + const bucket = this.props.bucketName; + + const hash = md5hash(data); + const key = `${s3KeyPrefix}${hash}${s3KeySuffix}`; + const url = `s3://${bucket}/${key}`; + + debug(`${url}: checking if already exists`); + if (await objectExists(s3, bucket, key)) { + debug(`${url}: found (skipping upload)`); + return { key, changed: false }; + } + + debug(`${url}: uploading`); + await s3.putObject({ + Bucket: bucket, + Key: key, + Body: data, + ContentType: props.contentType + }).promise(); + + debug(`${url}: upload complete`); + + return { key, changed: true }; + } + +} + +async function objectExists(s3: S3, bucket: string, key: string) { + try { + await s3.headObject({ Bucket: bucket, Key: key }).promise(); + return true; + } catch (e) { + if (e.code === 'NotFound') { + return false; + } + + throw e; + } } export async function loadToolkitInfo(environment: Environment, sdk: SDK, stackName: string): Promise { @@ -20,10 +87,11 @@ export async function loadToolkitInfo(environment: Environment, sdk: SDK, stackN environment.name, stackName, colors.blue(`cdk bootstrap "${environment.name}"`)); return undefined; } - return { + return new ToolkitInfo({ + sdk, environment, bucketName: getOutputValue(stack, BUCKET_NAME_OUTPUT), bucketEndpoint: getOutputValue(stack, BUCKET_DOMAIN_NAME_OUTPUT) - }; + }); } function getOutputValue(stack: CloudFormation.Stack, output: string): string { diff --git a/packages/aws-cdk/lib/archive.ts b/packages/aws-cdk/lib/archive.ts new file mode 100644 index 0000000000000..d0495a0062b77 --- /dev/null +++ b/packages/aws-cdk/lib/archive.ts @@ -0,0 +1,21 @@ +import archiver = require('archiver'); +import crypto = require('crypto'); +import fs = require('fs-extra'); + +export function zipDirectory(directory: string, outputFile: string): Promise { + return new Promise((ok, fail) => { + const output = fs.createWriteStream(outputFile); + const archive = archiver('zip'); + archive.directory(directory, false); + archive.pipe(output); + archive.finalize(); + + archive.on('warning', fail); + archive.on('error', fail); + output.once('close', () => ok()); + }); +} + +export function md5hash(data: any) { + return crypto.createHash('sha256').update(data).digest('hex'); +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/assets.ts b/packages/aws-cdk/lib/assets.ts new file mode 100644 index 0000000000000..bd4407191b1e2 --- /dev/null +++ b/packages/aws-cdk/lib/assets.ts @@ -0,0 +1,91 @@ +import { ASSET_METADATA, AssetMetadataEntry, StackMetadata, SynthesizedStack } from "@aws-cdk/cx-api"; +import { CloudFormation } from "aws-sdk"; +import fs = require('fs-extra'); +import path = require('path'); +import { ToolkitInfo } from "./api/toolkit-info"; +import { zipDirectory } from './archive'; +import { debug, success } from "./logging"; + +export async function prepareAssets(stack: SynthesizedStack, toolkitInfo?: ToolkitInfo): Promise { + const assets = findAssets(stack.metadata); + if (assets.length === 0) { + return []; + } + + if (!toolkitInfo) { + throw new Error('Since this stack uses assets, the toolkit stack must be deployed to the environment ("cdk bootstrap")'); + } + + debug('Preparing assets'); + let params = new Array(); + for (const asset of assets) { + debug(` - ${asset.path} (${asset.packaging})`); + + params = params.concat(await prepareAsset(asset, toolkitInfo)); + } + + return params; +} + +async function prepareAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo): Promise { + debug('Preparing asset', JSON.stringify(asset)); + switch (asset.packaging) { + case 'zip': + return await prepareZipAsset(asset, toolkitInfo); + case 'file': + return await prepareFileAsset(asset, toolkitInfo); + default: + throw new Error(`Unsupported packaging type: ${asset.packaging}`); + } +} + +async function prepareZipAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo): Promise { + debug('Preparing zip asset from directory:', asset.path); + const staging = await fs.mkdtemp('/tmp/cdk-assets'); + try { + const archiveFile = path.join(staging, 'archive.zip'); + await zipDirectory(asset.path, archiveFile); + debug('zip archive:', archiveFile); + return await prepareFileAsset(asset, toolkitInfo, 'application/zip'); + } finally { + await fs.remove(staging); + } +} + +async function prepareFileAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo, contentType?: string): Promise { + debug('Preparing file asset:', asset.path); + + const data = await fs.readFile(asset.path); + + const { key, changed } = await toolkitInfo.uploadIfChanged(data, { + s3KeyPrefix: 'assets/', + s3KeySuffix: path.extname(asset.path), + contentType + }); + + const s3url = `s3://${toolkitInfo.bucketName}/${key}`; + if (changed) { + success(` 👜 Asset ${asset.path} uploaded to ${s3url}`); + } else { + success(` 👜 Asset ${asset.path} is already up-to-date in ${s3url}`); + } + + return [ + { ParameterKey: asset.s3BucketParameter, ParameterValue: toolkitInfo.bucketName }, + { ParameterKey: asset.s3KeyParameter, ParameterValue: key } + ]; +} + +function findAssets(metadata: StackMetadata): AssetMetadataEntry[] { + const assets = new Array(); + + for (const k of Object.keys(metadata)) { + for (const entry of metadata[k]) { + if (entry.type === ASSET_METADATA) { + assets.push(entry.data); + } + } + } + + return assets; +} \ No newline at end of file diff --git a/packages/aws-cdk/package-lock.json b/packages/aws-cdk/package-lock.json index 4427cef0316e9..ba923d521dd4e 100644 --- a/packages/aws-cdk/package-lock.json +++ b/packages/aws-cdk/package-lock.json @@ -1,16 +1,35 @@ { - "requires": true, + "name": "aws-cdk", + "version": "0.7.3-beta", "lockfileVersion": 1, + "requires": true, "dependencies": { + "@types/archiver": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-2.1.2.tgz", + "integrity": "sha512-UGcGgeMoGpFh97pQkzR+cgSrJDjVmO+CZ2N+WDI7i0QCOJE+PocpfHEEtePhlttULdbfOrzNi0yexg66E52Prw==", + "dev": true, + "requires": { + "@types/glob": "*" + } + }, "@types/caseless": { "version": "0.12.1", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz", - "integrity": "sha512-FhlMa34NHp9K5MY1Uz8yb+ZvuX0pnvn3jScRSNAb75KHGB8d3rEU6hqMs3Z2vjuytcMfRg6c5CHMc3wtYyD2/A==" + "integrity": "sha512-FhlMa34NHp9K5MY1Uz8yb+ZvuX0pnvn3jScRSNAb75KHGB8d3rEU6hqMs3Z2vjuytcMfRg6c5CHMc3wtYyD2/A==", + "dev": true + }, + "@types/events": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@types/events/-/events-1.2.0.tgz", + "integrity": "sha512-KEIlhXnIutzKwRbQkGWb/I4HFqBuUykAdHgDED6xqwXJfONCjF5VoE0cXEiurh3XauygxzeDzgtXUqvLkxFzzA==", + "dev": true }, "@types/form-data": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz", "integrity": "sha512-JAMFhOaHIciYVh8fb5/83nmuO/AHwmto+Hq7a9y8FzLDcC1KCU344XDOMEmahnrTFlHjgh4L0WJFczNIX2GxnQ==", + "dev": true, "requires": { "@types/node": "*" } @@ -19,29 +38,45 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-4.0.8.tgz", "integrity": "sha512-Z5nu9Pbxj9yNeXIK3UwGlRdJth4cZ5sCq05nI7FaI6B0oz28nxkOtp6Lsz0ZnmLHJGvOJfB/VHxSTbVq/i6ujA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/glob": { + "version": "5.0.35", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-5.0.35.tgz", + "integrity": "sha512-wc+VveszMLyMWFvXLkloixT4n0harUIVZjnpzztaZ0nKLuul7Z32iMt2fUFGAaZ4y1XWjFRMtCI5ewvyh4aIeg==", + "dev": true, "requires": { + "@types/events": "*", + "@types/minimatch": "*", "@types/node": "*" } }, "@types/minimatch": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", - "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==" + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true }, "@types/mockery": { "version": "1.4.29", "resolved": "https://registry.npmjs.org/@types/mockery/-/mockery-1.4.29.tgz", - "integrity": "sha1-m6It838H43gP/4Ux0aOOYz+UV6U=" + "integrity": "sha1-m6It838H43gP/4Ux0aOOYz+UV6U=", + "dev": true }, "@types/node": { "version": "8.10.17", "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.17.tgz", - "integrity": "sha512-3N3FRd/rA1v5glXjb90YdYUa+sOB7WrkU2rAhKZnF4TKD86Cym9swtulGuH0p9nxo7fP5woRNa8b0oFTpCO1bg==" + "integrity": "sha512-3N3FRd/rA1v5glXjb90YdYUa+sOB7WrkU2rAhKZnF4TKD86Cym9swtulGuH0p9nxo7fP5woRNa8b0oFTpCO1bg==", + "dev": true }, "@types/request": { "version": "2.47.1", "resolved": "https://registry.npmjs.org/@types/request/-/request-2.47.1.tgz", "integrity": "sha512-TV3XLvDjQbIeVxJ1Z3oCTDk/KuYwwcNKVwz2YaT0F5u86Prgc4syDAp6P96rkTQQ4bIdh+VswQIC9zS6NjY7/g==", + "dev": true, "requires": { "@types/caseless": "*", "@types/form-data": "*", @@ -52,12 +87,14 @@ "@types/tough-cookie": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.3.tgz", - "integrity": "sha512-MDQLxNFRLasqS4UlkWMSACMKeSm1x4Q3TxzUC7KQUsh6RK1ZrQ0VEyE3yzXcBu+K8ejVj4wuX32eUG02yNp+YQ==" + "integrity": "sha512-MDQLxNFRLasqS4UlkWMSACMKeSm1x4Q3TxzUC7KQUsh6RK1ZrQ0VEyE3yzXcBu+K8ejVj4wuX32eUG02yNp+YQ==", + "dev": true }, "@types/uuid": { "version": "3.4.3", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.3.tgz", "integrity": "sha512-5fRLCYhLtDb3hMWqQyH10qtF+Ud2JnNCXTCZ+9ktNdCcgslcuXkDTkFcJNk++MT29yDntDnlF1+jD+uVGumsbw==", + "dev": true, "requires": { "@types/node": "*" } @@ -65,12 +102,14 @@ "@types/yamljs": { "version": "0.2.30", "resolved": "https://registry.npmjs.org/@types/yamljs/-/yamljs-0.2.30.tgz", - "integrity": "sha1-0DTh0ynkbo0Pc3yajbl/aPgbU4I=" + "integrity": "sha1-0DTh0ynkbo0Pc3yajbl/aPgbU4I=", + "dev": true }, "@types/yargs": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-8.0.3.tgz", - "integrity": "sha512-YdxO7zGQf2qJeMgR0fNO8QTlj88L2zCP5GOddovoTyetgLiNDOUXcWzhWKb4EdZZlOjLQUA0JM8lW7VcKQL+9w==" + "integrity": "sha512-YdxO7zGQf2qJeMgR0fNO8QTlj88L2zCP5GOddovoTyetgLiNDOUXcWzhWKb4EdZZlOjLQUA0JM8lW7VcKQL+9w==", + "dev": true }, "ajv": { "version": "5.5.2", @@ -88,6 +127,34 @@ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" }, + "archiver": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-2.1.1.tgz", + "integrity": "sha1-/2YrSnggFJSj7lRNOjP+dJZQnrw=", + "requires": { + "archiver-utils": "^1.3.0", + "async": "^2.0.0", + "buffer-crc32": "^0.2.1", + "glob": "^7.0.0", + "lodash": "^4.8.0", + "readable-stream": "^2.0.0", + "tar-stream": "^1.5.0", + "zip-stream": "^1.2.0" + } + }, + "archiver-utils": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-1.3.0.tgz", + "integrity": "sha1-5QtMCccL89aA4y/xt5lOn52JUXQ=", + "requires": { + "glob": "^7.0.0", + "graceful-fs": "^4.1.0", + "lazystream": "^1.0.0", + "lodash": "^4.8.0", + "normalize-path": "^2.0.0", + "readable-stream": "^2.0.0" + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -106,6 +173,14 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, + "async": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz", + "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==", + "requires": { + "lodash": "^4.17.10" + } + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -156,6 +231,15 @@ "tweetnacl": "^0.14.3" } }, + "bl": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", + "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", + "requires": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -175,6 +259,30 @@ "isarray": "^1.0.0" } }, + "buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "requires": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==" + }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" + }, + "buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=" + }, "buffer-from": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.0.0.tgz", @@ -248,6 +356,17 @@ "delayed-stream": "~1.0.0" } }, + "compress-commons": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-1.2.2.tgz", + "integrity": "sha1-UkqfEJA/OoEzibAiXSfEi7dRiQ8=", + "requires": { + "buffer-crc32": "^0.2.1", + "crc32-stream": "^2.0.0", + "normalize-path": "^2.0.0", + "readable-stream": "^2.0.0" + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -258,6 +377,34 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "crc": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/crc/-/crc-3.7.0.tgz", + "integrity": "sha512-ZwmUex488OBjSVOMxnR/dIa1yxisBMJNEi+UxzXpKhax8MPsQtoRQtl5Qgo+W7pcSVkRXa3BEVjaniaWKtvKvw==", + "requires": { + "buffer": "^5.1.0" + }, + "dependencies": { + "buffer": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.1.0.tgz", + "integrity": "sha512-YkIRgwsZwJWTnyQrsBTWefizHh+8GYj3kbL1BTiAQ/9pwpino0G7B2gp5tx/FUBqUlvtxV85KNR3mwfAtv15Yw==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + } + } + }, + "crc32-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-2.0.0.tgz", + "integrity": "sha1-483TtN8xaN10494/u8t7KX/pCPQ=", + "requires": { + "crc": "^3.4.4", + "readable-stream": "^2.0.0" + } + }, "cross-spawn": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", @@ -314,6 +461,14 @@ "jsbn": "~0.1.0" } }, + "end-of-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", + "requires": { + "once": "^1.4.0" + } + }, "error-ex": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.1.tgz", @@ -389,6 +544,11 @@ "mime-types": "^2.1.12" } }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "fs-extra": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.3.tgz", @@ -599,6 +759,14 @@ "verror": "1.10.0" } }, + "lazystream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.0.tgz", + "integrity": "sha1-9plf4PggOS9hOWvolGJAe7dxaOQ=", + "requires": { + "readable-stream": "^2.0.5" + } + }, "lcid": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", @@ -678,7 +846,8 @@ "mockery": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mockery/-/mockery-2.1.0.tgz", - "integrity": "sha512-9VkOmxKlWXoDO/h1jDZaS4lH33aWfRiJiNT/tKj+8OGzrcFDLo8d0syGdbsc3Bc4GvRXPb+NMMvojotmuGJTvA==" + "integrity": "sha512-9VkOmxKlWXoDO/h1jDZaS4lH33aWfRiJiNT/tKj+8OGzrcFDLo8d0syGdbsc3Bc4GvRXPb+NMMvojotmuGJTvA==", + "dev": true }, "mute-stream": { "version": "0.0.7", @@ -696,6 +865,14 @@ "validate-npm-package-license": "^3.0.1" } }, + "normalize-path": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", + "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=", + "requires": { + "remove-trailing-separator": "^1.0.1" + } + }, "npm-run-path": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", @@ -799,6 +976,11 @@ "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, "promptly": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/promptly/-/promptly-0.2.1.tgz", @@ -854,6 +1036,25 @@ "read-pkg": "^2.0.0" } }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "remove-trailing-separator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", + "integrity": "sha1-wkvOKig62tW8P1jg1IJJuSN52O8=" + }, "request": { "version": "2.87.0", "resolved": "https://registry.npmjs.org/request/-/request-2.87.0.tgz", @@ -1020,6 +1221,14 @@ } } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", @@ -1038,6 +1247,25 @@ "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" }, + "tar-stream": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.1.tgz", + "integrity": "sha512-IFLM5wp3QrJODQFPm6/to3LJZrONdBY/otxcvDIQzu217zKye6yVR3hhi9lAjrC2Z+m/j5oDxMPb1qcd8cIvpA==", + "requires": { + "bl": "^1.0.0", + "buffer-alloc": "^1.1.0", + "end-of-stream": "^1.0.0", + "fs-constants": "^1.0.0", + "readable-stream": "^2.3.0", + "to-buffer": "^1.1.0", + "xtend": "^4.0.0" + } + }, + "to-buffer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", + "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==" + }, "tough-cookie": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", @@ -1081,6 +1309,11 @@ "querystring": "0.2.0" } }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, "uuid": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", @@ -1171,6 +1404,11 @@ "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.0.0.tgz", "integrity": "sha512-PHyM+sQouu7xspQQwELlGwwd05mXUFqwFYfqPO0cC7x4fxyHnnuetmQr6CjJiafIDoH4MogHb9dOoJzR/Y4rFg==" }, + "xtend": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" + }, "y18n": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", @@ -1236,6 +1474,17 @@ "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=" } } + }, + "zip-stream": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-1.2.0.tgz", + "integrity": "sha1-qLxF9MG0lpnGuQGYuqyqzbzUugQ=", + "requires": { + "archiver-utils": "^1.3.0", + "compress-commons": "^1.2.0", + "lodash": "^4.8.0", + "readable-stream": "^2.0.0" + } } } } diff --git a/packages/aws-cdk/package.json b/packages/aws-cdk/package.json index b92ee67b9ed2a..52847586b6aae 100644 --- a/packages/aws-cdk/package.json +++ b/packages/aws-cdk/package.json @@ -15,7 +15,9 @@ "test": "cdk-test" }, "cdk-build": { - "pre": ["./generate.sh"] + "pre": [ + "./generate.sh" + ] }, "nyc": { "lines": 8, @@ -27,6 +29,7 @@ }, "license": "LicenseRef-LICENSE", "devDependencies": { + "@types/archiver": "^2.1.2", "@types/fs-extra": "^4.0.8", "@types/minimatch": "^3.0.3", "@types/mockery": "^1.4.29", @@ -34,14 +37,15 @@ "@types/uuid": "^3.4.3", "@types/yamljs": "^0.2.0", "@types/yargs": "^8.0.3", + "cdk-build-tools": "^0.7.3-beta", "mockery": "^2.1.0", - "pkglint": "^0.7.3-beta", - "cdk-build-tools": "^0.7.3-beta" + "pkglint": "^0.7.3-beta" }, "dependencies": { "@aws-cdk/cloudformation-diff": "^0.7.3-beta", "@aws-cdk/cx-api": "^0.7.3-beta", "@aws-cdk/util": "^0.7.3-beta", + "archiver": "^2.1.1", "aws-sdk": "^2.259.1", "camelcase": "^5.0.0", "colors": "^1.2.1", diff --git a/packages/aws-cdk/test/test-archive/file1.txt b/packages/aws-cdk/test/test-archive/file1.txt new file mode 100644 index 0000000000000..7bb7edbd4b634 --- /dev/null +++ b/packages/aws-cdk/test/test-archive/file1.txt @@ -0,0 +1 @@ +I am file1 \ No newline at end of file diff --git a/packages/aws-cdk/test/test-archive/file2.txt b/packages/aws-cdk/test/test-archive/file2.txt new file mode 100644 index 0000000000000..ccb69856e7157 --- /dev/null +++ b/packages/aws-cdk/test/test-archive/file2.txt @@ -0,0 +1,2 @@ +I am file2 +BLA! \ No newline at end of file diff --git a/packages/aws-cdk/test/test-archive/subdir/file3.txt b/packages/aws-cdk/test/test-archive/subdir/file3.txt new file mode 100644 index 0000000000000..976606ef5a8ac --- /dev/null +++ b/packages/aws-cdk/test/test-archive/subdir/file3.txt @@ -0,0 +1 @@ +I am in a subdirectory diff --git a/packages/aws-cdk/test/test.archive.ts b/packages/aws-cdk/test/test.archive.ts new file mode 100644 index 0000000000000..48bdb9849ae99 --- /dev/null +++ b/packages/aws-cdk/test/test.archive.ts @@ -0,0 +1,46 @@ +import { exec as _exec } from 'child_process'; +import fs = require('fs-extra'); +import { Test } from 'nodeunit'; +import path = require('path'); +import { promisify } from 'util'; +import { md5hash, zipDirectory } from '../lib/archive'; +const exec = promisify(_exec); + +export = { + async 'zipDirectory can take a directory and produce a zip from it'(test: Test) { + const stagingDir = await fs.mkdtemp('/tmp/test.archive'); + const zipFile = path.join(stagingDir, 'output.zip'); + const originalDir = path.join(__dirname, 'test-archive'); + const extractDir = await fs.mkdtemp('/tmp/test.archive.extract'); + await zipDirectory(originalDir, zipFile); + + // unzip and verify that the resulting tree is the same + await exec(`unzip ${zipFile}`, { cwd: extractDir }); + + try { + await exec(`diff -bur ${originalDir} ${extractDir}`); + } catch (e) { + test.ok(false, `extracted directory ${extractDir} differs from original ${originalDir}`); + } + + await fs.remove(stagingDir); + await fs.remove(extractDir); + test.done(); + }, + + async 'md5 hash of a zip stays consistent across invocations'(test: Test) { + const stagingDir = await fs.mkdtemp('/tmp/test.archive'); + const zipFile1 = path.join(stagingDir, 'output.zip'); + const zipFile2 = path.join(stagingDir, 'output.zip'); + const originalDir = path.join(__dirname, 'test-archive'); + await zipDirectory(originalDir, zipFile1); + await new Promise(ok => setTimeout(ok, 2000)); // wait 2s + await zipDirectory(originalDir, zipFile2); + + const hash1 = md5hash(await fs.readFile(zipFile1)); + const hash2 = md5hash(await fs.readFile(zipFile2)); + + test.deepEqual(hash1, hash2, 'md5 hash of two zips of the same content are not the same'); + test.done(); + } +};