Skip to content

Commit

Permalink
feat(toolkit): add '--reuse-asset' option (#1918)
Browse files Browse the repository at this point in the history 
Reusing assets avoids rebuilding an asset and just reuses the currently
deployed one. Especially helpful for Docker containers that take a long
time to build.

Fixes #1916
  • Loading branch information
@rix0rrr
rix0rrr authored Mar 6, 2019
1 parent aa08b95 commit 1767b61
Show file tree
Hide file tree
Showing 10 changed files with 327 additions and 111 deletions.
87 changes: 14 additions & 73 deletions packages/aws-cdk/bin/cdk.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ import fs = require('fs-extra');
import util = require('util');
import yargs = require('yargs');

import { bootstrapEnvironment, deployStack, destroyStack, loadToolkitInfo, SDK } from '../lib';
import { bootstrapEnvironment, destroyStack, SDK } from '../lib';
import { environmentsFromDescriptors, globEnvironmentsFromStacks } from '../lib/api/cxapp/environments';
import { execProgram } from '../lib/api/cxapp/exec';
import { AppStacks, ExtendedStackSelection, listStackNames } from '../lib/api/cxapp/stacks';
import { CloudFormationDeploymentTarget } from '../lib/api/deployment-target';
import { CloudFormationDeploymentTarget, DEFAULT_TOOLKIT_STACK_NAME } from '../lib/api/deployment-target';
import { leftPad } from '../lib/api/util/string-manipulation';
import { CdkToolkit } from '../lib/cdk-toolkit';
import { printSecurityDiff, RequireApproval } from '../lib/diff';
import { RequireApproval } from '../lib/diff';
import { availableInitLanguages, cliInit, printAvailableTemplates } from '../lib/init';
import { interactive } from '../lib/interactive';
import { data, debug, error, highlight, print, setVerbose, success } from '../lib/logging';
Expand All @@ -27,8 +27,6 @@ import { VERSION } from '../lib/version';
const promptly = require('promptly');
const confirm = util.promisify(promptly.confirm);

const DEFAULT_TOOLKIT_STACK_NAME = 'CDKToolkit';

// tslint:disable:no-shadowed-variable max-line-length
async function parseCommandLineArguments() {
const initTemplateLanuages = await availableInitLanguages;
Expand Down Expand Up @@ -61,6 +59,7 @@ async function parseCommandLineArguments() {
.option('numbered', { type: 'boolean', alias: 'n', desc: 'prefix filenames with numbers to indicate deployment ordering' }))
.command('bootstrap [ENVIRONMENTS..]', 'Deploys the CDK toolkit stack into an AWS environment')
.command('deploy [STACKS..]', 'Deploys the stack(s) named STACKS into your AWS account', yargs => yargs
.option('build-exclude', { type: 'array', alias: 'E', nargs: 1, desc: 'do not rebuild asset with the given ID. Can be specified multiple times.', default: [] })
.option('exclusively', { type: 'boolean', alias: 'e', desc: 'only deploy requested stacks, don\'t include dependencies' })
.option('require-approval', { type: 'string', choices: [RequireApproval.Never, RequireApproval.AnyChange, RequireApproval.Broadening], desc: 'what security-sensitive changes need manual approval' }))
.option('ci', { type: 'boolean', desc: 'Force CI detection. Use --no-ci to disable CI autodetection.', default: process.env.CI !== undefined })
Expand All @@ -87,6 +86,7 @@ async function parseCommandLineArguments() {
].join('\n\n'))
.argv;
}

if (!process.stdout.isTTY) {
colors.disable();
}
Expand Down Expand Up @@ -191,7 +191,15 @@ async function initCommandLine() {
return await cliBootstrap(args.ENVIRONMENTS, toolkitStackName, args.roleArn);

case 'deploy':
return await cliDeploy(args.STACKS, args.exclusively, toolkitStackName, args.roleArn, configuration.settings.get(['requireApproval']), args.ci);
return await cli.deploy({
stackNames: args.STACKS,
exclusively: args.exclusively,
toolkitStackName,
roleArn: args.roleArn,
requireApproval: configuration.settings.get(['requireApproval']),
ci: args.ci,
reuseAssets: args['build-exclude']
});

case 'destroy':
return await cliDestroy(args.STACKS, args.exclusively, args.force, args.roleArn);
Expand Down Expand Up @@ -340,73 +348,6 @@ async function initCommandLine() {
return 0; // exit-code
}

async function cliDeploy(stackNames: string[],
exclusively: boolean,
toolkitStackName: string,
roleArn: string | undefined,
requireApproval: RequireApproval,
ci: boolean) {
if (requireApproval === undefined) { requireApproval = RequireApproval.Broadening; }

const stacks = await appStacks.selectStacks(stackNames, exclusively ? ExtendedStackSelection.None : ExtendedStackSelection.Upstream);

for (const stack of stacks) {
if (stacks.length !== 1) { highlight(stack.name); }
if (!stack.environment) {
// tslint:disable-next-line:max-line-length
throw new Error(`Stack ${stack.name} does not define an environment, and AWS credentials could not be obtained from standard locations or no region was configured.`);
}
const toolkitInfo = await loadToolkitInfo(stack.environment, aws, toolkitStackName);

if (requireApproval !== RequireApproval.Never) {
const currentTemplate = await provisioner.readCurrentTemplate(stack);
if (printSecurityDiff(currentTemplate, stack, requireApproval)) {

// only talk to user if we STDIN is a terminal (otherwise, fail)
if (!process.stdin.isTTY) {
throw new Error(
'"--require-approval" is enabled and stack includes security-sensitive updates, ' +
'but terminal (TTY) is not attached so we are unable to get a confirmation from the user');
}

const confirmed = await confirm(`Do you wish to deploy these changes (y/n)?`);
if (!confirmed) { throw new Error('Aborted by user'); }
}
}

if (stack.name !== stack.originalName) {
print('%s: deploying... (was %s)', colors.bold(stack.name), colors.bold(stack.originalName));
} else {
print('%s: deploying...', colors.bold(stack.name));
}

try {
const result = await deployStack({ stack, sdk: aws, toolkitInfo, deployName: stack.name, roleArn, ci });
const message = result.noOp
? ` ✅ %s (no changes)`
: ` ✅ %s`;

success('\n' + message, stack.name);

if (Object.keys(result.outputs).length > 0) {
print('\nOutputs:');
}

for (const name of Object.keys(result.outputs)) {
const value = result.outputs[name];
print('%s.%s = %s', colors.cyan(stack.name), colors.cyan(name), colors.underline(colors.cyan(value)));
}

print('\nStack ARN:');

data(result.stackArn);
} catch (e) {
error('\n ❌ %s failed: %s', colors.bold(stack.name), e);
throw e;
}
}
}

async function cliDestroy(stackNames: string[], exclusively: boolean, force: boolean, roleArn: string | undefined) {
const stacks = await appStacks.selectStacks(stackNames, exclusively ? ExtendedStackSelection.None : ExtendedStackSelection.Downstream);

Expand Down
3 changes: 2 additions & 1 deletion packages/aws-cdk/lib/api/deploy-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ export interface DeployStackOptions {
deployName?: string;
quiet?: boolean;
ci?: boolean;
reuseAssets?: string[];
}

const LARGE_TEMPLATE_SIZE_KB = 50;
Expand All @@ -40,7 +41,7 @@ export async function deployStack(options: DeployStackOptions): Promise<DeploySt
throw new Error(`The stack ${options.stack.name} does not have an environment`);
}

const params = await prepareAssets(options.stack, options.toolkitInfo, options.ci);
const params = await prepareAssets(options.stack, options.toolkitInfo, options.ci, options.reuseAssets);

const deployName = options.deployName || options.stack.name;

Expand Down
36 changes: 34 additions & 2 deletions packages/aws-cdk/lib/api/deployment-target.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,12 @@ import cxapi = require('@aws-cdk/cx-api');
import { debug } from '../logging';
import { deserializeStructure } from '../serialize';
import { Mode } from './aws-auth/credentials';
import { deployStack, DeployStackResult } from './deploy-stack';
import { loadToolkitInfo } from './toolkit-info';
import { SDK } from './util/sdk';

export const DEFAULT_TOOLKIT_STACK_NAME = 'CDKToolkit';

export type Template = { [key: string]: any };

/**
Expand All @@ -13,6 +17,17 @@ export type Template = { [key: string]: any };
*/
export interface IDeploymentTarget {
readCurrentTemplate(stack: cxapi.SynthesizedStack): Promise<Template>;
deployStack(options: DeployStackOptions): Promise<DeployStackResult>;
}

export interface DeployStackOptions {
stack: cxapi.SynthesizedStack;
roleArn?: string;
deployName?: string;
quiet?: boolean;
ci?: boolean;
toolkitStackName?: string;
reuseAssets?: string[];
}

export interface ProvisionerProps {
Expand All @@ -23,13 +38,16 @@ export interface ProvisionerProps {
* Default provisioner (applies to CloudFormation).
*/
export class CloudFormationDeploymentTarget implements IDeploymentTarget {
constructor(private readonly props: ProvisionerProps) {
private readonly aws: SDK;

constructor(props: ProvisionerProps) {
this.aws = props.aws;
}

public async readCurrentTemplate(stack: cxapi.SynthesizedStack): Promise<Template> {
debug(`Reading existing template for stack ${stack.name}.`);

const cfn = await this.props.aws.cloudFormation(stack.environment, Mode.ForReading);
const cfn = await this.aws.cloudFormation(stack.environment, Mode.ForReading);
try {
const response = await cfn.getTemplate({ StackName: stack.name }).promise();
return (response.TemplateBody && deserializeStructure(response.TemplateBody)) || {};
Expand All @@ -41,4 +59,18 @@ export class CloudFormationDeploymentTarget implements IDeploymentTarget {
}
}
}

public async deployStack(options: DeployStackOptions): Promise<DeployStackResult> {
const toolkitInfo = await loadToolkitInfo(options.stack.environment, this.aws, options.toolkitStackName || DEFAULT_TOOLKIT_STACK_NAME);
return deployStack({
stack: options.stack,
deployName: options.deployName,
roleArn: options.roleArn,
quiet: options.quiet,
sdk: this.aws,
ci: options.ci,
reuseAssets: options.reuseAssets,
toolkitInfo,
});
}
}
44 changes: 33 additions & 11 deletions packages/aws-cdk/lib/assets.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,11 @@ import { zipDirectory } from './archive';
import { prepareContainerAsset } from './docker';
import { debug, success } from './logging';

export async function prepareAssets(stack: SynthesizedStack, toolkitInfo?: ToolkitInfo, ci?: boolean): Promise<CloudFormation.Parameter[]> {
// tslint:disable-next-line:max-line-length
export async function prepareAssets(stack: SynthesizedStack, toolkitInfo?: ToolkitInfo, ci?: boolean, reuse?: string[]): Promise<CloudFormation.Parameter[]> {
reuse = reuse || [];
const assets = findAssets(stack.metadata);

if (assets.length === 0) {
return [];
}
Expand All @@ -21,40 +24,51 @@ export async function prepareAssets(stack: SynthesizedStack, toolkitInfo?: Toolk
throw new Error(`This stack uses assets, so the toolkit stack must be deployed to the environment (Run "${colors.blue("cdk bootstrap " + stack.environment!.name)}")`);
}

debug('Preparing assets');
let params = new Array<CloudFormation.Parameter>();
for (const asset of assets) {
debug(` - ${asset.path} (${asset.packaging})`);
// FIXME: Should have excluded by construct path here instead of by unique ID, preferably using
// minimatch so we can support globs. Maybe take up during artifact refactoring.
const reuseAsset = reuse.indexOf(asset.id) > -1;

if (reuseAsset) {
debug(`Preparing asset ${asset.id}: ${JSON.stringify(asset)} (reusing)`);
} else {
debug(`Preparing asset ${asset.id}: ${JSON.stringify(asset)}`);
}

params = params.concat(await prepareAsset(asset, toolkitInfo, ci));
params = params.concat(await prepareAsset(asset, toolkitInfo, reuseAsset, ci));
}

return params;
}

async function prepareAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo, ci?: boolean): Promise<CloudFormation.Parameter[]> {
debug('Preparing asset', JSON.stringify(asset));
// tslint:disable-next-line:max-line-length
async function prepareAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo, reuse: boolean, ci?: boolean): Promise<CloudFormation.Parameter[]> {
switch (asset.packaging) {
case 'zip':
return await prepareZipAsset(asset, toolkitInfo);
return await prepareZipAsset(asset, toolkitInfo, reuse);
case 'file':
return await prepareFileAsset(asset, toolkitInfo);
return await prepareFileAsset(asset, toolkitInfo, reuse);
case 'container-image':
return await prepareContainerAsset(asset, toolkitInfo, ci);
return await prepareContainerAsset(asset, toolkitInfo, reuse, ci);
default:
// tslint:disable-next-line:max-line-length
throw new Error(`Unsupported packaging type: ${(asset as any).packaging}. You might need to upgrade your aws-cdk toolkit to support this asset type.`);
}
}

async function prepareZipAsset(asset: FileAssetMetadataEntry, toolkitInfo: ToolkitInfo): Promise<CloudFormation.Parameter[]> {
async function prepareZipAsset(asset: FileAssetMetadataEntry, toolkitInfo: ToolkitInfo, reuse: boolean): Promise<CloudFormation.Parameter[]> {
if (reuse) {
return await prepareFileAsset(asset, toolkitInfo, reuse);
}

debug('Preparing zip asset from directory:', asset.path);
const staging = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-assets'));
try {
const archiveFile = path.join(staging, 'archive.zip');
await zipDirectory(asset.path, archiveFile);
debug('zip archive:', archiveFile);
return await prepareFileAsset(asset, toolkitInfo, archiveFile, 'application/zip');
return await prepareFileAsset(asset, toolkitInfo, reuse, archiveFile, 'application/zip');
} finally {
await fs.remove(staging);
}
Expand All @@ -69,9 +83,17 @@ async function prepareZipAsset(asset: FileAssetMetadataEntry, toolkitInfo: Toolk
async function prepareFileAsset(
asset: FileAssetMetadataEntry,
toolkitInfo: ToolkitInfo,
reuse: boolean,
filePath?: string,
contentType?: string): Promise<CloudFormation.Parameter[]> {

if (reuse) {
return [
{ ParameterKey: asset.s3BucketParameter, UsePreviousValue: true },
{ ParameterKey: asset.s3KeyParameter, UsePreviousValue: true },
];
}

filePath = filePath || asset.path;
debug('Preparing file asset:', filePath);

Expand Down
Loading

0 comments on commit 1767b61

Please sign in to comment.