From a315a05396d55b7ba3ac45cd8f3976843ff48bfd Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Mon, 25 Feb 2019 15:17:50 +0100 Subject: [PATCH 1/4] fix(toolkit): support diff on multiple stacks If there are stack dependencies, 'diff' would fail because it does not know how to diff multiple stacks. Make diff support the same stack selection mechanisms as 'cdk deploy'. Move 'stack rename' facilities into the class that deals with the CDK app, which is the source of thruth for stacks. This way, all downstream code doesn't have to deal with the renames every time. Start factoring out toolkit code into logical layers. Introducing the class `CdkToolkit`, which represents the toolkit logic and forms the bridge between `AppStacks` which deals with the CDK model source (probably needs to be renamed to something better) and `CfnProvisioner`, which deals with the deployed stacks. N.B.: The indirection to a provisioner class with an interface is because the interface is going to be complex (therefore, interface over a set of functions that take callbacks) and we want to depend just on the interface so it's easy to stub out for testing. --- .../cloudformation-diff/lib/format.ts | 8 +- packages/aws-cdk/bin/cdk.ts | 100 ++++++----------- packages/aws-cdk/lib/api/cxapp/stacks.ts | 42 +++++++- packages/aws-cdk/lib/api/provisioner.ts | 44 ++++++++ packages/aws-cdk/lib/cdk-toolkit.ts | 101 ++++++++++++++++++ packages/aws-cdk/lib/diff.ts | 11 +- packages/aws-cdk/test/api/test.stacks.ts | 20 ++++ packages/aws-cdk/test/test.diff.ts | 86 +++++++++++++++ 8 files changed, 336 insertions(+), 76 deletions(-) create mode 100644 packages/aws-cdk/lib/api/provisioner.ts create mode 100644 packages/aws-cdk/lib/cdk-toolkit.ts create mode 100644 packages/aws-cdk/test/test.diff.ts diff --git a/packages/@aws-cdk/cloudformation-diff/lib/format.ts b/packages/@aws-cdk/cloudformation-diff/lib/format.ts index fcf6c35a2b162..740418b9c711e 100644 --- a/packages/@aws-cdk/cloudformation-diff/lib/format.ts +++ b/packages/@aws-cdk/cloudformation-diff/lib/format.ts @@ -11,6 +11,10 @@ import { SecurityGroupChanges } from './network/security-group-changes'; // tslint:disable-next-line:no-var-requires const { structuredPatch } = require('diff'); +export interface FormatStream extends NodeJS.WritableStream { + columns?: number; +} + /** * Renders template differences to the process' console. * @@ -20,7 +24,7 @@ const { structuredPatch } = require('diff'); * case there is no aws:cdk:path metadata in the template. * @param context the number of context lines to use in arbitrary JSON diff (defaults to 3). */ -export function formatDifferences(stream: NodeJS.WriteStream, +export function formatDifferences(stream: FormatStream, templateDiff: TemplateDiff, logicalToPathMap: { [logicalId: string]: string } = { }, context: number = 3) { @@ -72,7 +76,7 @@ const UPDATE = colors.yellow('[~]'); const REMOVAL = colors.red('[-]'); class Formatter { - constructor(private readonly stream: NodeJS.WriteStream, + constructor(private readonly stream: FormatStream, private readonly logicalToPathMap: { [logicalId: string]: string }, diff?: TemplateDiff, private readonly context: number = 3) { diff --git a/packages/aws-cdk/bin/cdk.ts b/packages/aws-cdk/bin/cdk.ts index 5ec9d31ab6ad7..a23cb8c5ade88 100644 --- a/packages/aws-cdk/bin/cdk.ts +++ b/packages/aws-cdk/bin/cdk.ts @@ -1,24 +1,25 @@ #!/usr/bin/env node import 'source-map-support/register'; -import cxapi = require('@aws-cdk/cx-api'); import colors = require('colors/safe'); import fs = require('fs-extra'); import util = require('util'); import yargs = require('yargs'); -import { bootstrapEnvironment, deployStack, destroyStack, loadToolkitInfo, Mode, SDK } from '../lib'; +import { bootstrapEnvironment, deployStack, destroyStack, loadToolkitInfo, SDK } from '../lib'; import { environmentsFromDescriptors, globEnvironmentsFromStacks } from '../lib/api/cxapp/environments'; import { execProgram } from '../lib/api/cxapp/exec'; import { AppStacks, ExtendedStackSelection, listStackNames } from '../lib/api/cxapp/stacks'; +import { CfnProvisioner } from '../lib/api/provisioner'; import { leftPad } from '../lib/api/util/string-manipulation'; -import { printSecurityDiff, printStackDiff, RequireApproval } from '../lib/diff'; +import { CdkToolkit } from '../lib/cdk-toolkit'; +import { printSecurityDiff, RequireApproval } from '../lib/diff'; import { availableInitLanguages, cliInit, printAvailableTemplates } from '../lib/init'; import { interactive } from '../lib/interactive'; import { data, debug, error, highlight, print, setVerbose, success } from '../lib/logging'; import { PluginHost } from '../lib/plugin'; import { parseRenames } from '../lib/renames'; -import { deserializeStructure, serializeStructure } from '../lib/serialize'; +import { serializeStructure } from '../lib/serialize'; import { Configuration, Settings } from '../lib/settings'; import { VERSION } from '../lib/version'; @@ -66,7 +67,8 @@ async function parseCommandLineArguments() { .command('destroy [STACKS..]', 'Destroy the stack(s) named STACKS', yargs => yargs .option('exclusively', { type: 'boolean', alias: 'x', desc: 'only deploy requested stacks, don\'t include dependees' }) .option('force', { type: 'boolean', alias: 'f', desc: 'Do not ask for confirmation before destroying the stacks' })) - .command('diff [STACK]', 'Compares the specified stack with the deployed stack or a local template file, and returns with status 1 if any difference is found', yargs => yargs + .command('diff [STACKS..]', 'Compares the specified stack with the deployed stack or a local template file, and returns with status 1 if any difference is found', yargs => yargs + .option('exclusively', { type: 'boolean', alias: 'e', desc: 'only deploy requested stacks, don\'t include dependencies' }) .option('context-lines', { type: 'number', desc: 'number of context lines to include in arbitrary JSON diff rendering', default: 3 }) .option('template', { type: 'string', desc: 'the path to the CloudFormation template to compare with' }) .option('strict', { type: 'boolean', desc: 'do not filter out AWS::CDK::Metadata resources', default: false })) @@ -108,13 +110,17 @@ async function initCommandLine() { await configuration.load(); configuration.logDefaults(); + const provisioner = new CfnProvisioner({ aws }); + const appStacks = new AppStacks({ verbose: argv.trace || argv.verbose, ignoreErrors: argv.ignoreErrors, strict: argv.strict, - configuration, aws, synthesizer: execProgram }); - - const renames = parseRenames(argv.rename); + configuration, + aws, + synthesizer: execProgram, + renames: parseRenames(argv.rename) + }); /** Function to load plug-ins, using configurations additively. */ function loadPlugins(...settings: Settings[]) { @@ -166,13 +172,21 @@ async function initCommandLine() { args.STACKS = args.STACKS || []; args.ENVIRONMENTS = args.ENVIRONMENTS || []; + const cli = new CdkToolkit({ appStacks, provisioner }); + switch (command) { case 'ls': case 'list': return await cliList({ long: args.long }); case 'diff': - return await diffStack(await findStack(args.STACK), args.template, args.strict, args.contextLines); + return await cli.diff({ + stackNames: args.STACKS, + exclusively: args.exclusively, + templatePath: args.template, + strict: args.strict, + contextLines: args.contextLines + }); case 'bootstrap': return await cliBootstrap(args.ENVIRONMENTS, toolkitStackName, args.roleArn); @@ -259,7 +273,6 @@ async function initCommandLine() { const autoSelectDependencies = !exclusively && outputDir !== undefined; const stacks = await appStacks.selectStacks(stackNames, autoSelectDependencies ? ExtendedStackSelection.Upstream : ExtendedStackSelection.None); - renames.validateSelectedStacks(stacks); if (doInteractive) { if (stacks.length !== 1) { @@ -295,9 +308,8 @@ async function initCommandLine() { let i = 0; for (const stack of stacks) { - const finalName = renames.finalName(stack.name); const prefix = numbered ? leftPad(`${i}`, 3, '0') + '.' : ''; - const fileName = `${outputDir}/${prefix}${finalName}.template.${json ? 'json' : 'yaml'}`; + const fileName = `${outputDir}/${prefix}${stack.name}.template.${json ? 'json' : 'yaml'}`; highlight(fileName); await fs.writeFile(fileName, toJsonOrYaml(stack.template)); i++; @@ -338,7 +350,6 @@ async function initCommandLine() { if (requireApproval === undefined) { requireApproval = RequireApproval.Broadening; } const stacks = await appStacks.selectStacks(stackNames, exclusively ? ExtendedStackSelection.None : ExtendedStackSelection.Upstream); - renames.validateSelectedStacks(stacks); for (const stack of stacks) { if (stacks.length !== 1) { highlight(stack.name); } @@ -347,10 +358,9 @@ async function initCommandLine() { throw new Error(`Stack ${stack.name} does not define an environment, and AWS credentials could not be obtained from standard locations or no region was configured.`); } const toolkitInfo = await loadToolkitInfo(stack.environment, aws, toolkitStackName); - const deployName = renames.finalName(stack.name); if (requireApproval !== RequireApproval.Never) { - const currentTemplate = await readCurrentTemplate(stack); + const currentTemplate = await provisioner.readCurrentTemplate(stack); if (printSecurityDiff(currentTemplate, stack, requireApproval)) { // only talk to user if we STDIN is a terminal (otherwise, fail) @@ -365,14 +375,14 @@ async function initCommandLine() { } } - if (deployName !== stack.name) { - print('%s: deploying... (was %s)', colors.bold(deployName), colors.bold(stack.name)); + if (stack.name !== stack.originalName) { + print('%s: deploying... (was %s)', colors.bold(stack.name), colors.bold(stack.originalName)); } else { print('%s: deploying...', colors.bold(stack.name)); } try { - const result = await deployStack({ stack, sdk: aws, toolkitInfo, deployName, roleArn, ci }); + const result = await deployStack({ stack, sdk: aws, toolkitInfo, deployName: stack.name, roleArn, ci }); const message = result.noOp ? ` ✅ %s (no changes)` : ` ✅ %s`; @@ -385,7 +395,7 @@ async function initCommandLine() { for (const name of Object.keys(result.outputs)) { const value = result.outputs[name]; - print('%s.%s = %s', colors.cyan(deployName), colors.cyan(name), colors.underline(colors.cyan(value))); + print('%s.%s = %s', colors.cyan(stack.name), colors.cyan(name), colors.underline(colors.cyan(value))); } print('\nStack ARN:'); @@ -404,8 +414,6 @@ async function initCommandLine() { // The stacks will have been ordered for deployment, so reverse them for deletion. stacks.reverse(); - renames.validateSelectedStacks(stacks); - if (!force) { // tslint:disable-next-line:max-line-length const confirmed = await confirm(`Are you sure you want to delete: ${colors.blue(stacks.map(s => s.name).join(', '))} (y/n)?`); @@ -415,59 +423,17 @@ async function initCommandLine() { } for (const stack of stacks) { - const deployName = renames.finalName(stack.name); - - success('%s: destroying...', colors.blue(deployName)); + success('%s: destroying...', colors.blue(stack.name)); try { - await destroyStack({ stack, sdk: aws, deployName, roleArn }); - success('\n ✅ %s: destroyed', colors.blue(deployName)); + await destroyStack({ stack, sdk: aws, deployName: stack.name, roleArn }); + success('\n ✅ %s: destroyed', colors.blue(stack.name)); } catch (e) { - error('\n ❌ %s: destroy failed', colors.blue(deployName), e); + error('\n ❌ %s: destroy failed', colors.blue(stack.name), e); throw e; } } } - async function diffStack(stackName: string, templatePath: string | undefined, strict: boolean, context: number): Promise { - const stack = await appStacks.synthesizeStack(stackName); - const currentTemplate = await readCurrentTemplate(stack, templatePath); - if (printStackDiff(currentTemplate, stack, strict, context) === 0) { - return 0; - } else { - return 1; - } - } - - async function readCurrentTemplate(stack: cxapi.SynthesizedStack, templatePath?: string): Promise<{ [key: string]: any }> { - if (templatePath) { - if (!await fs.pathExists(templatePath)) { - throw new Error(`There is no file at ${templatePath}`); - } - const fileContent = await fs.readFile(templatePath, { encoding: 'UTF-8' }); - return parseTemplate(fileContent); - } else { - const stackName = renames.finalName(stack.name); - debug(`Reading existing template for stack ${stackName}.`); - - const cfn = await aws.cloudFormation(stack.environment, Mode.ForReading); - try { - const response = await cfn.getTemplate({ StackName: stackName }).promise(); - return (response.TemplateBody && parseTemplate(response.TemplateBody)) || {}; - } catch (e) { - if (e.code === 'ValidationError' && e.message === `Stack with id ${stackName} does not exist`) { - return {}; - } else { - throw e; - } - } - } - - /* Attempt to parse YAML, fall back to JSON. */ - function parseTemplate(text: string): any { - return deserializeStructure(text); - } - } - /** * Match a single stack from the list of available stacks */ diff --git a/packages/aws-cdk/lib/api/cxapp/stacks.ts b/packages/aws-cdk/lib/api/cxapp/stacks.ts index 7ae9ee348a79a..857dd1bfd2eea 100644 --- a/packages/aws-cdk/lib/api/cxapp/stacks.ts +++ b/packages/aws-cdk/lib/api/cxapp/stacks.ts @@ -3,6 +3,7 @@ import colors = require('colors/safe'); import minimatch = require('minimatch'); import contextproviders = require('../../context-providers'); import { debug, error, print, warning } from '../../logging'; +import { Renames } from '../../renames'; import { Configuration, Settings } from '../../settings'; import cdkUtil = require('../../util'); import { SDK } from '../util/sdk'; @@ -42,6 +43,11 @@ export interface AppStacksProps { */ aws: SDK; + /** + * Renames to apply + */ + renames?: Renames; + /** * Callback invoked to synthesize the actual stacks */ @@ -59,8 +65,10 @@ export class AppStacks { * we can invoke it once and cache the response for subsequent calls. */ private cachedResponse?: cxapi.SynthesizeResponse; + private readonly renames: Renames; constructor(private readonly props: AppStacksProps) { + this.renames = props.renames || new Renames({}); } /** @@ -69,7 +77,7 @@ export class AppStacks { * It's an error if there are no stacks to select, or if one of the requested parameters * refers to a nonexistant stack. */ - public async selectStacks(selectors: string[], extendedSelection: ExtendedStackSelection): Promise { + public async selectStacks(selectors: string[], extendedSelection: ExtendedStackSelection): Promise { selectors = selectors.filter(s => s != null); // filter null/undefined const stacks: cxapi.SynthesizedStack[] = await this.listStacks(); @@ -79,7 +87,7 @@ export class AppStacks { if (selectors.length === 0) { debug('Stack name not specified, so defaulting to all available stacks: ' + listStackNames(stacks)); - return stacks; + return this.applyRenames(stacks); } const allStacks = new Map(); @@ -118,7 +126,7 @@ export class AppStacks { // Only check selected stacks for errors this.processMessages(selectedList); - return selectedList; + return this.applyRenames(selectedList); } /** @@ -128,6 +136,8 @@ export class AppStacks { * topologically sorted order. If there are dependencies that are not in the * set, they will be ignored; it is the user's responsibility that the * non-selected stacks have already been deployed previously. + * + * Renames are *NOT* applied in list mode. */ public async listStacks(): Promise { const response = await this.synthesizeStacks(); @@ -137,13 +147,13 @@ export class AppStacks { /** * Synthesize a single stack */ - public async synthesizeStack(stackName: string): Promise { + public async synthesizeStack(stackName: string): Promise { const resp = await this.synthesizeStacks(); const stack = resp.stacks.find(s => s.name === stackName); if (!stack) { throw new Error(`Stack ${stackName} not found`); } - return stack; + return this.applyRenames([stack])[0]; } /** @@ -253,6 +263,21 @@ export class AppStacks { logFn(` ${entry.trace.join('\n ')}`); } } + + private applyRenames(stacks: cxapi.SynthesizedStack[]): SelectedStack[] { + this.renames.validateSelectedStacks(stacks); + + const ret = []; + for (const stack of stacks) { + ret.push({ + ...stack, + originalName: stack.name, + name: this.renames.finalName(stack.name), + }); + } + + return ret; + } } /** @@ -335,4 +360,11 @@ function includeUpstreamStacks(selectedStacks: Map 0) { print('Including dependency stacks: %s', colors.bold(added.join(', '))); } +} + +export interface SelectedStack extends cxapi.SynthesizedStack { + /** + * The original name of the stack before renaming + */ + originalName: string; } \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/provisioner.ts b/packages/aws-cdk/lib/api/provisioner.ts new file mode 100644 index 0000000000000..adccd0035b768 --- /dev/null +++ b/packages/aws-cdk/lib/api/provisioner.ts @@ -0,0 +1,44 @@ +import cxapi = require('@aws-cdk/cx-api'); +import { debug } from '../logging'; +import { deserializeStructure } from '../serialize'; +import { Mode } from './aws-auth/credentials'; +import { SDK } from './util/sdk'; + +export type Template = { [key: string]: any }; + +/** + * Interface for provisioners + * + * Provisioners apply templates to the cloud infrastructure. + */ +export interface IProvisioner { + readCurrentTemplate(stack: cxapi.SynthesizedStack): Promise