From 772cb3db027c10921146932ff63e1a47a5345460 Mon Sep 17 00:00:00 2001 From: Kendra Neil <53584728+TheRealAmazonKendra@users.noreply.github.com> Date: Mon, 18 Sep 2023 12:38:49 -0700 Subject: [PATCH] chore: add --compress flag to cdk migrate This also updates some of the aliases used because they clashed with other flags. We'll just remove these. The archive file is shamelessly ripped off from the cdk-assets package. --- packages/aws-cdk/lib/cdk-toolkit.ts | 8 +- packages/aws-cdk/lib/cli.ts | 19 ++-- packages/aws-cdk/lib/commands/migrate.ts | 22 ++++- packages/aws-cdk/lib/init.ts | 35 ++++--- packages/aws-cdk/lib/util/archive.ts | 90 ++++++++++++++++++ .../aws-cdk/test/commands/migrate.test.ts | 25 +++++ packages/aws-cdk/test/init.test.ts | 92 ++++++++++++++++--- 7 files changed, 249 insertions(+), 42 deletions(-) create mode 100644 packages/aws-cdk/lib/util/archive.ts diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 5a76fafd3adac..52b458f692ffa 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -713,7 +713,7 @@ export class CdkToolkit { await readFromStack(options.stackName, this.props.sdkProvider, setEnvironment(options.account, options.region)); const stack = generateStack(template!, options.stackName, language); success(' ⏳ Generating CDK app for %s...', chalk.blue(options.stackName)); - await generateCdkApp(options.stackName, stack!, language, options.outputPath); + await generateCdkApp(options.stackName, stack!, language, options.outputPath, options.compress); } catch (e) { error(' ❌ Migrate failed for `%s`: %s', chalk.blue(options.stackName), (e as Error).message); throw e; @@ -1244,6 +1244,12 @@ export interface MigrateOptions { */ readonly region?: string; + /** + * Whether to zip the generated cdk app folder. + * + * @default false + */ + readonly compress?: boolean; } /** diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index df074eb79894b..85e3bdaaf4996 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -274,11 +274,12 @@ async function parseCommandLineArguments(args: string[]) { .command('migrate', false /* hidden from "cdk --help" */, (yargs: Argv) => yargs .option('stack-name', { type: 'string', alias: 'n', desc: 'The name assigned to the stack created in the new project. The name of the app will be based off this name as well.', requiresArg: true }) .option('language', { type: 'string', default: 'typescript', alias: 'l', desc: 'The language to be used for the new project', choices: MIGRATE_SUPPORTED_LANGUAGES }) - .option('account', { type: 'string', alias: 'a' }) - .option('region', { type: 'string' }) - .option('from-path', { type: 'string', alias: 'p', desc: 'The path to the CloudFormation template to migrate. Use this for locally stored templates' }) - .option('from-stack', { type: 'boolean', alias: 's', desc: 'USe this flag to retrieve the template for an existing CloudFormation stack' }) - .option('output-path', { type: 'string', alias: 'o', desc: 'The output path for the migrated cdk app' }), + .option('account', { type: 'string', desc: 'The account to retrieve the CloudFormation stack template from' }) + .option('region', { type: 'string', desc: 'The region to retrieve the CloudFormation stack template from' }) + .option('from-path', { type: 'string', desc: 'The path to the CloudFormation template to migrate. Use this for locally stored templates' }) + .option('from-stack', { type: 'boolean', desc: 'Use this flag to retrieve the template for an existing CloudFormation stack' }) + .option('output-path', { type: 'string', desc: 'The output path for the migrated CDK app' }) + .option('compress', { type: 'boolean', desc: 'Use this flag to zip the generated CDK app' }), ) .command('context', 'Manage cached context values', (yargs: Argv) => yargs .option('reset', { alias: 'e', desc: 'The context key (or its index) to reset', type: 'string', requiresArg: true }) @@ -659,7 +660,12 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise t.hasName(type!)); if (!template) { - await printAvailableTemplates(language); + await printAvailableTemplates(options.language); throw new Error(`Unknown init template: ${type}`); } - if (!language && template.languages.length === 1) { - language = template.languages[0]; + if (!options.language && template.languages.length === 1) { + const language = template.languages[0]; warning(`No --language was provided, but '${type}' supports only '${language}', so defaulting to --language=${language}`); } - if (!language) { + if (!options.language) { print(`Available languages for ${chalk.green(type)}: ${template.languages.map(l => chalk.blue(l)).join(', ')}`); throw new Error('No language was selected'); } - await initializeProject(template, language, canUseNetwork, generateOnly, workDir, stackName); + await initializeProject(template, options.language, canUseNetwork, generateOnly, workDir, options.stackName); } /** diff --git a/packages/aws-cdk/lib/util/archive.ts b/packages/aws-cdk/lib/util/archive.ts new file mode 100644 index 0000000000000..e06575762bfd0 --- /dev/null +++ b/packages/aws-cdk/lib/util/archive.ts @@ -0,0 +1,90 @@ +import { error } from 'console'; +import { createWriteStream, promises as fs } from 'fs'; +import * as path from 'path'; +import * as glob from 'glob'; + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const archiver = require('archiver'); + +// Adapted from cdk-assets +export async function zipDirectory(directory: string, outputFile: string): Promise { + // We write to a temporary file and rename at the last moment. This is so that if we are + // interrupted during this process, we don't leave a half-finished file in the target location. + const temporaryOutputFile = `${outputFile}.${randomString()}._tmp`; + await writeZipFile(directory, temporaryOutputFile); + await moveIntoPlace(temporaryOutputFile, outputFile); +} + +function writeZipFile(directory: string, outputFile: string): Promise { + return new Promise(async (ok, fail) => { + // The below options are needed to support following symlinks when building zip files: + // - nodir: This will prevent symlinks themselves from being copied into the zip. + // - follow: This will follow symlinks and copy the files within. + const globOptions = { + dot: true, + nodir: true, + follow: true, + cwd: directory, + }; + const files = glob.sync('**', globOptions); // The output here is already sorted + + const output = createWriteStream(outputFile); + + const archive = archiver('zip'); + archive.on('warning', fail); + archive.on('error', fail); + + // archive has been finalized and the output file descriptor has closed, resolve promise + // this has to be done before calling `finalize` since the events may fire immediately after. + // see https://www.npmjs.com/package/archiver + output.once('close', ok); + + archive.pipe(output); + + // Append files serially to ensure file order + for (const file of files) { + const fullPath = path.resolve(directory, file); + const [data, stat] = await Promise.all([fs.readFile(fullPath), fs.stat(fullPath)]); + archive.append(data, { + name: file, + mode: stat.mode, + }); + } + + await archive.finalize(); + }); +} + +/** + * Rename the file to the target location, taking into account: + * + * - That we may see EPERM on Windows while an Antivirus scanner still has the + * file open, so retry a couple of times. + * - This same function may be called in parallel and be interrupted at any point. + */ +async function moveIntoPlace(source: string, target: string) { + let delay = 100; + let attempts = 5; + while (true) { + try { + // 'rename' is guaranteed to overwrite an existing target, as long as it is a file (not a directory) + await fs.rename(source, target); + return; + } catch (e: any) { + if (e.code !== 'EPERM' || attempts-- <= 0) { + throw e; + } + error(e.message); + await sleep(Math.floor(Math.random() * delay)); + delay *= 2; + } + } +} + +function sleep(ms: number) { + return new Promise(ok => setTimeout(ok, ms)); +} + +function randomString() { + return Math.random().toString(36).replace(/[^a-z0-9]+/g, ''); +} diff --git a/packages/aws-cdk/test/commands/migrate.test.ts b/packages/aws-cdk/test/commands/migrate.test.ts index 71e938de5391f..6e5416a5367c1 100644 --- a/packages/aws-cdk/test/commands/migrate.test.ts +++ b/packages/aws-cdk/test/commands/migrate.test.ts @@ -1,9 +1,13 @@ +import { exec as _exec } from 'child_process'; import * as os from 'os'; import * as path from 'path'; +import { promisify } from 'util'; import * as fs from 'fs-extra'; import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, validateSourceOptions } from '../../lib/commands/migrate'; import { MockSdkProvider, MockedObject, SyncHandlerSubsetOf } from '../util/mock-sdk'; +const exec = promisify(_exec); + describe('Migrate Function Tests', () => { let sdkProvider: MockSdkProvider; let getTemplateMock: jest.Mock; @@ -173,6 +177,27 @@ describe('Migrate Function Tests', () => { const replacedStack = fs.readFileSync(path.join(workDir, 'GoodCSharp', 'src', 'GoodCSharp', 'GoodCSharpStack.cs')); expect(replacedStack).toEqual(fs.readFileSync(path.join(...stackPath, 'S3Stack.cs'))); }); + + cliTest('generatedCdkApp generates a zip file when --compress is used', async (workDir) => { + const stack = generateStack(validTemplate, 'GoodTypeScript', 'typescript'); + await generateCdkApp('GoodTypeScript', stack, 'typescript', workDir, true); + + // Packages not in outDir + expect(fs.pathExistsSync(path.join(workDir, 'GoodTypeScript', 'package.json'))).toBeFalsy(); + expect(fs.pathExistsSync(path.join(workDir, 'GoodTypeScript', 'bin', 'good_type_script.ts'))).toBeFalsy(); + expect(fs.pathExistsSync(path.join(workDir, 'GoodTypeScript', 'lib', 'good_type_script-stack.ts'))).toBeFalsy(); + + // Zip file exists + expect(fs.pathExistsSync(path.join(workDir, 'GoodTypeScript.zip'))).toBeTruthy(); + + // Unzip it + await exec(`unzip ${path.join(workDir, 'GoodTypeScript.zip')}`, { cwd: workDir }); + + // Now the files should be there + expect(fs.pathExistsSync(path.join(workDir, 'package.json'))).toBeTruthy(); + expect(fs.pathExistsSync(path.join(workDir, 'bin', 'good_type_script.ts'))).toBeTruthy(); + expect(fs.pathExistsSync(path.join(workDir, 'lib', 'good_type_script-stack.ts'))).toBeTruthy(); + }); }); function cliTest(name: string, handler: (dir: string) => void | Promise): void { diff --git a/packages/aws-cdk/test/init.test.ts b/packages/aws-cdk/test/init.test.ts index 4ecdc53789530..6d9bd2dc4769b 100644 --- a/packages/aws-cdk/test/init.test.ts +++ b/packages/aws-cdk/test/init.test.ts @@ -6,7 +6,11 @@ import { availableInitTemplates, cliInit } from '../lib/init'; describe('constructs version', () => { cliTest('create a TypeScript library project', async (workDir) => { - await cliInit('lib', 'typescript', false, undefined /* canUseNetwork */, workDir); + await cliInit({ + type: 'lib', + language: 'typescript', + workDir, + }); // Check that package.json and lib/ got created in the current directory expect(await fs.pathExists(path.join(workDir, 'package.json'))).toBeTruthy(); @@ -14,7 +18,11 @@ describe('constructs version', () => { }); cliTest('create a TypeScript app project', async (workDir) => { - await cliInit('app', 'typescript', false, undefined /* canUseNetwork */, workDir); + await cliInit({ + type: 'app', + language: 'typescript', + workDir, + }); // Check that package.json and bin/ got created in the current directory expect(await fs.pathExists(path.join(workDir, 'package.json'))).toBeTruthy(); @@ -22,7 +30,11 @@ describe('constructs version', () => { }); cliTest('create a JavaScript app project', async (workDir) => { - await cliInit('app', 'javascript', false, undefined /* canUseNetwork */, workDir); + await cliInit({ + type: 'app', + language: 'javascript', + workDir, + }); // Check that package.json and bin/ got created in the current directory expect(await fs.pathExists(path.join(workDir, 'package.json'))).toBeTruthy(); @@ -31,7 +43,13 @@ describe('constructs version', () => { }); cliTest('create a Java app project', async (workDir) => { - await cliInit('app', 'java', false, true, workDir); + await cliInit({ + type: 'app', + language: 'java', + canUseNetwork: false, + generateOnly: true, + workDir, + }); expect(await fs.pathExists(path.join(workDir, 'pom.xml'))).toBeTruthy(); @@ -47,7 +65,13 @@ describe('constructs version', () => { }); cliTest('create a .NET app project in csharp', async (workDir) => { - await cliInit('app', 'csharp', false, true, workDir); + await cliInit({ + type: 'app', + language: 'csharp', + canUseNetwork: false, + generateOnly: true, + workDir, + }); const csprojFile = (await recursiveListFiles(workDir)).filter(f => f.endsWith('.csproj'))[0]; const slnFile = (await recursiveListFiles(workDir)).filter(f => f.endsWith('.sln'))[0]; @@ -63,7 +87,13 @@ describe('constructs version', () => { }); cliTest('create a .NET app project in fsharp', async (workDir) => { - await cliInit('app', 'fsharp', false, true, workDir); + await cliInit({ + type: 'app', + language: 'fsharp', + canUseNetwork: false, + generateOnly: true, + workDir, + }); const fsprojFile = (await recursiveListFiles(workDir)).filter(f => f.endsWith('.fsproj'))[0]; const slnFile = (await recursiveListFiles(workDir)).filter(f => f.endsWith('.sln'))[0]; @@ -79,7 +109,13 @@ describe('constructs version', () => { }); cliTestWithDirSpaces('csharp app with spaces', async (workDir) => { - await cliInit('app', 'csharp', false, true, workDir); + await cliInit({ + type: 'app', + language: 'csharp', + canUseNetwork: false, + generateOnly: true, + workDir, + }); const csprojFile = (await recursiveListFiles(workDir)).filter(f => f.endsWith('.csproj'))[0]; expect(csprojFile).toBeDefined(); @@ -91,7 +127,13 @@ describe('constructs version', () => { }); cliTestWithDirSpaces('fsharp app with spaces', async (workDir) => { - await cliInit('app', 'fsharp', false, true, workDir); + await cliInit({ + type: 'app', + language: 'fsharp', + canUseNetwork: false, + generateOnly: true, + workDir, + }); const fsprojFile = (await recursiveListFiles(workDir)).filter(f => f.endsWith('.fsproj'))[0]; expect(fsprojFile).toBeDefined(); @@ -103,7 +145,13 @@ describe('constructs version', () => { }); cliTest('create a Python app project', async (workDir) => { - await cliInit('app', 'python', false, true, workDir); + await cliInit({ + type: 'app', + language: 'python', + canUseNetwork: false, + generateOnly: true, + workDir, + }); expect(await fs.pathExists(path.join(workDir, 'requirements.txt'))).toBeTruthy(); const setupPy = (await fs.readFile(path.join(workDir, 'requirements.txt'), 'utf8')).split(/\r?\n/); @@ -119,7 +167,13 @@ describe('constructs version', () => { }); cliTest('--generate-only should skip git init', async (workDir) => { - await cliInit('app', 'javascript', false, true, workDir); + await cliInit({ + type: 'app', + language: 'javascript', + canUseNetwork: false, + generateOnly: true, + workDir, + }); // Check that package.json and bin/ got created in the current directory expect(await fs.pathExists(path.join(workDir, 'package.json'))).toBeTruthy(); @@ -130,7 +184,12 @@ describe('constructs version', () => { cliTest('git directory does not throw off the initer!', async (workDir) => { fs.mkdirSync(path.join(workDir, '.git')); - await cliInit('app', 'typescript', false, undefined /* canUseNetwork */, workDir); + await cliInit({ + type: 'app', + language: 'typescript', + canUseNetwork: false, + workDir, + }); // Check that package.json and bin/ got created in the current directory expect(await fs.pathExists(path.join(workDir, 'package.json'))).toBeTruthy(); @@ -142,10 +201,13 @@ describe('constructs version', () => { for (const templ of await availableInitTemplates()) { for (const lang of templ.languages) { await withTempDir(async tmpDir => { - await cliInit(templ.name, lang, - /* canUseNetwork */ false, - /* generateOnly */ true, - tmpDir); + await cliInit({ + type: templ.name, + language: lang, + canUseNetwork: false, + generateOnly: true, + workDir: tmpDir, + }); // ok if template doesn't have a cdk.json file (e.g. the "lib" template) if (!await fs.pathExists(path.join(tmpDir, 'cdk.json'))) {