-
Notifications
You must be signed in to change notification settings - Fork 4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore(aws-cdk-lib): extract experimental L2s from aws-cdk-lib (#14135)
This PR adds logic to extract the L2 of experimental modules from `aws-cdk-lib`. The reason for keeping the L1s of experimental modules is twofold: 1. Removing the L1 means that modules with an experimental L2 are in a disadvantage in comparison to modules that does not have an L2. 2. `CfnInclude` requires the L1. The main logic was added in `ubergen`. If the monolithic package includes the key `ubergen.stripExperimental` in its `package.json` file, experimental modules source files will not be copied, instead we execute `cfn2ts` to regenerate the L1. #### Testing 1. Added a post-package script -`verify-stripped-exp.js`, to `aws-cdk-lib`. The script will install the tarball and verify that only L1 files are includes in experimental modules. 2. verified that the generated tarball can be used with a consumer application, and that experimental modules are working properly. adding `do-not-merege` as some stable modules still depends on experimental modules, the work to remove these dependencies is in progress. #### Unrelated callout We found a bug in a util function copied from `ubergen`, this PR adds a fix to both code path, but clearly this copy-pasta is not a great mechanism. This kind of copied util scripts is prevalent across our repo and we don't yet have a mechnisam to share sure common runtime functionally between modules. Not sure what is the right solution here but wanted to call it out, in case someone has a fix/methodology I have missed. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
- Loading branch information
Showing
6 changed files
with
241 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,158 @@ | ||
| // +------------------------------------------------------------------------------------------------ | ||
| // | this script is executed post packaging to verify that experimental modules in aws-cdk-lib includes **only** L1 autogenerated files. | ||
| // | The purpose is to avoid publishing L2 of experimental modules with aws-cdk-lib | ||
| // | | ||
| import { spawnSync } from 'child_process'; | ||
| import * as console from 'console'; | ||
| import * as os from 'os'; | ||
| import * as path from 'path'; | ||
| import * as fs from 'fs-extra'; | ||
|
|
||
| async function main(tempDir: string) { | ||
| console.log('🧐 Verifying all experimental modules includes only L1s files...'); | ||
| const cwd = process.cwd(); | ||
| const awsCdkModulesRepoPath = path.join(findWorkspacePath(), 'packages', '@aws-cdk'); | ||
| // eslint-disable-next-line @typescript-eslint/no-require-imports | ||
| const version = require('./../package.json').version; | ||
| const tarFullPath = path.join(cwd, 'dist', 'js', `aws-cdk-lib@${version}.jsii.tgz`); | ||
|
|
||
| const invalidCfnModules = new Map<string, Array<String>>(); | ||
| const invalidModules = new Array<string>(); | ||
|
|
||
| // install the tarball in a temp directory | ||
| console.log(`installing aws-cdk-lib from dist/js into ${tempDir}`); | ||
| exec('npm', ['install', '--prefix', tempDir, tarFullPath]); | ||
| const installedAwsCdkLibPath = path.join(tempDir, 'node_modules', 'aws-cdk-lib', 'lib'); | ||
|
|
||
| for (const module of fs.readdirSync(awsCdkModulesRepoPath)) { | ||
| // eslint-disable-next-line @typescript-eslint/no-require-imports | ||
| const pkgJson = require(path.join(awsCdkModulesRepoPath, module, 'package.json')); | ||
| if (pkgJson.stability !== 'experimental') { | ||
| continue; | ||
| } | ||
| if (pkgJson['cdk-build'].cloudformation) { | ||
| // if a cfn module, verify only the allowed files exists | ||
| const files = await listAllFiles(path.join(installedAwsCdkLibPath, module)); | ||
| const invalidFiles = new Array(); | ||
| files.forEach(file => { | ||
| if (!isAllowedFile(file)) { | ||
| invalidFiles.push(file); | ||
| } | ||
| }); | ||
| if (invalidFiles.length > 0) { | ||
| invalidCfnModules.set(module, invalidFiles); | ||
| } | ||
| } else { | ||
| // not a cfn module, verify it was entirely removed | ||
| if (fs.existsSync(path.join(installedAwsCdkLibPath, module))) { | ||
| invalidModules.push(module); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if (invalidCfnModules.size > 0 || invalidModules.length > 0) { | ||
| if (invalidCfnModules.size > 0 ) { | ||
| console.log('cfn module with invalid files:'); | ||
| for (let [module, files] of invalidCfnModules.entries()) { | ||
| console.log(`${module}:`); | ||
| files.forEach(file => console.log(`\t ${file}`)); | ||
| } | ||
| } | ||
| console.log('---------------------------------------------'); | ||
| if (invalidModules.length > 0) { | ||
| console.log('non-cfn experimental modules:'); | ||
| invalidModules.forEach(m => console.log(`\t ${m}`)); | ||
| } | ||
| throw new Error('Verification Error'); | ||
| } | ||
| } | ||
|
|
||
| const tempDir = fs.mkdtempSync(os.tmpdir()); | ||
|
|
||
| main(tempDir).then( | ||
| () => { | ||
| fs.removeSync(tempDir); | ||
| console.log('✅ All experimental modules includes only L1s files!'); | ||
| process.exit(0); | ||
| }, | ||
| (err) => { | ||
| process.stderr.write(`${err}\n`); | ||
| process.stderr.write(`❌ Verification failed, Some experimental modules includes non L1 files, see details above. Inspect working directory: '${tempDir}'`); | ||
| process.exit(1); | ||
| }, | ||
| ); | ||
|
|
||
|
|
||
| /** | ||
| * Spawn sync with error handling | ||
| */ | ||
| function exec(cmd: string, args: string[]) { | ||
| const proc = spawnSync(cmd, args); | ||
|
|
||
| if (proc.error) { | ||
| throw proc.error; | ||
| } | ||
|
|
||
| if (proc.status !== 0) { | ||
| if (proc.stdout || proc.stderr) { | ||
| throw new Error(`${cmd} exited with status ${proc.status}; stdout: ${proc.stdout?.toString().trim()}\n\n\nstderr: ${proc.stderr?.toString().trim()}`); | ||
| } | ||
| throw new Error(`${cmd} exited with status ${proc.status}`); | ||
| } | ||
|
|
||
| return proc; | ||
| } | ||
|
|
||
| const GENERATED_SUFFIX_REGEX = new RegExp(/generated\.(js|d\.ts)$/); | ||
| const ALLOWED_FILES = ['.jsiirc.json', 'index.ts', 'index.js', 'index.d.ts']; | ||
|
|
||
| /** | ||
| * Recursively collect all files in dir | ||
| */ | ||
| async function listAllFiles(dir: string) { | ||
| const ret = new Array(); | ||
|
|
||
| async function recurse(part: string) { | ||
| const files = await fs.readdir(part); | ||
| for (const file of files) { | ||
| const fullPath = path.join(part, file); | ||
| if ((await fs.stat(fullPath)).isDirectory()) { | ||
| await recurse(fullPath); | ||
| } else { | ||
| ret.push(file); | ||
| } | ||
| } | ||
| } | ||
| await recurse(dir); | ||
| return ret; | ||
| } | ||
|
|
||
| /** | ||
| * Find the workspace root path. Walk up the directory tree until you find lerna.json | ||
| */ | ||
| function findWorkspacePath() { | ||
|
|
||
| return _findRootPath(process.cwd()); | ||
|
|
||
| function _findRootPath(part: string): string { | ||
| if (part === path.resolve(part, '..')) { | ||
| throw new Error('couldn\'t find a \'lerna.json\' file when walking up the directory tree, are you in a aws-cdk project?'); | ||
| } | ||
|
|
||
| if (fs.existsSync(path.resolve(part, 'lerna.json'))) { | ||
| return part; | ||
| } | ||
| return _findRootPath(path.resolve(part, '..')); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * @param file | ||
| * @returns true if the file allowed in an L1 only modules, otherwise false | ||
| */ | ||
| function isAllowedFile(file: string) { | ||
| if (GENERATED_SUFFIX_REGEX.test(file)) { | ||
| return true; | ||
| } | ||
| return ALLOWED_FILES.includes(file); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters