feat(integ-tests): add integ tests for the toolkit library (#430)

Add integration tests for the toolkit library.

Introduces the toolkit library as a new "component under test". 

Because we want to promote writing integ tests against the toolkit
library the same way we would write normal production code, the setup is
like this:

- The toolkit lib is installed as a `devDependency` into the integ test
package. This will allow writing tests as if the toolkit lib is "just" a
dependency.
- At runtime, a version of the toolkit library can be selected and it
will be installed into the dependency closure by the runner before the
tests are run, so that the imports will still function properly.

To signal this, we'll mark `@aws-cdk/toolkit-lib` as an
`optionalDependency` as well.

(PR includes #444)

---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

Signed-off-by: github-actions <github-actions@github.com>

Author: rix0rrr

.github/workflows/integ.yml

@@ -1573,6 +1573,7 @@ const cliInteg = configureProject(
     ],
     devDeps: [
       yarnCling,
+      toolkitLib,
       '@types/semver@^7',
       '@types/yargs@^16',
       '@types/fs-extra@^9',
@@ -1614,6 +1615,8 @@ const cliInteg = configureProject(
 );
 cliInteg.eslint?.addIgnorePattern('resources/**/*.ts');
 
+cliInteg.deps.addDependency('@aws-cdk/toolkit-lib', pj.DependencyType.OPTIONAL);
+
 const compiledDirs = ['tests', 'test', 'lib'];
 for (const compiledDir of compiledDirs) {
   cliInteg.gitignore.addPatterns(`${compiledDir}/**/*.js`);

.projenrc.ts

@@ -45,11 +45,12 @@ inject these dependencies into tests. Users can specify component versions, but
 Test Authors are responsible for taking these parameters and using it to set up
 the right environment for the tests.
 
-| Component             | Command-line argument                | Default     | Treatment by runner        | Treatment in test                         |
-|-----------------------|--------------------------------------|-------------|----------------------------|-------------------------------------------|
-| CDK Construct Library | `--framework-version=VERSION`        | Latest      | Nothing                    | `npm install` into temporary project dir. |
-| CDK CLI               | `--cli-version=VERSION`              | Auto source | `npm install` into tempdir | Add to `$PATH`.                           |
+| Component             | Command-line argument                | Default     | Treatment by runner        | Treatment in test                             |
+|-----------------------|--------------------------------------|-------------|----------------------------|-----------------------------------------------|
+| CDK Construct Library | `--framework-version=VERSION`        | Latest      | Nothing                    | `npm install` into temporary project dir.     |
+| CDK CLI               | `--cli-version=VERSION`              | Auto source | `npm install` into tempdir | Add to `$PATH`.                               |
 |                       | `--cli-source=ROOT` or `auto`        | Auto source |                            | Add `<ROOT>/packages/aws-cdk/bin` to `$PATH`. |
+| Toolkit Library       | `--toolkit-lib-version=VERSION`      | Devdep      | Install into its own deps  | Nothing
 
 ### Running a test suite
 

packages/@aws-cdk-testing/cli-integ/.projen/deps.json

@@ -5,7 +5,9 @@ import * as yargs from 'yargs';
 import { RunnerCliNpmSource } from '../package-sources/cli-npm-source';
 import { RunnerCliRepoSource } from '../package-sources/cli-repo-source';
 import { autoFindRepoRoot } from '../package-sources/find-root';
+import { RunnerLibraryGlobalInstallSource } from '../package-sources/library-globalinstall-source';
 import { RunnerLibraryNpmSource } from '../package-sources/library-npm-source';
+import { RunnerLibraryPreinstalledSource } from '../package-sources/library-preinstalled-source';
 import type { IRunnerSource, ITestCliSource, ITestLibrarySource } from '../package-sources/source';
 import { serializeSources } from '../package-sources/subprocess';
 
@@ -34,6 +36,11 @@ async function main() {
         alias: 'f',
         type: 'string',
       })
+      .options('toolkit-lib-version', {
+        describe: 'Toolkit lib version to use',
+        alias: 'l',
+        type: 'string',
+      })
       .option('use-source', {
         descripton: 'Use TypeScript packages from the given source repository (or "auto")',
         type: 'string',
@@ -117,11 +124,23 @@ async function main() {
   const librarySource: IRunnerSource<ITestLibrarySource>
     = new RunnerLibraryNpmSource('aws-cdk-lib', args['framework-version'] ? args['framework-version'] : 'latest');
 
+  const toolkitLibPackage = '@aws-cdk/toolkit-lib';
+  let toolkitSource: IRunnerSource<ITestLibrarySource> | undefined;
+  if (args['toolkit-lib-version']) {
+    toolkitSource = new RunnerLibraryGlobalInstallSource(toolkitLibPackage, args['toolkit-lib-version']);
+  }
+  if (!toolkitSource) {
+    toolkitSource = await RunnerLibraryPreinstalledSource.isPreinstalled(toolkitLibPackage)
+      ? new RunnerLibraryPreinstalledSource(toolkitLibPackage)
+      : new RunnerLibraryGlobalInstallSource(toolkitLibPackage, 'latest');
+  }
+
   console.log('------> Configuration');
-  console.log(`        Test suite:     ${suiteName}`);
-  console.log(`        Test version:   ${thisPackageVersion()}`);
-  console.log(`        CLI source:     ${cliSource.assert().sourceDescription}`);
-  console.log(`        Library source: ${librarySource.sourceDescription}`);
+  console.log(`        Test suite:         ${suiteName}`);
+  console.log(`        Test version:       ${thisPackageVersion()}`);
+  console.log(`        CLI source:         ${cliSource.assert().sourceDescription}`);
+  console.log(`        Library source:     ${librarySource.sourceDescription}`);
+  console.log(`        Toolkit lib source: ${toolkitSource.sourceDescription}`);
 
   if (args.verbose) {
     process.env.VERBOSE = '1';
@@ -142,17 +161,24 @@ async function main() {
     console.log('------> Resolved versions');
     const cli = await cliSource.assert().runnerPrepare();
     disposables.push(cli);
-    console.log(`        CLI:     ${cli.version}`);
+    console.log(`        CLI:             ${cli.version}`);
 
     const library = await librarySource.runnerPrepare();
     disposables.push(library);
-    console.log(`        Library: ${library.version}`);
+    console.log(`        Library:         ${library.version}`);
+
+    const toolkitLib = await toolkitSource.runnerPrepare();
+    disposables.push(toolkitLib);
+    console.log(`        Toolkit library: ${toolkitLib.version}`);
 
     serializeSources({
       cli,
       library,
+      toolkitLib,
     });
 
+    const jestConfig = path.resolve(__dirname, '..', '..', 'resources', 'integ.jest.config.js');
+
     await jest.run([
       '--randomize',
       ...args.runInBand ? ['-i'] : [],
@@ -161,7 +187,7 @@ async function main() {
       ...args.maxWorkers ? [`--maxWorkers=${args.maxWorkers}`] : [],
       ...passWithNoTests ? ['--passWithNoTests'] : [],
       ...args['test-file'] ? [args['test-file']] : [],
-    ], path.resolve(__dirname, '..', '..', 'resources', 'integ.jest.config.js'));
+    ], jestConfig);
   } finally {
     for (const disp of disposables) {
       await disp.dispose();

packages/@aws-cdk-testing/cli-integ/.projen/tasks.json

@@ -24,6 +24,15 @@ export async function npmMostRecentMatching(packageName: string, range: string)
   return output[output.length - 1];
 }
 
+export async function npmQueryInstalledVersion(packageName: string, dir: string) {
+  const reportStr = await shell(['node', require.resolve('npm'), 'list', '--json', '--depth', '0', packageName], {
+    cwd: dir,
+    show: 'error',
+  });
+  const report = JSON.parse(reportStr);
+  return report.dependencies[packageName].version;
+}
+
 /**
  * Use NPM preinstalled on the machine to look up a list of TypeScript versions
  */

packages/@aws-cdk-testing/cli-integ/README.md

@@ -2,6 +2,7 @@ import * as os from 'os';
 import * as path from 'path';
 import * as fs from 'fs-extra';
 import type { IRunnerSource, ITestCliSource, IPreparedRunnerSource } from './source';
+import { npmQueryInstalledVersion } from '../npm';
 import { addToShellPath, rimraf, shell } from '../shell';
 
 export class RunnerCliNpmSource implements IRunnerSource<ITestCliSource> {
@@ -18,13 +19,7 @@ export class RunnerCliNpmSource implements IRunnerSource<ITestCliSource> {
     await shell(['node', require.resolve('npm'), 'install', `aws-cdk@${this.range}`], {
       cwd: tempDir,
     });
-
-    const reportStr = await shell(['node', require.resolve('npm'), 'list', '--json', '--depth', '0', 'aws-cdk'], {
-      cwd: tempDir,
-      show: 'error',
-    });
-    const report = JSON.parse(reportStr);
-    const installedVersion = report.dependencies['aws-cdk'].version;
+    const installedVersion = await npmQueryInstalledVersion('aws-cdk', tempDir);
 
     return {
       version: installedVersion,

packages/@aws-cdk-testing/cli-integ/lib/cli/run-suite.ts

@@ -10,23 +10,25 @@ import { addToShellPath } from '../shell';
  */
 export class RunnerCliRepoSource implements IRunnerSource<ITestCliSource> {
   public readonly sourceDescription: string;
-  private readonly cliPath: string;
+  private readonly cliBinPath: string;
 
   constructor(private readonly repoRoot: string) {
-    this.cliPath = path.join(this.repoRoot, 'packages', 'aws-cdk', 'bin');
-    this.sourceDescription = this.cliPath;
+    this.cliBinPath = path.join(this.repoRoot, 'packages', 'aws-cdk', 'bin');
+    this.sourceDescription = this.cliBinPath;
   }
 
   public async runnerPrepare(): Promise<IPreparedRunnerSource<ITestCliSource>> {
     if (!await fs.pathExists(path.join(this.repoRoot, 'package.json')) || !await fs.pathExists(path.join(this.repoRoot, 'yarn.lock'))) {
       throw new Error(`${this.repoRoot}: does not look like the repository root`);
     }
 
+    const pj = JSON.parse(await fs.readFile(path.join(this.cliBinPath, '..', 'package.json'), 'utf-8'));
+
     return {
-      version: '*',
+      version: pj.version,
       dispose: () => Promise.resolve(),
       serialize: () => {
-        return [TestCliRepoSource, [this.cliPath]];
+        return [TestCliRepoSource, [this.cliBinPath]];
       },
     };
   }

packages/@aws-cdk-testing/cli-integ/lib/npm.ts

@@ -0,0 +1,70 @@
+import { promises as fs } from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import type { IRunnerSource, IPreparedRunnerSource, ITestLibrarySource } from './source';
+import { copyDirectoryContents } from '../files';
+import { npmQueryInstalledVersion } from '../npm';
+import { shell } from '../shell';
+
+/**
+ * A library dependency that cli-integ installs into its own `node_modules`.
+ */
+export class RunnerLibraryGlobalInstallSource implements IRunnerSource<ITestLibrarySource> {
+  public readonly sourceDescription: string;
+
+  constructor(private readonly packageName: string, private readonly range: string) {
+    this.sourceDescription = `${this.packageName}@${this.range}`;
+  }
+
+  public async runnerPrepare(): Promise<IPreparedRunnerSource<ITestLibrarySource>> {
+    // Create a tempdir where we install the requested package, then symlink into our `node_modules`
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'tmpcdk'));
+    await fs.mkdir(tempDir, { recursive: true });
+
+    await shell(['node', require.resolve('npm'), 'install', `${this.packageName}@${this.range}`], {
+      cwd: tempDir,
+    });
+
+    const symlinkPath = path.join(__dirname, '..', '..', 'node_modules', this.packageName);
+    await fs.mkdir(path.dirname(symlinkPath), { recursive: true });
+    await fs.symlink(path.join(tempDir, 'node_modules', this.packageName), symlinkPath, 'junction');
+
+    const version = await npmQueryInstalledVersion(this.packageName, tempDir);
+
+    return {
+      version,
+      async dispose() {
+        // Remove the symlink again
+        await fs.rm(symlinkPath);
+      },
+      serialize: () => {
+        return [TestLibraryGlobalInstallSource, [this.packageName, version]];
+      },
+    };
+  }
+}
+
+export class TestLibraryGlobalInstallSource implements ITestLibrarySource {
+  constructor(public readonly packageName: string, private readonly version: string) {
+  }
+
+  public requestedVersion(): string {
+    return this.version;
+  }
+
+  public assertJsiiPackagesAvailable(): void {
+    // FIXME: Always a no-op.
+  }
+
+  public async initializeDotnetPackages(currentDir: string): Promise<void> {
+    // FIXME: this code has nothing to do with the package source, really, so shouldn't be here.
+    if (process.env.CWD_FILES_DIR) {
+      await copyDirectoryContents(process.env.CWD_FILES_DIR, currentDir);
+    }
+  }
+
+  public requestedAlphaVersion(): string {
+    return this.version;
+  }
+}
+