chore(cli): prevent test interference (#32270)

Our CLI unit tests were interfering with each other because they were writing files from and to the current directory, which is shared between all of them.

Solve it by making a non-writeable directory before running the tests, so that the tests that do that start throwing errors and we can identify them. Then fix those.

I tried papering over the issue by trying to create tempdirs automatically, but that started to introduce all kinds of errors in tests that were already doing tempdir management themselves, and it took too long to go and figure out what was wrong there, so I'm doing this instead.

Also in this PR:

- Switch off global `silent` mode for the tests. It's very annoying if `console.log` statements don't make it out when you're debugging.
- This PR caused a couple of lines in `init.ts` to be useless (they were there for tests), and removing those causes a lack of coverage in `init.ts`'s entirety, so add some tests for that file.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: rix0rrr

packages/aws-cdk/jest.config.js

@@ -20,6 +20,5 @@ module.exports = {
 
     // We have many tests here that commonly time out
     testTimeout: 30_000,
-    // These tests are too chatty. Shush.
-    silent: true,
+    setupFilesAfterEnv: ["<rootDir>/test/jest-setup-after-env.ts"],
 };

packages/aws-cdk/lib/init.ts

@@ -324,13 +324,9 @@ async function initializeProject(
   if (migrate) {
     await template.addMigrateContext(workDir);
   }
-  if (await fs.pathExists('README.md')) {
-    const readme = await fs.readFile('README.md', { encoding: 'utf-8' });
-    // Save the logs!
-    // Without this statement, the readme of the CLI is printed in every init test
-    if (!readme.startsWith('# AWS CDK Toolkit')) {
-      print(chalk.green(readme));
-    }
+  if (await fs.pathExists(`${workDir}/README.md`)) {
+    const readme = await fs.readFile(`${workDir}/README.md`, { encoding: 'utf-8' });
+    print(chalk.green(readme));
   }
 
   if (!generateOnly) {

packages/aws-cdk/test/diff.test.ts

@@ -1,4 +1,6 @@
 import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
 import { Writable } from 'stream';
 import { StringDecoder } from 'string_decoder';
 import * as cxschema from '@aws-cdk/cloud-assembly-schema';
@@ -12,6 +14,22 @@ import { CdkToolkit } from '../lib/cdk-toolkit';
 let cloudExecutable: MockCloudExecutable;
 let cloudFormation: jest.Mocked<Deployments>;
 let toolkit: CdkToolkit;
+let oldDir: string;
+let tmpDir: string;
+
+beforeAll(() => {
+  // The toolkit writes and checks for temporary files in the current directory,
+  // so run these tests in a tempdir so they don't interfere with each other
+  // and other tests.
+  oldDir = process.cwd();
+  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'aws-cdk-test'));
+  process.chdir(tmpDir);
+});
+
+afterAll(() => {
+  process.chdir(oldDir);
+  fs.rmSync(tmpDir, { recursive: true, force: true });
+});
 
 describe('fixed template', () => {
   const templatePath = 'oldTemplate.json';

packages/aws-cdk/test/init.test.ts

@@ -2,7 +2,7 @@ import * as os from 'os';
 import * as path from 'path';
 import * as cxapi from '@aws-cdk/cx-api';
 import * as fs from 'fs-extra';
-import { availableInitTemplates, cliInit } from '../lib/init';
+import { availableInitLanguages, availableInitTemplates, cliInit, printAvailableTemplates } from '../lib/init';
 
 describe('constructs version', () => {
   cliTest('create a TypeScript library project', async (workDir) => {
@@ -17,6 +17,21 @@ describe('constructs version', () => {
     expect(await fs.pathExists(path.join(workDir, 'lib'))).toBeTruthy();
   });
 
+  cliTest('asking for a nonexistent template fails', async (workDir) => {
+    await expect(cliInit({
+      type: 'banana',
+      language: 'typescript',
+      workDir,
+    })).rejects.toThrow(/Unknown init template/);
+  });
+
+  cliTest('asking for a template but no language prints and throws', async (workDir) => {
+    await expect(cliInit({
+      type: 'app',
+      workDir,
+    })).rejects.toThrow(/No language/);
+  });
+
   cliTest('create a TypeScript app project', async (workDir) => {
     await cliInit({
       type: 'app',
@@ -237,6 +252,16 @@ test('when no version number is present (e.g., local development), the v2 templa
   expect((await availableInitTemplates()).length).toBeGreaterThan(0);
 });
 
+test('check available init languages', async () => {
+  const langs = await availableInitLanguages();
+  expect(langs.length).toBeGreaterThan(0);
+  expect(langs).toContain('typescript');
+});
+
+test('exercise printing available templates', async () => {
+  await printAvailableTemplates();
+});
+
 function cliTest(name: string, handler: (dir: string) => void | Promise<any>): void {
   test(name, () => withTempDir(handler));
 }

packages/aws-cdk/test/jest-setup-after-env.ts

@@ -0,0 +1,85 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { isPromise } from 'util/types';
+
+/**
+ * Global test setup for Jest tests
+ *
+ * It's easy to accidentally write tests that interfere with each other by
+ * writing files to disk in the "current directory". To prevent this, the global
+ * test setup creates a directory in the temporary directory and chmods it to
+ * being non-writable. That way, whenever a test tries to write to the current
+ * directory, it will produce an error and we'll be able to find and fix the
+ * test.
+ *
+ * If you see `EACCES: permission denied`, you have a test that creates files
+ * in the current directory, and you should be sure to do it in a temporary
+ * directory that you clean up afterwards.
+ *
+ * ## Alternate approach
+ *
+ * I tried an approach where I would automatically try to create and clean up
+ * temp directories for every test, but it was introducing too many conflicts
+ * with existing test behavior (around specific ordering of temp directory
+ * creation and cleanup tasks that are already present) in many places that I
+ * didn't want to go and chase down.
+ *
+ */
+
+let tmpDir: string;
+let oldDir: string;
+
+beforeAll(() => {
+  tmpDir = path.join(os.tmpdir(), 'cdk-nonwritable-on-purpose');
+  fs.mkdirSync(tmpDir, { recursive: true });
+  fs.chmodSync(tmpDir, 0o500);
+  oldDir = process.cwd();
+  process.chdir(tmpDir);
+  tmpDir = process.cwd(); // This will have resolved symlinks
+});
+
+const reverseAfterAll: Array<jest.ProvidesHookCallback> = [];
+
+/**
+ * We need a cleanup here
+ *
+ * 99% of the time, Jest runs the tests in a subprocess and this isn't
+ * necessary because we would have `chdir`ed in the subprocess.
+ *
+ * But sometimes we ask Jest with `-i` to run the tests in the main process,
+ * or if you only ask for a single test suite Jest runs the tests in the main
+ * process, and then we `chdir`ed the main process away.
+ *
+ * Jest will then try to write the `coverage` directory to the readonly directory,
+ * and fail. Chdir back to the original dir.
+ *
+ * If the test file has an `afterAll()` hook it installed as well, we need to run
+ * it before our cleanup, otherwise the wrong thing will happen (by default,
+ * all `afterAll()`s run in call order, but they should be run in reverse).
+ */
+afterAll(async () => {
+  for (const aft of reverseAfterAll.reverse()) {
+    await new Promise<void>((resolve, reject) => {
+      const response = aft(resolve as any);
+      if (isPromise(response)) {
+        response.then(() => { return resolve(); }, reject);
+      } else {
+        resolve();
+      }
+    });
+  }
+
+  // eslint-disable-next-line no-console
+  process.stderr.write(`${process.cwd()}, ${tmpDir}\n`);
+  if (process.cwd() === tmpDir) {
+    // eslint-disable-next-line no-console
+    process.stderr.write('chmod\n');
+    process.chdir(oldDir);
+  }
+});
+
+// Patch afterAll to make later-provided afterAll's run before us (in reverse order even).
+afterAll = (after: jest.ProvidesHookCallback) => {
+  reverseAfterAll.push(after);
+};

packages/aws-cdk/test/notices.test.ts

@@ -455,23 +455,20 @@ describe(CachedDataSource, () => {
   });
 
   test('retrieves data from the delegate when the file cannot be read', async () => {
-    const debugSpy = jest.spyOn(logging, 'debug');
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cdk-test'));
+    try {
+      const debugSpy = jest.spyOn(logging, 'debug');
 
-    if (fs.existsSync('does-not-exist.json')) {
-      fs.unlinkSync('does-not-exist.json');
-    }
-
-    const dataSource = dataSourceWithDelegateReturning(freshData, 'does-not-exist.json');
+      const dataSource = dataSourceWithDelegateReturning(freshData, `${tmpDir}/does-not-exist.json`);
 
-    const notices = await dataSource.fetch();
-
-    expect(notices).toEqual(freshData);
-    expect(debugSpy).not.toHaveBeenCalled();
+      const notices = await dataSource.fetch();
 
-    debugSpy.mockRestore();
+      expect(notices).toEqual(freshData);
+      expect(debugSpy).not.toHaveBeenCalled();
 
-    if (fs.existsSync('does-not-exist.json')) {
-      fs.unlinkSync('does-not-exist.json');
+      debugSpy.mockRestore();
+    } finally {
+      fs.rmSync(tmpDir, { recursive: true, force: true });
     }
   });