feat(rosetta): find fixtures based on submodules

packages/jsii-rosetta/lib/fixtures.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs-extra';
 import * as path from 'path';
 import { createSourceFile, ScriptKind, ScriptTarget, SyntaxKind } from 'typescript';
 
-import { TypeScriptSnippet, SnippetParameters } from './snippet';
+import { TypeScriptSnippet, SnippetParameters, ApiLocation } from './snippet';
 
 /**
  * Complete snippets with fixtures, if required
@@ -31,10 +31,10 @@ export function fixturize(snippet: TypeScriptSnippet, loose = false): TypeScript
     parameters[SnippetParameters.$COMPILATION_DIRECTORY] = path.join(directory, path.dirname(literateSource));
   } else if (parameters[SnippetParameters.FIXTURE]) {
     // Explicitly requested fixture must exist, unless we are operating in loose mode
-    source = loadAndSubFixture(directory, parameters.fixture, source, !loose);
+    source = loadAndSubFixture(directory, snippet.location.api, parameters.fixture, source, !loose);
   } else if (parameters[SnippetParameters.NO_FIXTURE] === undefined) {
-    // Don't explicitly request no fixture
-    source = loadAndSubFixture(directory, 'default', source, false);
+    // Don't explicitly request no fixture, load the default.
+    source = loadAndSubFixture(directory, snippet.location.api, 'default', source, false);
   }
 
   return {
@@ -54,15 +54,43 @@ function loadLiterateSource(directory: string, literateFileName: string) {
   return fs.readFileSync(fullPath, { encoding: 'utf-8' });
 }
 
-function loadAndSubFixture(directory: string, fixtureName: string, source: string, mustExist: boolean) {
-  const fixtureFileName = path.join(directory, `rosetta/${fixtureName}.ts-fixture`);
-  const exists = fs.existsSync(fixtureFileName);
-  if (!exists && mustExist) {
-    throw new Error(`Sample uses fixture ${fixtureName}, but not found: ${fixtureFileName}`);
-  }
-  if (!exists) {
+/**
+ * Load the fixture with the given name, and substitute the source into it
+ *
+ * If no fixture could be found and `mustExist` is true, and error will be thrown.
+ *
+ * In principle, the fixture we're looking for is `rosetta/FIXTURE.ts-fixture`.
+ * However, we want to support an automatic transform of many small packages
+ * combined into a single large package, perhaps into submodules (i.e., we want
+ * to support monocdk), and in those cases the names of fixtures might conflict.
+ * For example, all of them will have a `default.ts-fixture`, and there won't be
+ * any explicit reference to that file anywhere... yet in the combined
+ * monopackage we have to distinguish those fixtures.
+ *
+ * Therefore, we will consider submodule names as subdirectories, based on the
+ * API location of the snippet we're fixturizing.
+ *
+ * (For example, the fixtures for a type called `monocdk.aws_s3.Bucket` will be
+ * searched both in `rosetta/aws_s3/default.ts-fixture` as well as
+ * `rosetta/default.ts-fixture`).
+ */
+function loadAndSubFixture(
+  directory: string,
+  location: ApiLocation,
+  fixtureName: string,
+  source: string,
+  mustExist: boolean,
+) {
+  const candidates = fixtureCandidates(directory, fixtureName, location);
+  const fixtureFileName = candidates.find((n) => fs.existsSync(n));
+
+  if (!fixtureFileName) {
+    if (mustExist) {
+      throw new Error(`Sample uses fixture ${fixtureName}, but not found: ${JSON.stringify(candidates)}`);
+    }
     return source;
   }
+
   const fixtureContents = fs.readFileSync(fixtureFileName, {
     encoding: 'utf-8',
   });
@@ -99,6 +127,42 @@ function loadAndSubFixture(directory: string, fixtureName: string, source: strin
     : result;
 }
 
+function fixtureCandidates(directory: string, fixtureName: string, location: ApiLocation): string[] {
+  const ret = new Array<string>();
+  const fileName = `${fixtureName}.ts-fixture`;
+  const mods = submodules(location);
+
+  ret.push(path.join(directory, 'rosetta', fileName));
+  for (let i = 0; i < mods.length; i++) {
+    ret.push(path.join(directory, 'rosetta', ...mods.slice(0, i + 1), fileName));
+  }
+
+  // Most specific one up front
+  ret.reverse();
+  return ret;
+}
+
+/**
+ * Return the submodule parts from a given ApiLocation
+ */
+function submodules(location: ApiLocation): string[] {
+  switch (location.api) {
+    case 'file':
+      return [];
+    case 'initializer':
+    case 'member':
+    case 'type':
+    case 'parameter':
+      return middle(location.fqn.split('.'));
+    case 'moduleReadme':
+      return location.moduleFqn.split('.').slice(1);
+  }
+
+  function middle(xs: string[]) {
+    return xs.slice(1, xs.length - 1);
+  }
+}
+
 /**
  * When embedding code fragments in a fixture, "import" statements must be
  * hoisted up to the top of the resulting document, as TypeScript only allows

packages/jsii-rosetta/test/commands/extract.test.ts

@@ -3,7 +3,7 @@ import * as path from 'path';
 import { LanguageTablet } from '../../lib';
 import * as extract from '../../lib/commands/extract';
 import { TARGET_LANGUAGES } from '../../lib/languages';
-import { AssemblyFixture, DUMMY_ASSEMBLY_TARGETS } from '../testutil';
+import { TestJsiiModule, DUMMY_ASSEMBLY_TARGETS } from '../testutil';
 
 const DUMMY_README = `
   Here is an example of how to use ClassA:
@@ -20,10 +20,10 @@ const defaultExtractOptions = {
   validateAssemblies: false,
 };
 
-let assembly: AssemblyFixture;
+let assembly: TestJsiiModule;
 beforeAll(async () => {
   // Create an assembly in a temp directory
-  assembly = await AssemblyFixture.fromSource(
+  assembly = await TestJsiiModule.fromSource(
     {
       'index.ts': `
       export class ClassA {
@@ -43,8 +43,8 @@ beforeAll(async () => {
 afterAll(async () => assembly.cleanup());
 
 test('extract samples from test assembly', async () => {
-  const outputFile = path.join(assembly.directory, 'test.tabl.json');
-  await extract.extractSnippets([assembly.directory], {
+  const outputFile = path.join(assembly.moduleDirectory, 'test.tabl.json');
+  await extract.extractSnippets([assembly.moduleDirectory], {
     outputFile,
     ...defaultExtractOptions,
   });
@@ -58,8 +58,8 @@ test('extract samples from test assembly', async () => {
 describe('with cache file', () => {
   let cacheTabletFile: string;
   beforeAll(async () => {
-    cacheTabletFile = path.join(assembly.directory, 'cache.tabl.json');
-    await extract.extractSnippets([assembly.directory], {
+    cacheTabletFile = path.join(assembly.moduleDirectory, 'cache.tabl.json');
+    await extract.extractSnippets([assembly.moduleDirectory], {
       outputFile: cacheTabletFile,
       ...defaultExtractOptions,
     });
@@ -68,8 +68,8 @@ describe('with cache file', () => {
   test('translation does not happen if it can be read from cache', async () => {
     const translationFunction = jest.fn().mockResolvedValue({ diagnostics: [], translatedSnippets: [] });
 
-    await extract.extractSnippets([assembly.directory], {
-      outputFile: path.join(assembly.directory, 'dummy.tabl.json'),
+    await extract.extractSnippets([assembly.moduleDirectory], {
+      outputFile: path.join(assembly.moduleDirectory, 'dummy.tabl.json'),
       cacheTabletFile,
       translationFunction,
       ...defaultExtractOptions,
@@ -84,8 +84,8 @@ describe('with cache file', () => {
     const oldJavaVersion = TARGET_LANGUAGES.java.version;
     (TARGET_LANGUAGES.java as any).version = '999';
     try {
-      await extract.extractSnippets([assembly.directory], {
-        outputFile: path.join(assembly.directory, 'dummy.tabl.json'),
+      await extract.extractSnippets([assembly.moduleDirectory], {
+        outputFile: path.join(assembly.moduleDirectory, 'dummy.tabl.json'),
         cacheTabletFile,
         translationFunction,
         ...defaultExtractOptions,
@@ -100,7 +100,7 @@ describe('with cache file', () => {
 
 test('do not ignore example strings', async () => {
   // Create an assembly in a temp directory
-  const otherAssembly = await AssemblyFixture.fromSource(
+  const otherAssembly = await TestJsiiModule.fromSource(
     {
       'index.ts': `
       export class ClassA {
@@ -119,8 +119,8 @@ test('do not ignore example strings', async () => {
     },
   );
   try {
-    const outputFile = path.join(otherAssembly.directory, 'test.tabl.json');
-    await extract.extractSnippets([otherAssembly.directory], {
+    const outputFile = path.join(otherAssembly.moduleDirectory, 'test.tabl.json');
+    await extract.extractSnippets([otherAssembly.moduleDirectory], {
       outputFile,
       ...defaultExtractOptions,
     });

packages/jsii-rosetta/test/commands/infuse.test.ts

@@ -5,7 +5,7 @@ import { LanguageTablet } from '../../lib';
 import { extractSnippets } from '../../lib/commands/extract';
 import { infuse, DEFAULT_INFUSION_RESULTS_NAME } from '../../lib/commands/infuse';
 import { loadAssemblies } from '../../lib/jsii/assemblies';
-import { AssemblyFixture, DUMMY_ASSEMBLY_TARGETS } from '../testutil';
+import { TestJsiiModule, DUMMY_ASSEMBLY_TARGETS } from '../testutil';
 
 const DUMMY_README = `
   Here is an example of how to use ClassA:
@@ -19,10 +19,10 @@ const DUMMY_README = `
 
 const TABLET_FILE = 'text.tabl.json';
 
-let assembly: AssemblyFixture;
+let assembly: TestJsiiModule;
 beforeEach(async () => {
   // Create an assembly in a temp directory
-  assembly = await AssemblyFixture.fromSource(
+  assembly = await TestJsiiModule.fromSource(
     {
       'index.ts': `
       export class ClassA {
@@ -46,8 +46,8 @@ beforeEach(async () => {
   );
 
   // Create a tabletFile in the same directory
-  await extractSnippets([assembly.directory], {
-    outputFile: path.join(assembly.directory, TABLET_FILE),
+  await extractSnippets([assembly.moduleDirectory], {
+    outputFile: path.join(assembly.moduleDirectory, TABLET_FILE),
     includeCompilerDiagnostics: false,
     validateAssemblies: false,
   });
@@ -56,19 +56,19 @@ beforeEach(async () => {
 afterEach(async () => assembly.cleanup());
 
 test('examples are added in the assembly', async () => {
-  await infuse([assembly.directory], path.join(assembly.directory, TABLET_FILE));
+  await infuse([assembly.moduleDirectory], path.join(assembly.moduleDirectory, TABLET_FILE));
 
-  const assemblies = await loadAssemblies([assembly.directory], false);
+  const assemblies = await loadAssemblies([assembly.moduleDirectory], false);
   const types = assemblies[0].assembly.types;
   expect(types).toBeDefined();
   expect(types!['my_assembly.ClassA'].docs?.example).toBeDefined();
 });
 
 test('examples are added to the tablet under new keys', async () => {
-  const originalTabletFile = path.join(assembly.directory, TABLET_FILE);
-  const updatedTabletFile = path.join(assembly.directory, 'tablet2.tabl.json');
+  const originalTabletFile = path.join(assembly.moduleDirectory, TABLET_FILE);
+  const updatedTabletFile = path.join(assembly.moduleDirectory, 'tablet2.tabl.json');
 
-  await infuse([assembly.directory], originalTabletFile, {
+  await infuse([assembly.moduleDirectory], originalTabletFile, {
     tabletOutputFile: updatedTabletFile,
   });
 
@@ -79,13 +79,13 @@ test('examples are added to the tablet under new keys', async () => {
 });
 
 test('can log to output file', async () => {
-  await infuse([assembly.directory], path.join(assembly.directory, TABLET_FILE), {
+  await infuse([assembly.moduleDirectory], path.join(assembly.moduleDirectory, TABLET_FILE), {
     log: true,
-    outputFile: path.join(assembly.directory, DEFAULT_INFUSION_RESULTS_NAME),
+    outputFile: path.join(assembly.moduleDirectory, DEFAULT_INFUSION_RESULTS_NAME),
   });
 
   // assert that the output file exists and there is some information in the file.
-  const stats = await fs.stat(path.join(assembly.directory, DEFAULT_INFUSION_RESULTS_NAME));
+  const stats = await fs.stat(path.join(assembly.moduleDirectory, DEFAULT_INFUSION_RESULTS_NAME));
 
   expect(stats.isFile()).toBeTruthy();
   expect(stats.size).toBeGreaterThan(0);

packages/jsii-rosetta/test/jsii/assemblies.test.ts

@@ -1,9 +1,11 @@
 import * as spec from '@jsii/spec';
+import * as fs from 'fs-extra';
 import * as mockfs from 'mock-fs';
 import * as path from 'path';
 
 import { allTypeScriptSnippets } from '../../lib/jsii/assemblies';
 import { SnippetParameters } from '../../lib/snippet';
+import { TestJsiiModule, DUMMY_ASSEMBLY_TARGETS } from '../testutil';
 import { fakeAssembly } from './fake-assembly';
 
 test('Extract snippet from README', () => {
@@ -230,3 +232,40 @@ test('Backwards compatibility with literate integ tests', () => {
     mockfs.restore();
   }
 });
+
+test('rosetta fixture from submodule is preferred if it exists', async () => {
+  const jsiiModule = await TestJsiiModule.fromSource(
+    {
+      'index.ts': 'export * as submodule from "./submodule"',
+      'submodule.ts': `
+        /**
+         * @example new ClassA();
+         */
+        export class ClassA {
+          public someMethod() {
+          }
+        }`,
+    },
+    {
+      name: 'my_assembly',
+      jsii: DUMMY_ASSEMBLY_TARGETS,
+    },
+  );
+  try {
+    await fs.mkdirp(path.join(jsiiModule.moduleDirectory, 'rosetta', 'submodule'));
+    await fs.writeFile(
+      path.join(jsiiModule.moduleDirectory, 'rosetta', 'submodule', 'default.ts-fixture'),
+      'pick me\n/// here',
+    );
+    await fs.writeFile(
+      path.join(jsiiModule.moduleDirectory, 'rosetta', 'default.ts-fixture'),
+      'dont pick me\n/// here',
+    );
+
+    const snippets = allTypeScriptSnippets([{ assembly: jsiiModule.assembly, directory: jsiiModule.moduleDirectory }]);
+
+    expect(snippets[0].completeSource).toMatch(/^pick me/);
+  } finally {
+    await jsiiModule.cleanup();
+  }
+});

packages/jsii-rosetta/test/record-references.test.ts

@@ -1,8 +1,8 @@
-import { AssemblyFixture, DUMMY_ASSEMBLY_TARGETS } from './testutil';
+import { TestJsiiModule, DUMMY_ASSEMBLY_TARGETS } from './testutil';
 
-let assembly: AssemblyFixture;
+let assembly: TestJsiiModule;
 beforeAll(async () => {
-  assembly = await AssemblyFixture.fromSource(
+  assembly = await TestJsiiModule.fromSource(
     `
     export class ClassA {
       public someMethod() {

packages/jsii-rosetta/test/syntax-counter.test.ts

@@ -1,8 +1,8 @@
-import { AssemblyFixture, DUMMY_ASSEMBLY_TARGETS } from './testutil';
+import { TestJsiiModule, DUMMY_ASSEMBLY_TARGETS } from './testutil';
 
-let assembly: AssemblyFixture;
+let assembly: TestJsiiModule;
 beforeAll(async () => {
-  assembly = await AssemblyFixture.fromSource(
+  assembly = await TestJsiiModule.fromSource(
     `
     export class ClassA {
       public someMethod() {