feat(docs): enrich type information for docs-json Output Target (#4212)

This enriches the type information supplied in the `docs-json` output
target in order to make it easier to deliver a rich documentation
experience for a Stencil project.

This involves three significant changes.

Firstly, the information which the `docs-json` OT gathers about the
properties which comprise the API of a Stencil component is expanded to
include more information about the types used in the declaration of the
property. In particular, this means that if you have, say, a `@Method`
on a component whose signature involves several types imported from
other modules the JSON blob for that method will now include more
detailed information about all the types used in that declaration.

This information is stored under the `complexType` key for each field on
a component. This `complexType` object includes a `references` object
which now includes a new, globally-unique ID for each type in a Stencil
project which looks like:

```ts
const typeID = `${pathToTypeHomeModule}::${typeName}`;
```

where `pathToTypeHomeModule` is the path to the type's "home" module,
i.e. where it was originally declared, and `typeName` is the _original_
name it is declared under in that module. This if you had a type like
the following:

```ts src/shared-interfaces.ts
export interface SharedInterface {
  property: "value";
}
```

the `SharedInterface` type would have the following ID:

```ts
"src/shared-interfaces.ts::SharedInterface"
```

This unique identifier allows cross-references to be made between the
_usage sites_ of a type (as documented in the JSON blob for, say, a
`@Method` or `@Prop`) and canonical information about that type.

The second major addition is the creation of a new canonical reference
point for all exported types used within a Stencil project. This
reference is called the _type library_ and it is included in the
`docs-json` output target under the `"typeLibrary"` key. It's an object
which maps the unique type IDs described above to an object containing
information about those types, including their original declaration
rendered as a string, the path to their home module, and documentation,
if present

> Note that global types, types imported from `node_modules`, and
  types marked as private with the `@private` JSDoc tag are not included
  in the type library, even if they are used in the declaration of a
  property decorated with a Stencil decorator.

The combination of globally-unique type IDs and the type library will
allow Stencil component authors to create richer documentation
experiences where the usage sites of a widely imported type can easily
be linked to a canonical reference point for that type.

The third addition is a new configuration option for the `docs-json`
output target called `supplementalPublicTypes`. This option takes a
filepath which points to a file exporting types and interfaces which
should be included in the type library regardless of whether they are
used in the API of any component included in the Stencil project.

The motivation for `supplementalPublicTypes` is to facilitate the
documentation of types and interfaces which are important for a given
project but which may not be included in the type library because
they're not used by a `@Prop()`, `@Event()`, or another Stencil
decorator.

.prettierignore

@@ -45,6 +45,8 @@ test/**/components.d.ts
 test/end-to-end/screenshot/
 test/end-to-end/docs.d.ts
 test/end-to-end/docs.json
+test/docs-json/docs.d.ts
+test/docs-json/docs.json
 
 # minified angular that exists in the test directory
 test/karma/test-app/assets/angular.min.js

BREAKING_CHANGES.md

@@ -17,6 +17,7 @@ This is a comprehensive list of the breaking changes introduced in the major ver
   - [Legacy Browser Support Removed](#legacy-browser-support-removed)
   - [Legacy Cache Stats Config Flag Removed](#legacy-cache-stats-config-flag-removed)
   - [Drop Node 14 Support](#drop-node-14-support)
+  - [Information included in JSON documentation expanded](#information-included-in-docs-json-expanded)
 
 ### General
 
@@ -81,6 +82,86 @@ Stencil no longer supports Node 14.
 Please upgrade local development machines, continuous integration pipelines, etc. to use Node v16 or higher.
 For the full list of supported runtimes, please see [our Support Policy](../reference/support-policy.md#javascript-runtime).
 
+#### Information included in `docs-json` expanded
+
+For Stencil v4 the information included in the output of the `docs-json` output
+target was expanded to include more information about the types of properties
+and methods on Stencil components.
+
+For more context on this change, see the [documentation for the new
+`supplementalPublicTypes`](https://stenciljs.com/docs/docs-json#supplementalpublictypes)
+option for the JSON documentation output target.
+
+##### `JsonDocsEvent`
+
+The JSON-formatted documentation for an `@Event` now includes a field called
+`complexType` which includes more information about the types referenced in the
+type declarations for that property.
+
+Here's an example of what this looks like for the [ionBreakpointDidChange
+event](https://github.com/ionic-team/ionic-framework/blob/1f0c8049a339e3a77c468ddba243041d08ead0be/core/src/components/modal/modal.tsx#L289-L292)
+on the `Modal` component in Ionic Framework:
+
+```json
+{
+  "complexType": {
+    "original": "ModalBreakpointChangeEventDetail",
+    "resolved": "ModalBreakpointChangeEventDetail",
+    "references": {
+      "ModalBreakpointChangeEventDetail": {
+        "location": "import",
+        "path": "./modal-interface",
+        "id": "src/components/modal/modal.tsx::ModalBreakpointChangeEventDetail"
+      }
+    }
+  }
+}
+```
+
+##### `JsonDocsMethod`
+
+The JSON-formatted documentation for a `@Method` now includes a field called
+`complexType` which includes more information about the types referenced in 
+the type declarations for that property.
+
+Here's an example of what this looks like for the [open
+method](https://github.com/ionic-team/ionic-framework/blob/1f0c8049a339e3a77c468ddba243041d08ead0be/core/src/components/select/select.tsx#L261-L313)
+on the `Select` component in Ionic Framework:
+
+```json
+{
+  "complexType": {
+    "signature": "(event?: UIEvent) => Promise<any>",
+    "parameters": [
+      {
+        "tags": [
+          {
+            "name": "param",
+            "text": "event The user interface event that called the open."
+          }
+        ],
+        "text": "The user interface event that called the open."
+      }
+    ],
+    "references": {
+      "Promise": {
+        "location": "global",
+        "id": "global::Promise"
+      },
+      "UIEvent": {
+        "location": "global",
+        "id": "global::UIEvent"
+      },
+      "HTMLElement": {
+        "location": "global",
+        "id": "global::HTMLElement"
+      }
+    },
+    "return": "Promise<any>"
+  }
+}
+```
+
 ## Stencil v3.0.0
 
 * [General](#general)
@@ -1093,4 +1174,4 @@ When running Jest directly, previously most of Jest had to be manually configure
 -      "jsx"
 -    ]
   }
-```
+```

src/compiler/docs/generate-doc-data.ts

@@ -1,14 +1,17 @@
-import { flatOne, normalizePath, sortBy, unique } from '@utils';
+import { flatOne, isOutputTargetDocsJson, normalizePath, sortBy, unique } from '@utils';
 import { basename, dirname, join, relative } from 'path';
 
 import type * as d from '../../declarations';
 import { JsonDocsValue } from '../../declarations';
 import { typescriptVersion, version } from '../../version';
 import { getBuildTimestamp } from '../build/build-ctx';
+import { addFileToLibrary, getTypeLibrary } from '../transformers/type-library';
 import { AUTO_GENERATE_COMMENT } from './constants';
 
 /**
- * Generate metadata that will be used to generate any given documentation-related output target(s)
+ * Generate metadata that will be used to generate any given documentation-related
+ * output target(s)
+ *
  * @param config the configuration associated with the current Stencil task run
  * @param compilerCtx the current compiler context
  * @param buildCtx the build context for the current Stencil task run
@@ -19,6 +22,18 @@ export const generateDocData = async (
   compilerCtx: d.CompilerCtx,
   buildCtx: d.BuildCtx
 ): Promise<d.JsonDocs> => {
+  const jsonOutputTargets = config.outputTargets.filter(isOutputTargetDocsJson);
+  const supplementalPublicTypes = findSupplementalPublicTypes(jsonOutputTargets);
+
+  if (supplementalPublicTypes !== '') {
+    // if supplementalPublicTypes is set then we want to add all the public
+    // types in that file to the type library so that output targets producing
+    // documentation can make use of that data later.
+    addFileToLibrary(config, supplementalPublicTypes);
+  }
+
+  const typeLibrary = getTypeLibrary();
+
   return {
     timestamp: getBuildTimestamp(),
     compiler: {
@@ -27,11 +42,28 @@ export const generateDocData = async (
       typescriptVersion,
     },
     components: await getDocsComponents(config, compilerCtx, buildCtx),
+    typeLibrary,
   };
 };
 
+/**
+ * If the `supplementalPublicTypes` option is set on one output target, find that value and return it.
+ *
+ * @param outputTargets an array of docs-json output targets
+ * @returns the first value encountered for supplementalPublicTypes or an empty string
+ */
+function findSupplementalPublicTypes(outputTargets: d.OutputTargetDocsJson[]): string {
+  for (const docsJsonOT of outputTargets) {
+    if (docsJsonOT.supplementalPublicTypes) {
+      return docsJsonOT.supplementalPublicTypes;
+    }
+  }
+  return '';
+}
+
 /**
  * Derive the metadata for each Stencil component
+ *
  * @param config the configuration associated with the current Stencil task run
  * @param compilerCtx the current compiler context
  * @param buildCtx the build context for the current Stencil task run
@@ -50,11 +82,12 @@ const getDocsComponents = async (
       const usagesDir = normalizePath(join(dirPath, 'usage'));
       const readme = await getUserReadmeContent(compilerCtx, readmePath);
       const usage = await generateUsages(compilerCtx, usagesDir);
+
       return moduleFile.cmps
         .filter((cmp: d.ComponentCompilerMeta) => !cmp.internal && !cmp.isCollectionDependency)
         .map((cmp: d.ComponentCompilerMeta) => ({
           dirPath,
-          filePath: relative(config.rootDir, filePath),
+          filePath: normalizePath(relative(config.rootDir, filePath), false),
           fileName: basename(filePath),
           readmePath,
           usagesDir,
@@ -150,6 +183,7 @@ const getRealProperties = (properties: d.ComponentCompilerProperty[]): d.JsonDoc
     .map((member) => ({
       name: member.name,
       type: member.complexType.resolved,
+      complexType: member.complexType,
       mutable: member.mutable,
       attr: member.attribute,
       reflectToAttr: !!member.reflect,
@@ -243,6 +277,7 @@ const getDocsMethods = (methods: d.ComponentCompilerMethod[]): d.JsonDocsMethod[
           .map((t) => t.text)
           .join('\n'),
       },
+      complexType: member.complexType,
       signature: `${member.name}${member.complexType.signature}`,
       parameters: [], // TODO
       docs: member.docs.text,
@@ -258,6 +293,7 @@ const getDocsEvents = (events: d.ComponentCompilerEvent[]): d.JsonDocsEvent[] =>
       event: eventMeta.name,
       detail: eventMeta.complexType.resolved,
       bubbles: eventMeta.bubbles,
+      complexType: eventMeta.complexType,
       cancelable: eventMeta.cancelable,
       composed: eventMeta.composed,
       docs: eventMeta.docs.text,
@@ -356,7 +392,7 @@ const getUserReadmeContent = async (compilerCtx: d.CompilerCtx, readmePath: stri
  * @param jsdoc the JSDoc associated with the component's declaration
  * @returns the generated documentation
  */
-const generateDocs = (readme: string, jsdoc: d.CompilerJsDoc): string => {
+const generateDocs = (readme: string | undefined, jsdoc: d.CompilerJsDoc): string => {
   const docs = jsdoc.text;
   if (docs !== '' || !readme) {
     // just return the existing docs if they exist. these would have been captured earlier in the compilation process.

src/compiler/sys/in-memory-fs.ts

@@ -221,7 +221,7 @@ export const createInMemoryFs = (sys: d.CompilerSystem) => {
   const emptyDirs = async (dirs: string[]): Promise<void> => {
     dirs = dirs
       .filter(isString)
-      .map(normalizePath)
+      .map((s) => normalizePath(s))
       .reduce((dirs, dir) => {
         if (!dirs.includes(dir)) {
           dirs.push(dir);

src/compiler/sys/logger/test/terminal-logger.spec.ts

@@ -40,7 +40,7 @@ describe('terminal-logger', () => {
   });
 
   describe('basic logging functionality', () => {
-    const setupConsoleMocks = setupConsoleMocker();
+    const { setupConsoleMocks } = setupConsoleMocker();
 
     function setup() {
       const { logMock, warnMock, errorMock } = setupConsoleMocks();

src/compiler/transformers/decorators-to-static/convert-decorators.ts

@@ -27,12 +27,13 @@ import { watchDecoratorsToStatic } from './watch-decorator';
 export const convertDecoratorsToStatic = (
   config: d.Config,
   diagnostics: d.Diagnostic[],
-  typeChecker: ts.TypeChecker
+  typeChecker: ts.TypeChecker,
+  program: ts.Program
 ): ts.TransformerFactory<ts.SourceFile> => {
   return (transformCtx) => {
     const visit = (node: ts.Node): ts.VisitResult<ts.Node> => {
       if (ts.isClassDeclaration(node)) {
-        return visitClassDeclaration(config, diagnostics, typeChecker, node);
+        return visitClassDeclaration(config, diagnostics, typeChecker, program, node);
       }
       return ts.visitEachChild(node, visit, transformCtx);
     };
@@ -47,6 +48,7 @@ const visitClassDeclaration = (
   config: d.Config,
   diagnostics: d.Diagnostic[],
   typeChecker: ts.TypeChecker,
+  program: ts.Program,
   classNode: ts.ClassDeclaration
 ) => {
   const componentDecorator = retrieveTsDecorators(classNode)?.find(isDecoratorNamed('Component'));
@@ -69,10 +71,18 @@ const visitClassDeclaration = (
   const watchable = new Set<string>();
   // parse member decorators (Prop, State, Listen, Event, Method, Element and Watch)
   if (decoratedMembers.length > 0) {
-    propDecoratorsToStatic(diagnostics, decoratedMembers, typeChecker, watchable, filteredMethodsAndFields);
+    propDecoratorsToStatic(diagnostics, decoratedMembers, typeChecker, program, watchable, filteredMethodsAndFields);
     stateDecoratorsToStatic(decoratedMembers, watchable, filteredMethodsAndFields);
-    eventDecoratorsToStatic(diagnostics, decoratedMembers, typeChecker, filteredMethodsAndFields);
-    methodDecoratorsToStatic(config, diagnostics, classNode, decoratedMembers, typeChecker, filteredMethodsAndFields);
+    eventDecoratorsToStatic(diagnostics, decoratedMembers, typeChecker, program, filteredMethodsAndFields);
+    methodDecoratorsToStatic(
+      config,
+      diagnostics,
+      classNode,
+      decoratedMembers,
+      typeChecker,
+      program,
+      filteredMethodsAndFields
+    );
     elementDecoratorsToStatic(diagnostics, decoratedMembers, typeChecker, filteredMethodsAndFields);
     watchDecoratorsToStatic(config, diagnostics, decoratedMembers, watchable, filteredMethodsAndFields);
     listenDecoratorsToStatic(diagnostics, decoratedMembers, filteredMethodsAndFields);

src/compiler/transformers/decorators-to-static/event-decorator.ts

@@ -17,11 +17,12 @@ export const eventDecoratorsToStatic = (
   diagnostics: d.Diagnostic[],
   decoratedProps: ts.ClassElement[],
   typeChecker: ts.TypeChecker,
+  program: ts.Program,
   newMembers: ts.ClassElement[]
 ) => {
   const events = decoratedProps
     .filter(ts.isPropertyDeclaration)
-    .map((prop) => parseEventDecorator(diagnostics, typeChecker, prop))
+    .map((prop) => parseEventDecorator(diagnostics, typeChecker, program, prop))
     .filter((ev) => !!ev);
 
   if (events.length > 0) {
@@ -35,13 +36,15 @@ export const eventDecoratorsToStatic = (
  * @param diagnostics a list of diagnostics used as a part of the parsing process. Any parse errors/warnings shall be
  * added to this collection
  * @param typeChecker an instance of the TypeScript type checker, used to generate information about the `@Event()` and
+ * @param program a {@link ts.Program} object
  * its surrounding context in the AST
  * @param prop the property on the Stencil component class that is decorated with `@Event()`
  * @returns generated metadata for the class member decorated by `@Event()`, or `null` if none could be derived
  */
 const parseEventDecorator = (
   diagnostics: d.Diagnostic[],
   typeChecker: ts.TypeChecker,
+  program: ts.Program,
   prop: ts.PropertyDeclaration
 ): d.ComponentCompilerStaticEvent | null => {
   const eventDecorator = retrieveTsDecorators(prop)?.find(isDecoratorNamed('Event'));
@@ -68,7 +71,7 @@ const parseEventDecorator = (
     cancelable: eventOpts && typeof eventOpts.cancelable === 'boolean' ? eventOpts.cancelable : true,
     composed: eventOpts && typeof eventOpts.composed === 'boolean' ? eventOpts.composed : true,
     docs: serializeSymbol(typeChecker, symbol),
-    complexType: getComplexType(typeChecker, prop),
+    complexType: getComplexType(typeChecker, program, prop),
   };
   validateReferences(diagnostics, eventMeta.complexType.references, prop.type);
   return eventMeta;
@@ -85,19 +88,21 @@ export const getEventName = (eventOptions: d.EventOptions, memberName: string) =
 /**
  * Derive Stencil's class member type metadata from a node in the AST
  * @param typeChecker the TypeScript type checker
+ * @param program a {@link ts.Program} object
  * @param node the node in the AST to generate metadata for
  * @returns the generated metadata
  */
 const getComplexType = (
   typeChecker: ts.TypeChecker,
+  program: ts.Program,
   node: ts.PropertyDeclaration
 ): d.ComponentCompilerPropertyComplexType => {
   const sourceFile = node.getSourceFile();
   const eventType = node.type ? getEventType(node.type) : null;
   return {
     original: eventType ? eventType.getText() : 'any',
     resolved: eventType ? resolveType(typeChecker, typeChecker.getTypeFromTypeNode(eventType)) : 'any',
-    references: eventType ? getAttributeTypeInfo(eventType, sourceFile) : {},
+    references: eventType ? getAttributeTypeInfo(eventType, sourceFile, typeChecker, program) : {},
   };
 };
 

src/compiler/transformers/decorators-to-static/method-decorator.ts

@@ -23,12 +23,13 @@ export const methodDecoratorsToStatic = (
   cmpNode: ts.ClassDeclaration,
   decoratedProps: ts.ClassElement[],
   typeChecker: ts.TypeChecker,
+  program: ts.Program,
   newMembers: ts.ClassElement[]
 ) => {
   const tsSourceFile = cmpNode.getSourceFile();
   const methods = decoratedProps
     .filter(ts.isMethodDeclaration)
-    .map((method) => parseMethodDecorator(config, diagnostics, tsSourceFile, typeChecker, method))
+    .map((method) => parseMethodDecorator(config, diagnostics, tsSourceFile, typeChecker, program, method))
     .filter((method) => !!method);
 
   if (methods.length > 0) {
@@ -41,6 +42,7 @@ const parseMethodDecorator = (
   diagnostics: d.Diagnostic[],
   tsSourceFile: ts.SourceFile,
   typeChecker: ts.TypeChecker,
+  program: ts.Program,
   method: ts.MethodDeclaration
 ): ts.PropertyAssignment | null => {
   const methodDecorator = retrieveTsDecorators(method)?.find(isDecoratorNamed('Method'));
@@ -92,8 +94,8 @@ const parseMethodDecorator = (
       signature: signatureString,
       parameters: signature.parameters.map((symbol) => serializeSymbol(typeChecker, symbol)),
       references: {
-        ...getAttributeTypeInfo(returnTypeNode, tsSourceFile),
-        ...getAttributeTypeInfo(method, tsSourceFile),
+        ...getAttributeTypeInfo(returnTypeNode, tsSourceFile, typeChecker, program),
+        ...getAttributeTypeInfo(method, tsSourceFile, typeChecker, program),
       },
       return: returnString,
     },