API generation script uses MDX components (Qiskit#1026)

Part of Qiskit#1008

This PR teaches our API generation script to generate MDX components
that will define the styling of the classes, properties, attributes,
methods, functions, and exceptions.

The script will generate custom tags to wrap all the content and store
useful information as props. This is an example of git diff for an
Attribute:
```diff
- ### dtm
- Return the system time resolution of output signals :returns: The output signal timestep in seconds. :rtype: dtm
+  ### dtm
+  <Attribute id="qiskit_ibm_provider.IBMBackend.dtm" name="dtm">
+    Return the system time resolution of output signals :returns: The output signal timestep in seconds. :rtype: dtm
+  </Attribute>
```
The script continues generating headers for every apiType that needs it,
but this will be removed in a follow-up.

### Details

The PR adds a new script named `generateMdxComponents.ts` that will
generate all the components. The main function is `processMdxComponent`
where we prepare the necessary props for each component and then return
an open tag and a closed tag ready for its use.

To process the new tags, new handlers have been added to `htmlToMd.ts`.

---------

Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com>

scripts/lib/api/generateApiComponents.test.ts

@@ -79,6 +79,7 @@ describe("createOpeningTag()", () => {
     name='run'
     attributeTypeHint='undefined'
     attributeValue='undefined'
+    isDedicatedPage='undefined'
     github='undefined'
     signature='Estimator.run(circuits, observables, parameter_values=None, **kwargs)'
     extraSignatures='[]'
@@ -100,9 +101,10 @@ describe("createOpeningTag()", () => {
     name='run'
     attributeTypeHint='undefined'
     attributeValue='undefined'
+    isDedicatedPage='undefined'
     github='undefined'
     signature='Estimator.run(circuits, observables, parameter_values=None, **kwargs)'
-    extraSignatures='[${APOSTROPHE_HEX_CODE}Estimator.run(circuits, observables, parameter_values=None, **kwargs)${APOSTROPHE_HEX_CODE}, ${APOSTROPHE_HEX_CODE}Estimator.run(circuits, observables, parameter_values=None, **kwargs)${APOSTROPHE_HEX_CODE}]'
+    extraSignatures='["Estimator.run(circuits, observables, parameter_values=None, **kwargs)", "Estimator.run(circuits, observables, parameter_values=None, **kwargs)"]'
     >
   `);
   });
@@ -121,6 +123,7 @@ describe("createOpeningTag()", () => {
     name='instance'
     attributeTypeHint='str | None'
     attributeValue='None'
+    isDedicatedPage='undefined'
     github='undefined'
     signature=''
     extraSignatures='[]'
@@ -139,6 +142,7 @@ describe("createOpeningTag()", () => {
     name='undefined'
     attributeTypeHint='undefined'
     attributeValue='undefined'
+    isDedicatedPage='undefined'
     github='undefined'
     signature=''
     extraSignatures='[]'

scripts/lib/api/generateApiComponents.ts

@@ -10,24 +10,253 @@
 // copyright notice, and modified files need to carry a notice indicating
 // that they have been altered from the originals.
 
-import { CheerioAPI, Cheerio } from "cheerio";
+import { CheerioAPI, Cheerio, Element } from "cheerio";
 import { unified } from "unified";
 import rehypeParse from "rehype-parse";
 import rehypeRemark from "rehype-remark";
 import remarkStringify from "remark-stringify";
 
-import { APOSTROPHE_HEX_CODE } from "../stringUtils";
+import { ApiType } from "./Metadata";
+import {
+  getLastPartFromFullIdentifier,
+  APOSTROPHE_HEX_CODE,
+} from "../stringUtils";
 
 export type ComponentProps = {
-  id: string;
+  id?: string;
   name?: string;
   attributeTypeHint?: string;
   attributeValue?: string;
   githubSourceLink?: string;
   rawSignature?: string;
   extraRawSignatures?: string[];
+  isDedicatedPage?: boolean;
 };
 
+const APITYPE_TO_TAG: Record<string, string> = {
+  class: "class",
+  exception: "class",
+  attribute: "attribute",
+  property: "attribute",
+  function: "function",
+  method: "function",
+};
+
+export async function processMdxComponent(
+  $: CheerioAPI,
+  $main: Cheerio<any>,
+  signatures: Cheerio<Element>[],
+  $dl: Cheerio<any>,
+  priorApiType: ApiType | undefined,
+  apiType: ApiType,
+  id: string,
+): Promise<[string, string]> {
+  const tagName = APITYPE_TO_TAG[apiType];
+
+  const $firstSignature = signatures.shift()!;
+  const componentProps = prepareProps(
+    $,
+    $main,
+    $firstSignature,
+    $dl,
+    priorApiType,
+    apiType,
+    id,
+  );
+
+  const extraProps = signatures.flatMap(
+    ($overloadedSignature) =>
+      prepareProps($, $main, $overloadedSignature, $dl, apiType, apiType, id) ??
+      [],
+  );
+  addExtraSignatures(componentProps, extraProps);
+
+  return [await createOpeningTag(tagName, componentProps), `</${tagName}>`];
+}
+
+// ------------------------------------------------------------------
+// Prepare props for MDX components
+// ------------------------------------------------------------------
+
+function prepareProps(
+  $: CheerioAPI,
+  $main: Cheerio<any>,
+  $child: Cheerio<Element>,
+  $dl: Cheerio<any>,
+  priorApiType: ApiType | undefined,
+  apiType: ApiType,
+  id: string,
+): ComponentProps {
+  const preparePropsPerApiType: Record<string, () => ComponentProps> = {
+    class: () => prepareClassProps($child, githubSourceLink, id),
+    property: () =>
+      preparePropertyProps($child, $dl, priorApiType, githubSourceLink, id),
+    method: () =>
+      prepareMethodProps($, $child, $dl, priorApiType, githubSourceLink, id),
+    attribute: () =>
+      prepareAttributeProps($, $child, $dl, priorApiType, githubSourceLink, id),
+    function: () =>
+      prepareFunctionOrExceptionProps($, $child, $dl, id, githubSourceLink),
+    exception: () =>
+      prepareFunctionOrExceptionProps($, $child, $dl, id, githubSourceLink),
+  };
+
+  const githubSourceLink = prepareGitHubLink($child, apiType === "method");
+  findByText($, $main, "em.property", apiType).remove();
+
+  if (!(apiType in preparePropsPerApiType)) {
+    throw new Error(`Unhandled Python type: ${apiType}`);
+  }
+
+  return preparePropsPerApiType[apiType]();
+}
+
+function prepareClassProps(
+  $child: Cheerio<any>,
+  githubSourceLink: string | undefined,
+  id: string,
+): ComponentProps {
+  return {
+    id,
+    rawSignature: $child.html()!,
+    githubSourceLink,
+  };
+}
+
+function preparePropertyProps(
+  $child: Cheerio<any>,
+  $dl: Cheerio<any>,
+  priorApiType: string | undefined,
+  githubSourceLink: string | undefined,
+  id: string,
+): ComponentProps {
+  const rawSignature = $child.find("em").text()?.replace(/^:\s+/, "");
+  const props = {
+    id,
+    name: getLastPartFromFullIdentifier(id),
+    rawSignature,
+    githubSourceLink,
+  };
+
+  if (!priorApiType && id) {
+    $dl.siblings("h1").text(getLastPartFromFullIdentifier(id));
+    return {
+      ...props,
+      isDedicatedPage: true,
+    };
+  }
+
+  return props;
+}
+
+function prepareMethodProps(
+  $: CheerioAPI,
+  $child: Cheerio<any>,
+  $dl: Cheerio<any>,
+  priorApiType: string | undefined,
+  githubSourceLink: string | undefined,
+  id: string,
+): ComponentProps {
+  const props = {
+    id,
+    name: getLastPartFromFullIdentifier(id),
+    rawSignature: $child.html()!,
+    githubSourceLink,
+  };
+
+  if (id) {
+    if (!priorApiType) {
+      $dl.siblings("h1").text(getLastPartFromFullIdentifier(id));
+      return {
+        ...props,
+        isDedicatedPage: true,
+      };
+    } else if ($child.attr("id")) {
+      $(`<h3>${getLastPartFromFullIdentifier(id)}</h3>`).insertBefore($dl);
+    }
+  }
+  return props;
+}
+
+function prepareAttributeProps(
+  $: CheerioAPI,
+  $child: Cheerio<any>,
+  $dl: Cheerio<any>,
+  priorApiType: string | undefined,
+  githubSourceLink: string | undefined,
+  id: string,
+): ComponentProps {
+  if (!priorApiType) {
+    if (id) {
+      $dl.siblings("h1").text(getLastPartFromFullIdentifier(id));
+    }
+    const rawSignature = $child.find("em").text()?.replace(/^:\s+/, "");
+    return {
+      id,
+      rawSignature,
+      githubSourceLink,
+      isDedicatedPage: true,
+    };
+  }
+
+  // Else, the attribute is embedded on the class
+  const text = $child.text();
+
+  // Index of the default value of the attribute
+  let equalIndex = text.indexOf("=");
+  if (equalIndex == -1) {
+    equalIndex = text.length;
+  }
+  // Index of the attribute's type. The type should be
+  // found before the default value
+  let colonIndex = text.slice(0, equalIndex).indexOf(":");
+  if (colonIndex == -1) {
+    colonIndex = text.length;
+  }
+
+  $(`<h3>${getLastPartFromFullIdentifier(id)}</h3>`).insertBefore($dl);
+  // The attributes have the following shape: name [: type] [= value]
+  const name = text.slice(0, Math.min(colonIndex, equalIndex)).trim();
+  const attributeTypeHint = text
+    .slice(Math.min(colonIndex + 1, equalIndex), equalIndex)
+    .trim();
+  const attributeValue = text.slice(equalIndex, text.length).trim();
+
+  return {
+    id,
+    name,
+    attributeTypeHint,
+    attributeValue,
+  };
+}
+
+function prepareFunctionOrExceptionProps(
+  $: CheerioAPI,
+  $child: Cheerio<any>,
+  $dl: Cheerio<any>,
+  id: string,
+  githubSourceLink: string | undefined,
+): ComponentProps {
+  const props = {
+    id,
+    name: getLastPartFromFullIdentifier(id),
+    rawSignature: $child.html()!,
+    githubSourceLink,
+  };
+
+  const pageHeading = $dl.siblings("h1").text();
+  if (id.endsWith(pageHeading) && pageHeading != "") {
+    // Page is already dedicated to apiType; no heading needed
+    return {
+      ...props,
+      isDedicatedPage: true,
+    };
+  }
+  $(`<h3>${getLastPartFromFullIdentifier(id)}</h3>`).insertBefore($dl);
+
+  return props;
+}
+
 // ------------------------------------------------------------------
 // Generate MDX components
 // ------------------------------------------------------------------
@@ -52,18 +281,15 @@ export async function createOpeningTag(
   const signature = await htmlSignatureToMd(props.rawSignature!);
   const extraSignatures: string[] = [];
   for (const sig of props.extraRawSignatures ?? []) {
-    extraSignatures.push(
-      `${APOSTROPHE_HEX_CODE}${await htmlSignatureToMd(
-        sig!,
-      )}${APOSTROPHE_HEX_CODE}`,
-    );
+    extraSignatures.push(`"${await htmlSignatureToMd(sig!)}"`);
   }
 
   return `<${tagName} 
     id='${props.id}'
     name='${props.name}'
     attributeTypeHint='${attributeTypeHint}'
     attributeValue='${attributeValue}'
+    isDedicatedPage='${props.isDedicatedPage}'
     github='${props.githubSourceLink}'
     signature='${signature}'
     extraSignatures='[${extraSignatures.join(", ")}]'
@@ -139,6 +365,7 @@ export async function htmlSignatureToMd(
   return String(file)
     .replaceAll("\n", "")
     .replaceAll("'", APOSTROPHE_HEX_CODE)
+    .replaceAll('"', '\\"')
     .replace(/^`/, "")
     .replace(/`$/, "");
 }