Add mechanism to group API ToC into sections (#1256)

Prework for #1255. This PR
only adds the generic mechanism but doesn't yet use it for Qiskit to
keep the diff smaller.

scripts/lib/api/Pkg.ts

@@ -15,6 +15,7 @@ import { join } from "path/posix";
 import { findLegacyReleaseNotes } from "./releaseNotes";
 import { getRoot } from "../fs";
 import { determineHistoricalQiskitGithubUrl } from "../qiskitMetapackage";
+import { TocGrouping } from "./TocGrouping";
 
 export interface ReleaseNoteEntry {
   title: string;
@@ -36,6 +37,7 @@ export class Pkg {
   readonly type: PackageType;
   readonly releaseNoteEntries: ReleaseNoteEntry[];
   readonly nestModulesInToc: boolean;
+  readonly tocGrouping?: TocGrouping;
 
   static VALID_NAMES = ["qiskit", "qiskit-ibm-runtime", "qiskit-ibm-provider"];
 
@@ -48,7 +50,8 @@ export class Pkg {
     versionWithoutPatch: string;
     type: PackageType;
     releaseNoteEntries: ReleaseNoteEntry[];
-    nestModulesInToc: boolean;
+    nestModulesInToc?: boolean;
+    tocGrouping?: TocGrouping;
   }) {
     this.name = kwargs.name;
     this.title = kwargs.title;
@@ -58,7 +61,8 @@ export class Pkg {
     this.versionWithoutPatch = kwargs.versionWithoutPatch;
     this.type = kwargs.type;
     this.releaseNoteEntries = kwargs.releaseNoteEntries;
-    this.nestModulesInToc = kwargs.nestModulesInToc;
+    this.nestModulesInToc = kwargs.nestModulesInToc ?? false;
+    this.tocGrouping = kwargs.tocGrouping;
   }
 
   static async fromArgs(
@@ -95,7 +99,6 @@ export class Pkg {
         githubSlug: "qiskit/qiskit-ibm-runtime",
         hasSeparateReleaseNotes: false,
         releaseNoteEntries: [],
-        nestModulesInToc: false,
       });
     }
 
@@ -107,7 +110,6 @@ export class Pkg {
         githubSlug: "qiskit/qiskit-ibm-provider",
         hasSeparateReleaseNotes: false,
         releaseNoteEntries: [],
-        nestModulesInToc: false,
       });
     }
 
@@ -124,6 +126,7 @@ export class Pkg {
     type?: PackageType;
     releaseNoteEntries?: ReleaseNoteEntry[];
     nestModulesInToc?: boolean;
+    tocGrouping?: TocGrouping;
   }): Pkg {
     return new Pkg({
       name: kwargs.name ?? "my-quantum-project",
@@ -135,6 +138,7 @@ export class Pkg {
       type: kwargs.type ?? "latest",
       releaseNoteEntries: kwargs.releaseNoteEntries ?? [],
       nestModulesInToc: kwargs.nestModulesInToc ?? false,
+      tocGrouping: kwargs.tocGrouping ?? undefined,
     });
   }
 

scripts/lib/api/TocGrouping.ts

@@ -0,0 +1,40 @@
+// This code is a Qiskit project.
+//
+// (C) Copyright IBM 2024.
+//
+// This code is licensed under the Apache License, Version 2.0. You may
+// obtain a copy of this license in the LICENSE file in the root directory
+// of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+//
+// Any modifications or derivative works of this code must retain this
+// copyright notice, and modified files need to carry a notice indicating
+// that they have been altered from the originals.
+
+/** A human-written section used to group several modules, e.g. 'Circuit Construction'. */
+type Section = {
+  name: string;
+  kind: "section";
+};
+
+/** A single module that should be shown as top-level in the left ToC. */
+type TopLevelModule = {
+  /** E.g. 'qiskit.quantum_info'. */
+  moduleId: string;
+  /** The title the left ToC should use. This can be the module name itself, but it's often helpful
+   * to give a custom name like 'Quantum information (qiskit.quantum_info)'. */
+  title: string;
+  kind: "module";
+};
+
+export type TocGroupingEntry = Section | TopLevelModule;
+
+/** A custom grouping for the top-level of the left ToC. */
+export type TocGrouping = {
+  /** The top-level entries in the left ToC, made up of Sections like 'Circuit construction' and
+   * top-level modules. Ordering is respected. */
+  entries: readonly TocGroupingEntry[];
+  /** A function to associate an arbitrary module like 'qiskit.circuit' to the corresponding Section from
+   * `TocGrouping.entries`. The returned string must match a `Section.name` from `TocGrouping.entries`.
+   * Return `undefined` if the module does not belong in any specific Section.*/
+  moduleToSection: (module: string) => string | undefined;
+};

scripts/lib/api/conversionPipeline.test.ts

@@ -60,7 +60,6 @@ test("qiskit-sphinx-theme", async () => {
     versionWithoutPatch: "0.1",
     type: "latest",
     releaseNoteEntries: [],
-    nestModulesInToc: false,
   });
   const markdownFolder = pkg.outputDir(docsBaseFolder);
 

scripts/lib/api/generateToc.test.ts

@@ -14,6 +14,7 @@ import { describe, expect, test } from "@jest/globals";
 
 import { generateToc } from "./generateToc";
 import { Pkg } from "./Pkg";
+import type { TocGroupingEntry } from "./TocGrouping";
 
 const DEFAULT_ARGS = {
   markdown: "",
@@ -177,6 +178,82 @@ describe("generateToc", () => {
     });
   });
 
+  test("TOC with grouped modules", () => {
+    // This ordering is intentional.
+    const topLevelEntries: TocGroupingEntry[] = [
+      { moduleId: "my_project", title: "API index", kind: "module" },
+      { name: "Group 2", kind: "section" },
+      { name: "Group 1", kind: "section" },
+      // Ensure we can handle unused entries.
+      { moduleId: "unused_module", title: "unused", kind: "module" },
+      { name: "Unused section", kind: "section" },
+    ];
+    const tocGrouping = {
+      entries: topLevelEntries,
+      moduleToSection: (module: string) =>
+        module == "my_project.module" ? "Group 1" : "Group 2",
+    };
+
+    const toc = generateToc(Pkg.mock({ tocGrouping }), [
+      {
+        meta: {
+          apiType: "module",
+          apiName: "my_project",
+        },
+        url: "/docs/my_project",
+        ...DEFAULT_ARGS,
+      },
+      {
+        meta: {
+          apiType: "module",
+          apiName: "my_project.module",
+        },
+        url: "/docs/my_project.module",
+        ...DEFAULT_ARGS,
+      },
+      {
+        meta: {
+          apiType: "module",
+          apiName: "my_project.module.submodule",
+        },
+        url: "/docs/my_project.module.submodule",
+        ...DEFAULT_ARGS,
+      },
+    ]);
+    expect(toc).toEqual({
+      collapsed: true,
+      title: "My Quantum Project",
+      children: [
+        {
+          title: "API index",
+          url: "/docs/my_project",
+        },
+        {
+          title: "Group 2",
+          children: [
+            {
+              title: "my_project.module.submodule",
+              url: "/docs/my_project.module.submodule",
+            },
+          ],
+        },
+        {
+          title: "Group 1",
+          children: [
+            {
+              title: "my_project.module",
+              url: "/docs/my_project.module",
+            },
+          ],
+        },
+        {
+          title: "Release notes",
+          url: "/api/my-quantum-project/release-notes",
+        },
+      ],
+    });
+  });
+
   test("TOC with distinct release note files", () => {
     const toc = generateToc(
       Pkg.mock({

scripts/lib/api/generateToc.ts

@@ -15,6 +15,7 @@ import { isEmpty, orderBy } from "lodash";
 import { getLastPartFromFullIdentifier } from "../stringUtils";
 import { HtmlToMdResultWithUrl } from "./HtmlToMdResult";
 import { Pkg } from "./Pkg";
+import type { TocGrouping, TocGroupingEntry } from "./TocGrouping";
 
 export type TocEntry = {
   title: string;
@@ -45,11 +46,17 @@ export function generateToc(pkg: Pkg, results: HtmlToMdResultWithUrl[]): Toc {
 
   addItemsToModules(items, tocModulesByTitle, tocModuleTitles);
 
-  // Most packages don't nest submodules because their module list is so small,
-  // so it's more useful to show them all and have less nesting.
-  const sortedTocModules = pkg.nestModulesInToc
-    ? getNestedTocModulesSorted(tocModulesByTitle, tocModuleTitles)
-    : sortAndTruncateModules(tocModules);
+  let sortedTocModules;
+  if (pkg.tocGrouping) {
+    sortedTocModules = groupAndSortModules(pkg.tocGrouping, tocModulesByTitle);
+  } else if (pkg.nestModulesInToc) {
+    sortedTocModules = getNestedTocModulesSorted(
+      tocModulesByTitle,
+      tocModuleTitles,
+    );
+  } else {
+    sortedTocModules = sortAndTruncateModules(tocModules);
+  }
   generateOverviewPage(tocModules);
 
   return {
@@ -113,6 +120,69 @@ function addItemsToModules(
   }
 }
 
+/** Group the modules into the user-defined tocGrouping, which defines the order
+ * of the top-level entries in the ToC.
+ *
+ * Within each TocGrouping.Section, this function will also sort the modules alphabetically.
+ */
+function groupAndSortModules(
+  tocGrouping: TocGrouping,
+  tocModulesByTitle: Map<string, TocEntry>,
+): TocEntry[] {
+  // First, record every valid section and top-level module.
+  const topLevelModuleIds = new Set<string>();
+  const sectionsToModules = new Map<string, TocEntry[]>();
+  tocGrouping.entries.forEach((entry) => {
+    if (entry.kind === "module") {
+      topLevelModuleIds.add(entry.moduleId);
+    } else {
+      sectionsToModules.set(entry.name, []);
+    }
+  });
+
+  // Go through each module in use and ensure it is either a top-level module
+  // or assign it to its section.
+  for (const tocModule of tocModulesByTitle.values()) {
+    if (topLevelModuleIds.has(tocModule.title)) continue;
+    const section = tocGrouping.moduleToSection(tocModule.title);
+    if (!section) {
+      throw new Error(
+        `Unrecognized module '${tocModule.title}'. It must either be listed as a module in TocGrouping.entries or be matched in TocGrouping.moduleToSection().`,
+      );
+    }
+    const sectionModules = sectionsToModules.get(section);
+    if (!sectionModules) {
+      throw new Error(
+        `Unknown section '${section}' set for the module '${tocModule.title}'. This means TocGrouping.moduleToSection() is not aligned with TocGrouping.entries`,
+      );
+    }
+    sectionModules.push(tocModule);
+  }
+
+  // Finally, create the ToC by using the ordering from moduleGrouping.entries.
+  // Note that moduleGrouping.entries might be a superset of the modules/sections
+  // actually in use for the API version, so we sometimes skip adding individual
+  // entries to the final result.
+  const result: TocEntry[] = [];
+  tocGrouping.entries.forEach((entry) => {
+    if (entry.kind === "module") {
+      const module = tocModulesByTitle.get(entry.moduleId);
+      if (!module) return;
+      module.title = entry.title;
+      result.push(module);
+    } else {
+      const modules = sectionsToModules.get(entry.name);
+      if (!modules || modules.length === 0) return;
+      result.push({
+        title: entry.name,
+        // Within a section, sort alphabetically.
+        children: orderEntriesByTitle(modules),
+      });
+    }
+  });
+  return result;
+}
+
 /** Nest modules so that only top-level modules like qiskit.circuit are at the top
  * and submodules like qiskit.circuit.library are nested.
  *