(DOCSP-32243) Handle and test pages with page-level code block select (…

…#94) * handle and test pages with page-level codeblock select * clean up comment * add spacing around tables * refactor per CB feedback * Refactor to remove undefined case which should never occur
mongodb · Aug 23, 2023 · f6c3f7e · f6c3f7e
1 parent 64dbd0d
commit f6c3f7e
Show file tree

Hide file tree

Showing 5 changed files with 26,532 additions and 59 deletions.
diff --git a/ingest/src/snootyAstToMd.test.ts b/ingest/src/snootyAstToMd.test.ts
@@ -1,11 +1,11 @@
 import Path from "path";
-import { readFileSync } from "fs";
+import fs from "fs";
 import { snootyAstToMd, getTitleFromSnootyAst } from "./snootyAstToMd";
 import { SnootyNode } from "./SnootyDataSource";
 
 describe("snootyAstToMd", () => {
   const samplePage = JSON.parse(
-    readFileSync(Path.resolve(__dirname, "./test_data/samplePage.json"), {
+    fs.readFileSync(Path.resolve(__dirname, "./test_data/samplePage.json"), {
       encoding: "utf-8",
     })
   );
@@ -79,7 +79,7 @@ describe("snootyAstToMd", () => {
   });
   describe("Renders code blocks", () => {
     const samplePage = JSON.parse(
-      readFileSync(
+      fs.readFileSync(
         Path.resolve(__dirname, "./test_data/samplePageWithCodeExamples.json"),
         {
           encoding: "utf-8",
@@ -100,8 +100,8 @@ describe("snootyAstToMd", () => {
 
   it("renders HTML tables", () => {
     const samplePage = JSON.parse(
-      readFileSync(
-        Path.resolve(__dirname, "./test_data/sampleListTable.json"),
+      fs.readFileSync(
+        Path.resolve(__dirname, "./test_data/samplePageWithListTable.json"),
         {
           encoding: "utf-8",
         }
@@ -110,7 +110,9 @@ describe("snootyAstToMd", () => {
     const result = snootyAstToMd(samplePage.data.ast, {
       baseUrl: "/",
     });
-    const expected = `<table>
+    const expected = `before text
+
+<table>
 <tr>
 <th>
 Action
@@ -185,7 +187,11 @@ Stop and fail the aggregation operation. Any changes to the output collection fr
 
 </td>
 </tr>
-</table>`;
+</table>
+
+after text
+
+`;
     expect(result).toBe(expected);
     const openingTagCount = result.split("<td>").length - 1;
     const closingTagCount = result.split("</td>").length - 1;
@@ -195,7 +201,7 @@ Stop and fail the aggregation operation. Any changes to the output collection fr
 
   it("renders HTML tables with multiple header rows", () => {
     const ast: SnootyNode = JSON.parse(
-      readFileSync(
+      fs.readFileSync(
         Path.resolve(
           __dirname,
           "./test_data/samplePageWithMultiHeaderTableAst.json"
@@ -206,7 +212,9 @@ Stop and fail the aggregation operation. Any changes to the output collection fr
     const result = snootyAstToMd(ast, {
       baseUrl: "/",
     });
-    const expected = `<table>
+    const expected = `before text
+
+<table>
 <tr>
 <th>
 h1
@@ -237,13 +245,17 @@ d2
 
 </td>
 </tr>
-</table>`;
+</table>
+
+after text
+
+`;
     expect(result).toBe(expected);
   });
 
   it("strips comments", () => {
     const ast: SnootyNode = JSON.parse(
-      readFileSync(
+      fs.readFileSync(
         Path.resolve(__dirname, "./test_data/samplePageWithCommentAst.json"),
         "utf-8"
       )
@@ -284,7 +296,7 @@ Describes one method for supporting keyword search by storing keywords in an arr
   });
   it("renders tab sets", () => {
     const samplePageWithTabs = JSON.parse(
-      readFileSync(
+      fs.readFileSync(
         Path.resolve(__dirname, "./test_data/samplePageWithTabs.json"),
         {
           encoding: "utf-8",
@@ -309,10 +321,37 @@ Describes one method for supporting keyword search by storing keywords in an arr
     const numberTabEnd = [...result.matchAll(/<\/Tab>/g)].length;
     expect(numberTabStart).toBe(numberTabEnd);
   });
+  it("renders driver tab sets", () => {
+    const samplePageWithTabs = JSON.parse(
+      fs.readFileSync(
+        Path.resolve(__dirname, "./test_data/samplePageWithTabsDrivers.json"),
+        {
+          encoding: "utf-8",
+        }
+      )
+    );
+    const result = snootyAstToMd(samplePageWithTabs.data.ast, {
+      baseUrl: "/",
+    });
+    const expectedToContainTabsStart = "\n\n<Tabs>\n\n";
+    const expectedToContainTabsEnd = "\n\n</Tabs>\n\n";
+    const expectedToContainTabStart = '\n\n<Tab name="Java (Sync)">\n\n';
+    const expectedToContainTabEnd = "\n\n</Tab>\n\n";
+    expect(result).toContain(expectedToContainTabsStart);
+    expect(result).toContain(expectedToContainTabsEnd);
+    expect(result).toContain(expectedToContainTabStart);
+    expect(result).toContain(expectedToContainTabEnd);
+    const numberTabsStart = [...result.matchAll(/<Tabs>/g)].length;
+    const numberTabsEnd = [...result.matchAll(/<\/Tabs>/g)].length;
+    expect(numberTabsStart).toBe(numberTabsEnd);
+    const numberTabStart = [...result.matchAll(/<Tab name=/g)].length;
+    const numberTabEnd = [...result.matchAll(/<\/Tab>/g)].length;
+    expect(numberTabStart).toBe(numberTabEnd);
+  });
 });
 describe("getTitleFromSnootyAst", () => {
   const samplePage = JSON.parse(
-    readFileSync(Path.resolve(__dirname, "./test_data/samplePage.json"), {
+    fs.readFileSync(Path.resolve(__dirname, "./test_data/samplePage.json"), {
       encoding: "utf-8",
     })
   );

diff --git a/ingest/src/snootyAstToMd.ts b/ingest/src/snootyAstToMd.ts
@@ -1,4 +1,5 @@
 import { SnootyNode } from "./SnootyDataSource";
+import { strict as assert } from "assert";
 
 enum TableMode {
   IN_TABLE = 0,
@@ -145,52 +146,7 @@ export const snootyAstToMd = (
         .join("")}\``;
       break;
     case "directive":
-      if (node.name === "list-table") {
-        const directiveOptions = (node as { options?: Record<string, unknown> })
-          .options;
-        const headerRows =
-          directiveOptions &&
-          typeof directiveOptions["header-rows"] === "number"
-            ? directiveOptions["header-rows"]
-            : 0;
-        text += [
-          "<table>",
-          node.children
-            .map((child) =>
-              snootyAstToMd(
-                child,
-                {
-                  ...options,
-                  table: {
-                    mode: TableMode.IN_TABLE,
-                    headerRows,
-                  },
-                },
-                parentHeadingLevel
-              )
-            )
-            .join(""),
-          "</table>",
-        ].join("\n");
-      } else if (node.name === "tab") {
-        const tabName = (
-          node.argument && Array.isArray(node.argument) && node.argument.length
-            ? node.argument.find((arg) => arg.type === "text")?.value ?? ""
-            : ""
-        ).trim();
-        text += `\n\n<Tab ${`name="${tabName ?? ""}"`}>\n\n${node.children
-          .map((child) => snootyAstToMd(child, options, parentHeadingLevel))
-          .join("")}\n\n</Tab>\n\n`;
-      } else if (node.name === "tabs") {
-        text += `\n\n<Tabs>\n\n${node.children
-          .map((child) => snootyAstToMd(child, options, parentHeadingLevel))
-          .join("")}\n\n</Tabs>\n\n`;
-      } else {
-        // other "directive" nodes not parsed in particular way
-        text += node.children
-          .map((subnode) => snootyAstToMd(subnode, options, parentHeadingLevel))
-          .join("");
-      }
+      text += handleDirective(node, options, parentHeadingLevel);
       break;
     // No longer including links
     // case "ref_role": {
@@ -228,6 +184,69 @@ export const snootyAstToMd = (
   return text.replaceAll(/\n{3,}/g, "\n\n").trimStart(); // remove extra newlines with just 2
 };
 
+/**
+  Helper function to handle directives. Directives are special nodes that
+  contain a variety of different content types.
+ */
+const handleDirective = (
+  node: SnootyNode,
+  options: SnootyAstToMdOptions,
+  parentHeadingLevel: number
+) => {
+  assert(
+    node.children,
+    "This function should only be called if node has children"
+  );
+  switch (node.name) {
+    case "list-table":
+      // eslint-disable-next-line no-case-declarations
+      const directiveOptions = (node as { options?: Record<string, unknown> })
+        .options;
+      // eslint-disable-next-line no-case-declarations
+      const headerRows =
+        directiveOptions && typeof directiveOptions["header-rows"] === "number"
+          ? directiveOptions["header-rows"]
+          : 0;
+      return [
+        "\n\n<table>",
+        node.children
+          .map((child) =>
+            snootyAstToMd(
+              child,
+              {
+                ...options,
+                table: {
+                  mode: TableMode.IN_TABLE,
+                  headerRows,
+                },
+              },
+              parentHeadingLevel
+            )
+          )
+          .join(""),
+        "</table>\n\n",
+      ].join("\n");
+    case "tab":
+      // eslint-disable-next-line no-case-declarations
+      const tabName = (
+        node.argument && Array.isArray(node.argument) && node.argument.length
+          ? node.argument.find((arg) => arg.type === "text")?.value ?? ""
+          : ""
+      ).trim();
+      return `\n\n<Tab ${`name="${tabName ?? ""}"`}>\n\n${node.children
+        .map((child) => snootyAstToMd(child, options, parentHeadingLevel))
+        .join("")}\n\n</Tab>\n\n`;
+    case "tabs" || "tabs-drivers":
+      return `\n\n<Tabs>\n\n${node.children
+        .map((child) => snootyAstToMd(child, options, parentHeadingLevel))
+        .join("")}\n\n</Tabs>\n\n`;
+    default:
+      return node.children
+        .map((subnode) => snootyAstToMd(subnode, options, parentHeadingLevel))
+        .join("");
+  }
+};
+
 const findNode = (
   node: SnootyNode,
   predicate: (node: SnootyNode) => boolean

diff --git a/ingest/src/test_data/sampleListTable.json → ...rc/test_data/samplePageWithListTable.json b/ingest/src/test_data/sampleListTable.json → ...rc/test_data/samplePageWithListTable.json
@@ -7,6 +7,15 @@
       "type": "root",
       "position": { "start": { "line": 0 } },
       "children": [
+        {
+          "type": "paragraph",
+          "children": [
+            {
+              "type": "text",
+              "value": "before text"
+            }
+          ]
+        },
         {
           "type": "directive",
           "position": {
@@ -1353,6 +1362,15 @@
             "header-rows": 1,
             "widths": "20 80"
           }
+        },
+        {
+          "type": "paragraph",
+          "children": [
+            {
+              "type": "text",
+              "value": "after text"
+            }
+          ]
         }
       ]
     }

diff --git a/ingest/src/test_data/samplePageWithMultiHeaderTableAst.json b/ingest/src/test_data/samplePageWithMultiHeaderTableAst.json
@@ -2,6 +2,15 @@
   "type": "root",
   "position": { "start": { "line": 0 } },
   "children": [
+    {
+      "type": "paragraph",
+      "children": [
+        {
+          "type": "text",
+          "value": "before text"
+        }
+      ]
+    },
     {
       "type": "directive",
       "children": [
@@ -129,6 +138,15 @@
       "options": {
         "header-rows": 2
       }
+    },
+    {
+      "type": "paragraph",
+      "children": [
+        {
+          "type": "text",
+          "value": "after text"
+        }
+      ]
     }
   ]
 }