fix(mdx-loader): get correct error line numbers, handle front matter + contentTitle with remark

packages/docusaurus-mdx-loader/package.json

@@ -35,13 +35,15 @@
     "mdast-util-to-string": "^3.2.0",
     "rehype-raw": "^6.1.1",
     "remark-directive": "^2.0.1",
+    "remark-frontmatter": "^5.0.0",
     "remark-emoji": "^2.2.0",
     "remark-gfm": "^3.0.1",
     "stringify-object": "^3.3.0",
     "tslib": "^2.6.0",
     "unified": "^10.1.2",
     "unist-util-visit": "^2.0.3",
     "url-loader": "^4.1.1",
+    "vfile": "^5.3.7",
     "webpack": "^5.88.1"
   },
   "devDependencies": {

packages/docusaurus-mdx-loader/src/loader.ts

@@ -9,7 +9,6 @@ import fs from 'fs-extra';
 import logger from '@docusaurus/logger';
 import {
   parseFrontMatter,
-  parseMarkdownContentTitle,
   escapePath,
   getFileLoaderUtils,
 } from '@docusaurus/utils';
@@ -122,6 +121,15 @@ function ensureMarkdownConfig(reqOptions: Options) {
   }
 }
 
+/**
+ * data.contentTitle is set by the remark contentTitle plugin
+ */
+function extractContentTitleData(data: {
+  [key: string]: unknown;
+}): string | undefined {
+  return data.contentTitle as string | undefined;
+}
+
 export async function mdxLoader(
   this: LoaderContext<Options>,
   fileString: string,
@@ -132,18 +140,11 @@ export async function mdxLoader(
   const {query} = this;
   ensureMarkdownConfig(reqOptions);
 
-  const {frontMatter, content: contentWithTitle} = parseFrontMatter(fileString);
+  const {frontMatter} = parseFrontMatter(fileString);
   const mdxFrontMatter = validateMDXFrontMatter(frontMatter.mdx);
 
-  const {content: contentUnprocessed, contentTitle} = parseMarkdownContentTitle(
-    contentWithTitle,
-    {
-      removeContentTitle: reqOptions.removeContentTitle,
-    },
-  );
-
-  const content = preprocessor({
-    fileContent: contentUnprocessed,
+  const preprocessedContent = preprocessor({
+    fileContent: fileString,
     filePath,
     admonitions: reqOptions.admonitions,
     markdownConfig: reqOptions.markdownConfig,
@@ -158,9 +159,13 @@ export async function mdxLoader(
     mdxFrontMatter,
   });
 
-  let result: string;
+  let result: {content: string; data: {[key: string]: unknown}};
   try {
-    result = await processor.process({content, filePath});
+    result = await processor.process({
+      content: preprocessedContent,
+      filePath,
+      frontMatter,
+    });
   } catch (errorUnknown) {
     const error = errorUnknown as Error;
 
@@ -184,6 +189,8 @@ export async function mdxLoader(
     );
   }
 
+  const contentTitle = extractContentTitleData(result.data);
+
   // MDX partials are MDX files starting with _ or in a folder starting with _
   // Partial are not expected to have associated metadata files or front matter
   const isMDXPartial = reqOptions.isMDXPartial?.(filePath);
@@ -244,7 +251,7 @@ ${assets ? `export const assets = ${createAssetsExportCode(assets)};` : ''}
 
   const code = `
 ${exportsCode}
-${result}
+${result.content}
 `;
 
   return callback(null, code);

packages/docusaurus-mdx-loader/src/processor.ts

@@ -7,6 +7,7 @@
 
 import emoji from 'remark-emoji';
 import headings from './remark/headings';
+import contentTitle from './remark/contentTitle';
 import toc from './remark/toc';
 import transformImage from './remark/transformImage';
 import transformLinks from './remark/transformLinks';
@@ -28,15 +29,19 @@ import type {ProcessorOptions} from '@mdx-js/mdx';
 // See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
 type Pluggable = any; // TODO fix this asap
 
+type SimpleProcessorResult = {content: string; data: {[key: string]: unknown}};
+
 // TODO alt interface because impossible to import type Processor (ESM + TS :/)
 type SimpleProcessor = {
   process: ({
     content,
     filePath,
+    frontMatter,
   }: {
     content: string;
     filePath: string;
-  }) => Promise<string>;
+    frontMatter: {[key: string]: unknown};
+  }) => Promise<SimpleProcessorResult>;
 };
 
 const DEFAULT_OPTIONS: MDXOptions = {
@@ -74,11 +79,13 @@ function getAdmonitionsPlugins(
 // Need to be async due to ESM dynamic imports...
 async function createProcessorFactory() {
   const {createProcessor: createMdxProcessor} = await import('@mdx-js/mdx');
+  const {default: frontmatter} = await import('remark-frontmatter');
   const {default: rehypeRaw} = await import('rehype-raw');
   const {default: gfm} = await import('remark-gfm');
   // TODO using fork until PR merged: https://github.com/leebyron/remark-comment/pull/3
   const {default: comment} = await import('@slorber/remark-comment');
   const {default: directive} = await import('remark-directive');
+  const {VFile} = await import('vfile');
 
   // /!\ this method is synchronous on purpose
   // Using async code here can create cache entry race conditions!
@@ -91,7 +98,9 @@ async function createProcessorFactory() {
   }): SimpleProcessor {
     const remarkPlugins: MDXPlugin[] = [
       ...(options.beforeDefaultRemarkPlugins ?? []),
+      frontmatter,
       directive,
+      [contentTitle, {removeContentTitle: options.removeContentTitle}],
       ...getAdmonitionsPlugins(options.admonitions ?? false),
       ...DEFAULT_OPTIONS.remarkPlugins,
       details,
@@ -158,13 +167,19 @@ async function createProcessorFactory() {
     });
 
     return {
-      process: async ({content, filePath}) =>
-        mdxProcessor
-          .process({
-            value: content,
-            path: filePath,
-          })
-          .then((res) => res.toString()),
+      process: async ({content, filePath, frontMatter}) => {
+        const vfile = new VFile({
+          value: content,
+          path: filePath,
+          data: {
+            frontMatter,
+          },
+        });
+        return mdxProcessor.process(vfile).then((result) => ({
+          content: result.toString(),
+          data: result.data,
+        }));
+      },
     };
   }
 

packages/docusaurus-mdx-loader/src/remark/contentTitle/__tests__/index.test.ts

@@ -0,0 +1,151 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import plugin from '../index';
+
+async function process(
+  content: string,
+  options: {removeContentTitle?: boolean} = {},
+) {
+  const {remark} = await import('remark');
+  const processor = await remark().use({plugins: [[plugin, options]]});
+  return processor.process(content);
+}
+
+describe('contentTitle remark plugin', () => {
+  describe('extracts data.contentTitle', () => {
+    it('extracts h1 heading', async () => {
+      const result = await process(`
+# contentTitle 1
+
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+  `);
+
+      expect(result.data.contentTitle).toBe('contentTitle 1');
+    });
+
+    it('extracts h1 heading alt syntax', async () => {
+      const result = await process(`
+contentTitle alt
+===
+
+# contentTitle 1
+
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+  `);
+
+      expect(result.data.contentTitle).toBe('contentTitle alt');
+    });
+
+    it('works with no contentTitle', async () => {
+      const result = await process(`
+## Heading Two {#custom-heading-two}
+
+some **markdown** *content*
+  `);
+
+      expect(result.data.contentTitle).toBeUndefined();
+    });
+
+    it('ignore contentTitle if not in first position', async () => {
+      const result = await process(`
+## Heading Two {#custom-heading-two}
+
+# contentTitle 1
+
+some **markdown** *content*
+  `);
+
+      expect(result.data.contentTitle).toBeUndefined();
+    });
+
+    it('is able to decently serialize Markdown syntax', async () => {
+      const result = await process(`
+# some **markdown** \`content\` _italic_
+
+some **markdown** *content*
+  `);
+
+      expect(result.data.contentTitle).toBe('some markdown content italic');
+    });
+  });
+
+  describe('returns appropriate content', () => {
+    it('returns content unmodified', async () => {
+      const content = `
+# contentTitle 1
+
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+`.trim();
+
+      const result = await process(content);
+
+      expect(result.toString().trim()).toEqual(content);
+    });
+
+    it('can strip contentTitle', async () => {
+      const content = `
+# contentTitle 1
+
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+`.trim();
+
+      const result = await process(content, {removeContentTitle: true});
+
+      expect(result.toString().trim()).toEqual(
+        `
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+`.trim(),
+      );
+    });
+
+    it('can strip contentTitle alt', async () => {
+      const content = `
+contentTitle alt
+===
+
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+`.trim();
+
+      const result = await process(content, {removeContentTitle: true});
+
+      expect(result.toString().trim()).toEqual(
+        `
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+`.trim(),
+      );
+    });
+  });
+});

packages/docusaurus-mdx-loader/src/remark/contentTitle/index.ts

@@ -0,0 +1,53 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import visit, {EXIT} from 'unist-util-visit';
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+import type {Transformer} from 'unified';
+import type {Heading} from 'mdast';
+
+// TODO as of April 2023, no way to import/re-export this ESM type easily :/
+// TODO upgrade to TS 5.3
+// See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
+// import type {Plugin} from 'unified';
+type Plugin = any; // TODO fix this asap
+
+interface PluginOptions {
+  removeContentTitle?: boolean;
+}
+
+/**
+ * A remark plugin to extract the h1 heading found in Markdown files
+ * This is exposed as "data.contentTitle" to the processed vfile
+ * Also gives the ability to strip that content title (used for the blog plugin)
+ */
+const plugin: Plugin = function plugin(
+  options: PluginOptions = {},
+): Transformer {
+  // content title is
+  const removeContentTitle = options.removeContentTitle ?? false;
+
+  return async (root, vfile) => {
+    const {toString} = await import('mdast-util-to-string');
+    visit(root, 'heading', (headingNode: Heading, index, parent) => {
+      if (headingNode.depth === 1) {
+        vfile.data.contentTitle = toString(headingNode);
+        if (removeContentTitle) {
+          parent!.children.splice(index, 1);
+        }
+        return EXIT; // We only handle the very first heading
+      }
+      // We only handle contentTitle if it's the very first heading found
+      if (headingNode.depth >= 1) {
+        return EXIT;
+      }
+      return undefined;
+    });
+  };
+};
+
+export default plugin;

packages/docusaurus-mdx-loader/src/remark/toc/index.ts

@@ -17,7 +17,7 @@ import type {Heading, Literal} from 'mdast';
 import type {Transformer} from 'unified';
 
 // TODO as of April 2023, no way to import/re-export this ESM type easily :/
-// This might change soon, likely after TS 5.2
+// TODO upgrade to TS 5.3
 // See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
 // import type {Plugin} from 'unified';
 type Plugin = any; // TODO fix this asap