fix(mdx-loader): get correct error line numbers, handle front matter …

…+ contentTitle with remark (#9386)

packages/docusaurus-mdx-loader/package.json

@@ -35,13 +35,15 @@
  "mdast-util-to-string": "^3.2.0",
  "rehype-raw": "^6.1.1",
  "remark-directive": "^2.0.1",
+ "remark-frontmatter": "^5.0.0",
  "remark-emoji": "^2.2.0",
  "remark-gfm": "^3.0.1",
  "stringify-object": "^3.3.0",
  "tslib": "^2.6.0",
  "unified": "^10.1.2",
  "unist-util-visit": "^2.0.3",
  "url-loader": "^4.1.1",
+ "vfile": "^5.3.7",
  "webpack": "^5.88.1"
  },
  "devDependencies": {

packages/docusaurus-mdx-loader/src/loader.ts

@@ -9,7 +9,6 @@ import fs from 'fs-extra';
 import logger from '@docusaurus/logger';
 import {
  parseFrontMatter,
- parseMarkdownContentTitle,
  escapePath,
  getFileLoaderUtils,
 } from '@docusaurus/utils';
@@ -122,6 +121,15 @@ function ensureMarkdownConfig(reqOptions: Options) {
  }
 }
 
+/**
+ * data.contentTitle is set by the remark contentTitle plugin
+ */
+function extractContentTitleData(data: {
+ [key: string]: unknown;
+}): string | undefined {
+ return data.contentTitle as string | undefined;
+}
+
 export async function mdxLoader(
  this: LoaderContext<Options>,
  fileString: string,
@@ -132,18 +140,11 @@ export async function mdxLoader(
  const {query} = this;
  ensureMarkdownConfig(reqOptions);
 
- const {frontMatter, content: contentWithTitle} = parseFrontMatter(fileString);
+ const {frontMatter} = parseFrontMatter(fileString);
  const mdxFrontMatter = validateMDXFrontMatter(frontMatter.mdx);
 
- const {content: contentUnprocessed, contentTitle} = parseMarkdownContentTitle(
- contentWithTitle,
- {
- removeContentTitle: reqOptions.removeContentTitle,
- },
- );
-
- const content = preprocessor({
- fileContent: contentUnprocessed,
+ const preprocessedContent = preprocessor({
+ fileContent: fileString,
  filePath,
  admonitions: reqOptions.admonitions,
  markdownConfig: reqOptions.markdownConfig,
@@ -158,9 +159,13 @@ export async function mdxLoader(
  mdxFrontMatter,
  });
 
- let result: string;
+ let result: {content: string; data: {[key: string]: unknown}};
  try {
- result = await processor.process({content, filePath});
+ result = await processor.process({
+ content: preprocessedContent,
+ filePath,
+ frontMatter,
+ });
  } catch (errorUnknown) {
  const error = errorUnknown as Error;
 
@@ -184,6 +189,8 @@ export async function mdxLoader(
  );
  }
 
+ const contentTitle = extractContentTitleData(result.data);
+
  // MDX partials are MDX files starting with _ or in a folder starting with _
  // Partial are not expected to have associated metadata files or front matter
  const isMDXPartial = reqOptions.isMDXPartial?.(filePath);
@@ -244,7 +251,7 @@ ${assets ? `export const assets = ${createAssetsExportCode(assets)};` : ''}
 
  const code = `
 ${exportsCode}
-${result}
+${result.content}
 `;
 
  return callback(null, code);

packages/docusaurus-mdx-loader/src/processor.ts

@@ -7,6 +7,7 @@
 
 import emoji from 'remark-emoji';
 import headings from './remark/headings';
+import contentTitle from './remark/contentTitle';
 import toc from './remark/toc';
 import transformImage from './remark/transformImage';
 import transformLinks from './remark/transformLinks';
@@ -28,15 +29,19 @@ import type {ProcessorOptions} from '@mdx-js/mdx';
 // See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
 type Pluggable = any; // TODO fix this asap
 
+type SimpleProcessorResult = {content: string; data: {[key: string]: unknown}};
+
 // TODO alt interface because impossible to import type Processor (ESM + TS :/)
 type SimpleProcessor = {
  process: ({
  content,
  filePath,
+ frontMatter,
  }: {
  content: string;
  filePath: string;
- }) => Promise<string>;
+ frontMatter: {[key: string]: unknown};
+ }) => Promise<SimpleProcessorResult>;
 };
 
 const DEFAULT_OPTIONS: MDXOptions = {
@@ -74,11 +79,13 @@ function getAdmonitionsPlugins(
 // Need to be async due to ESM dynamic imports...
 async function createProcessorFactory() {
  const {createProcessor: createMdxProcessor} = await import('@mdx-js/mdx');
+ const {default: frontmatter} = await import('remark-frontmatter');
  const {default: rehypeRaw} = await import('rehype-raw');
  const {default: gfm} = await import('remark-gfm');
  // TODO using fork until PR merged: https://github.com/leebyron/remark-comment/pull/3
  const {default: comment} = await import('@slorber/remark-comment');
  const {default: directive} = await import('remark-directive');
+ const {VFile} = await import('vfile');
 
  // /!\ this method is synchronous on purpose
  // Using async code here can create cache entry race conditions!
@@ -91,7 +98,9 @@ async function createProcessorFactory() {
  }): SimpleProcessor {
  const remarkPlugins: MDXPlugin[] = [
  ...(options.beforeDefaultRemarkPlugins ?? []),
+ frontmatter,
  directive,
+ [contentTitle, {removeContentTitle: options.removeContentTitle}],
  ...getAdmonitionsPlugins(options.admonitions ?? false),
  ...DEFAULT_OPTIONS.remarkPlugins,
  details,
@@ -158,13 +167,19 @@ async function createProcessorFactory() {
  });
 
  return {
- process: async ({content, filePath}) =>
- mdxProcessor
- .process({
- value: content,
- path: filePath,
- })
- .then((res) => res.toString()),
+ process: async ({content, filePath, frontMatter}) => {
+ const vfile = new VFile({
+ value: content,
+ path: filePath,
+ data: {
+ frontMatter,
+ },
+ });
+ return mdxProcessor.process(vfile).then((result) => ({
+ content: result.toString(),
+ data: result.data,
+ }));
+ },
  };
  }
 

packages/docusaurus-mdx-loader/src/remark/contentTitle/__tests__/index.test.ts

@@ -0,0 +1,151 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import plugin from '../index';
+
+async function process(
+ content: string,
+ options: {removeContentTitle?: boolean} = {},
+) {
+ const {remark} = await import('remark');
+ const processor = await remark().use({plugins: [[plugin, options]]});
+ return processor.process(content);
+}
+
+describe('contentTitle remark plugin', () => {
+ describe('extracts data.contentTitle', () => {
+ it('extracts h1 heading', async () => {
+ const result = await process(`
+# contentTitle 1
+
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+ `);
+
+ expect(result.data.contentTitle).toBe('contentTitle 1');
+ });
+
+ it('extracts h1 heading alt syntax', async () => {
+ const result = await process(`
+contentTitle alt
+===
+
+# contentTitle 1
+
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+ `);
+
+ expect(result.data.contentTitle).toBe('contentTitle alt');
+ });
+
+ it('works with no contentTitle', async () => {
+ const result = await process(`
+## Heading Two {#custom-heading-two}
+
+some **markdown** *content*
+ `);
+
+ expect(result.data.contentTitle).toBeUndefined();
+ });
+
+ it('ignore contentTitle if not in first position', async () => {
+ const result = await process(`
+## Heading Two {#custom-heading-two}
+
+# contentTitle 1
+
+some **markdown** *content*
+ `);
+
+ expect(result.data.contentTitle).toBeUndefined();
+ });
+
+ it('is able to decently serialize Markdown syntax', async () => {
+ const result = await process(`
+# some **markdown** \`content\` _italic_
+
+some **markdown** *content*
+ `);
+
+ expect(result.data.contentTitle).toBe('some markdown content italic');
+ });
+ });
+
+ describe('returns appropriate content', () => {
+ it('returns content unmodified', async () => {
+ const content = `
+# contentTitle 1
+
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+`.trim();
+
+ const result = await process(content);
+
+ expect(result.toString().trim()).toEqual(content);
+ });
+
+ it('can strip contentTitle', async () => {
+ const content = `
+# contentTitle 1
+
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+`.trim();
+
+ const result = await process(content, {removeContentTitle: true});
+
+ expect(result.toString().trim()).toEqual(
+ `
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+`.trim(),
+ );
+ });
+
+ it('can strip contentTitle alt', async () => {
+ const content = `
+contentTitle alt
+===
+
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+`.trim();
+
+ const result = await process(content, {removeContentTitle: true});
+
+ expect(result.toString().trim()).toEqual(
+ `
+## Heading Two {#custom-heading-two}
+
+# contentTitle 2
+
+some **markdown** *content*
+`.trim(),
+ );
+ });
+ });
+});

packages/docusaurus-mdx-loader/src/remark/contentTitle/index.ts

@@ -0,0 +1,53 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import visit, {EXIT} from 'unist-util-visit';
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+import type {Transformer} from 'unified';
+import type {Heading} from 'mdast';
+
+// TODO as of April 2023, no way to import/re-export this ESM type easily :/
+// TODO upgrade to TS 5.3
+// See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
+// import type {Plugin} from 'unified';
+type Plugin = any; // TODO fix this asap
+
+interface PluginOptions {
+ removeContentTitle?: boolean;
+}
+
+/**
+ * A remark plugin to extract the h1 heading found in Markdown files
+ * This is exposed as "data.contentTitle" to the processed vfile
+ * Also gives the ability to strip that content title (used for the blog plugin)
+ */
+const plugin: Plugin = function plugin(
+ options: PluginOptions = {},
+): Transformer {
+ // content title is
+ const removeContentTitle = options.removeContentTitle ?? false;
+
+ return async (root, vfile) => {
+ const {toString} = await import('mdast-util-to-string');
+ visit(root, 'heading', (headingNode: Heading, index, parent) => {
+ if (headingNode.depth === 1) {
+ vfile.data.contentTitle = toString(headingNode);
+ if (removeContentTitle) {
+ parent!.children.splice(index, 1);
+ }
+ return EXIT; // We only handle the very first heading
+ }
+ // We only handle contentTitle if it's the very first heading found
+ if (headingNode.depth >= 1) {
+ return EXIT;
+ }
+ return undefined;
+ });
+ };
+};
+
+export default plugin;

packages/docusaurus-mdx-loader/src/remark/toc/index.ts

@@ -17,7 +17,7 @@ import type {Heading, Literal} from 'mdast';
 import type {Transformer} from 'unified';
 
 // TODO as of April 2023, no way to import/re-export this ESM type easily :/
-// This might change soon, likely after TS 5.2
+// TODO upgrade to TS 5.3
 // See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
 // import type {Plugin} from 'unified';
 type Plugin = any; // TODO fix this asap