Parse frontmatter ourselves (#12075)

.changeset/sweet-timers-smash.md

@@ -0,0 +1,8 @@
+---
+'@astrojs/markdoc': patch
+'@astrojs/mdx': patch
+'@astrojs/markdown-remark': patch
+'astro': patch
+---
+
+Parses frontmatter ourselves

packages/astro/package.json

@@ -152,7 +152,6 @@
     "fastq": "^1.17.1",
     "flattie": "^1.1.1",
     "github-slugger": "^2.0.0",
-    "gray-matter": "^4.0.3",
     "html-escaper": "^3.0.3",
     "http-cache-semantics": "^4.1.1",
     "js-yaml": "^4.1.0",

packages/astro/src/content/utils.ts

@@ -1,8 +1,8 @@
 import fsMod from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
+import { parseFrontmatter } from '@astrojs/markdown-remark';
 import { slug as githubSlug } from 'github-slugger';
-import matter from 'gray-matter';
 import type { PluginContext } from 'rollup';
 import type { ViteDevServer } from 'vite';
 import xxhash from 'xxhash-wasm';
@@ -455,7 +455,7 @@ function getYAMLErrorLine(rawData: string | undefined, objectKey: string) {
 
 export function safeParseFrontmatter(source: string, id?: string) {
 	try {
-		return matter(source);
+		return parseFrontmatter(source, { frontmatter: 'empty-with-spaces' });
 	} catch (err: any) {
 		const markdownError = new MarkdownError({
 			name: 'MarkdownError',

packages/astro/src/vite-plugin-markdown/content-entry-type.ts

@@ -8,10 +8,10 @@ export const markdownContentEntryType: ContentEntryType = {
 	async getEntryInfo({ contents, fileUrl }: { contents: string; fileUrl: URL }) {
 		const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl));
 		return {
-			data: parsed.data,
-			body: parsed.content,
-			slug: parsed.data.slug,
-			rawData: parsed.matter,
+			data: parsed.frontmatter,
+			body: parsed.content.trim(),
+			slug: parsed.frontmatter.slug,
+			rawData: parsed.rawFrontmatter,
 		};
 	},
 	// We need to handle propagation for Markdown because they support layouts which will bring in styles.

packages/astro/src/vite-plugin-markdown/index.ts

@@ -66,7 +66,7 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug
 				const renderResult = await (await processor).render(raw.content, {
 					// @ts-expect-error passing internal prop
 					fileURL,
-					frontmatter: raw.data,
+					frontmatter: raw.frontmatter,
 				});
 
 				// Improve error message for invalid astro frontmatter

packages/integrations/markdoc/package.json

@@ -68,7 +68,6 @@
     "@markdoc/markdoc": "^0.4.0",
     "esbuild": "^0.21.5",
     "github-slugger": "^2.0.0",
-    "gray-matter": "^4.0.3",
     "htmlparser2": "^9.1.0"
   },
   "peerDependencies": {

packages/integrations/markdoc/src/content-entry-type.ts

@@ -1,11 +1,11 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
+import { parseFrontmatter } from '@astrojs/markdown-remark';
 import type { Config as MarkdocConfig, Node } from '@markdoc/markdoc';
 import Markdoc from '@markdoc/markdoc';
 import type { AstroConfig, ContentEntryType } from 'astro';
 import { emitESMImage } from 'astro/assets/utils';
-import matter from 'gray-matter';
 import type { Rollup, ErrorPayload as ViteErrorPayload } from 'vite';
 import type { ComponentConfig } from './config.js';
 import { htmlTokenTransform } from './html/transform/html-token-transform.js';
@@ -26,12 +26,20 @@ export async function getContentEntryType({
 }): Promise<ContentEntryType> {
 	return {
 		extensions: ['.mdoc'],
-		getEntryInfo,
+		getEntryInfo({ fileUrl, contents }) {
+			const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl));
+			return {
+				data: parsed.frontmatter,
+				body: parsed.content.trim(),
+				slug: parsed.frontmatter.slug,
+				rawData: parsed.rawFrontmatter,
+			};
+		},
 		handlePropagation: true,
 		async getRenderModule({ contents, fileUrl, viteId }) {
-			const entry = getEntryInfo({ contents, fileUrl });
+			const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl));
 			const tokenizer = getMarkdocTokenizer(options);
-			let tokens = tokenizer.tokenize(entry.body);
+			let tokens = tokenizer.tokenize(parsed.content);
 
 			if (options?.allowHTML) {
 				tokens = htmlTokenTransform(tokenizer, tokens);
@@ -47,7 +55,6 @@ export async function getContentEntryType({
 				ast,
 				/* Raised generics issue with Markdoc core https://github.com/markdoc/markdoc/discussions/400 */
 				markdocConfig: markdocConfig as MarkdocConfig,
-				entry,
 				viteId,
 				astroConfig,
 				filePath,
@@ -64,7 +71,6 @@ export async function getContentEntryType({
 					raiseValidationErrors({
 						ast: partialAst,
 						markdocConfig: markdocConfig as MarkdocConfig,
-						entry,
 						viteId,
 						astroConfig,
 						filePath: partialPath,
@@ -224,14 +230,12 @@ async function resolvePartials({
 function raiseValidationErrors({
 	ast,
 	markdocConfig,
-	entry,
 	viteId,
 	astroConfig,
 	filePath,
 }: {
 	ast: Node;
 	markdocConfig: MarkdocConfig;
-	entry: ReturnType<typeof getEntryInfo>;
 	viteId: string;
 	astroConfig: AstroConfig;
 	filePath: string;
@@ -250,8 +254,6 @@ function raiseValidationErrors({
 	});
 
 	if (validationErrors.length) {
-		// Heuristic: take number of newlines for `rawData` and add 2 for the `---` fences
-		const frontmatterBlockOffset = entry.rawData.split('\n').length + 2;
 		const rootRelativePath = path.relative(fileURLToPath(astroConfig.root), filePath);
 		throw new MarkdocError({
 			message: [
@@ -261,7 +263,7 @@ function raiseValidationErrors({
 			location: {
 				// Error overlay does not support multi-line or ranges.
 				// Just point to the first line.
-				line: frontmatterBlockOffset + validationErrors[0].lines[0],
+				line: validationErrors[0].lines[0],
 				file: viteId,
 			},
 		});
@@ -282,16 +284,6 @@ function getUsedTags(markdocAst: Node) {
 	return tags;
 }
 
-function getEntryInfo({ fileUrl, contents }: { fileUrl: URL; contents: string }) {
-	const parsed = parseFrontmatter(contents, fileURLToPath(fileUrl));
-	return {
-		data: parsed.data,
-		body: parsed.content,
-		slug: parsed.data.slug,
-		rawData: parsed.matter,
-	};
-}
-
 /**
  * Emits optimized images, and appends the generated `src` to each AST node
  * via the `__optimizedSrc` attribute.
@@ -410,12 +402,11 @@ function getStringifiedMap(
  * Match YAML exception handling from Astro core errors
  * @see 'astro/src/core/errors.ts'
  */
-function parseFrontmatter(fileContents: string, filePath: string) {
+function safeParseFrontmatter(fileContents: string, filePath: string) {
 	try {
-		// `matter` is empty string on cache results
-		// clear cache to prevent this
-		(matter as any).clearCache();
-		return matter(fileContents);
+		// empty with lines to preserve sourcemap location, but not `empty-with-spaces`
+		// because markdoc struggles with spaces
+		return parseFrontmatter(fileContents, { frontmatter: 'empty-with-lines' });
 	} catch (e: any) {
 		if (e.name === 'YAMLException') {
 			const err: Error & ViteErrorPayload['err'] = e;

packages/integrations/markdoc/test/content-collections.test.js

@@ -83,7 +83,7 @@ const post1Entry = {
 		schemaWorks: true,
 		title: 'Post 1',
 	},
-	body: '\n## Post 1\n\nThis is the contents of post 1.\n',
+	body: '## Post 1\n\nThis is the contents of post 1.',
 };
 
 const post2Entry = {
@@ -94,7 +94,7 @@ const post2Entry = {
 		schemaWorks: true,
 		title: 'Post 2',
 	},
-	body: '\n## Post 2\n\nThis is the contents of post 2.\n',
+	body: '## Post 2\n\nThis is the contents of post 2.',
 };
 
 const post3Entry = {
@@ -105,5 +105,5 @@ const post3Entry = {
 		schemaWorks: true,
 		title: 'Post 3',
 	},
-	body: '\n## Post 3\n\nThis is the contents of post 3.\n',
+	body: '## Post 3\n\nThis is the contents of post 3.',
 };

packages/integrations/mdx/package.json

@@ -39,7 +39,6 @@
     "acorn": "^8.12.1",
     "es-module-lexer": "^1.5.4",
     "estree-util-visit": "^2.0.0",
-    "gray-matter": "^4.0.3",
     "hast-util-to-html": "^9.0.2",
     "kleur": "^4.1.5",
     "rehype-raw": "^7.0.0",

packages/integrations/mdx/src/index.ts

@@ -11,7 +11,7 @@ import type {
 import type { Options as RemarkRehypeOptions } from 'remark-rehype';
 import type { PluggableList } from 'unified';
 import type { OptimizeOptions } from './rehype-optimize-static.js';
-import { ignoreStringPlugins, parseFrontmatter } from './utils.js';
+import { ignoreStringPlugins, safeParseFrontmatter } from './utils.js';
 import { vitePluginMdxPostprocess } from './vite-plugin-mdx-postprocess.js';
 import { vitePluginMdx } from './vite-plugin-mdx.js';
 
@@ -60,12 +60,12 @@ export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroI
 				addContentEntryType({
 					extensions: ['.mdx'],
 					async getEntryInfo({ fileUrl, contents }: { fileUrl: URL; contents: string }) {
-						const parsed = parseFrontmatter(contents, fileURLToPath(fileUrl));
+						const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl));
 						return {
-							data: parsed.data,
-							body: parsed.content,
-							slug: parsed.data.slug,
-							rawData: parsed.matter,
+							data: parsed.frontmatter,
+							body: parsed.content.trim(),
+							slug: parsed.frontmatter.slug,
+							rawData: parsed.rawFrontmatter,
 						};
 					},
 					contentModuleTypes: await fs.readFile(

packages/integrations/mdx/src/utils.ts

@@ -1,7 +1,7 @@
+import { parseFrontmatter } from '@astrojs/markdown-remark';
 import type { Options as AcornOpts } from 'acorn';
 import { parse } from 'acorn';
 import type { AstroConfig, AstroIntegrationLogger, SSRError } from 'astro';
-import matter from 'gray-matter';
 import { bold } from 'kleur/colors';
 import type { MdxjsEsm } from 'mdast-util-mdx';
 import type { PluggableList } from 'unified';
@@ -48,9 +48,9 @@ export function getFileInfo(id: string, config: AstroConfig): FileInfo {
  * Match YAML exception handling from Astro core errors
  * @see 'astro/src/core/errors.ts'
  */
-export function parseFrontmatter(code: string, id: string) {
+export function safeParseFrontmatter(code: string, id: string) {
 	try {
-		return matter(code);
+		return parseFrontmatter(code, { frontmatter: 'empty-with-spaces' });
 	} catch (e: any) {
 		if (e.name === 'YAMLException') {
 			const err: SSRError = e;

packages/integrations/mdx/src/vite-plugin-mdx.ts

@@ -4,7 +4,7 @@ import { VFile } from 'vfile';
 import type { Plugin } from 'vite';
 import type { MdxOptions } from './index.js';
 import { createMdxProcessor } from './plugins.js';
-import { parseFrontmatter } from './utils.js';
+import { safeParseFrontmatter } from './utils.js';
 
 export function vitePluginMdx(mdxOptions: MdxOptions): Plugin {
 	let processor: ReturnType<typeof createMdxProcessor> | undefined;
@@ -38,11 +38,10 @@ export function vitePluginMdx(mdxOptions: MdxOptions): Plugin {
 		async transform(code, id) {
 			if (!id.endsWith('.mdx')) return;
 
-			const { data: frontmatter, content: pageContent, matter } = parseFrontmatter(code, id);
-			const frontmatterLines = matter ? matter.match(/\n/g)?.join('') + '\n\n' : '';
+			const { frontmatter, content } = safeParseFrontmatter(code, id);
 
 			const vfile = new VFile({
-				value: frontmatterLines + pageContent,
+				value: content,
 				path: id,
 				data: {
 					astro: {

packages/markdown/remark/package.json

@@ -37,6 +37,7 @@
     "hast-util-from-html": "^2.0.2",
     "hast-util-to-text": "^4.0.2",
     "import-meta-resolve": "^4.1.0",
+    "js-yaml": "^4.1.0",
     "mdast-util-definitions": "^6.0.0",
     "rehype-raw": "^7.0.0",
     "rehype-stringify": "^10.0.0",
@@ -54,6 +55,7 @@
   "devDependencies": {
     "@types/estree": "^1.0.5",
     "@types/hast": "^3.0.4",
+    "@types/js-yaml": "^4.0.9",
     "@types/mdast": "^4.0.4",
     "@types/unist": "^3.0.3",
     "astro-scripts": "workspace:*",

packages/markdown/remark/src/frontmatter.ts

@@ -1,3 +1,5 @@
+import yaml from 'js-yaml';
+
 export function isFrontmatterValid(frontmatter: Record<string, any>) {
 	try {
 		// ensure frontmatter is JSON-serializable
@@ -7,3 +9,66 @@ export function isFrontmatterValid(frontmatter: Record<string, any>) {
 	}
 	return typeof frontmatter === 'object' && frontmatter !== null;
 }
+
+const frontmatterRE = /^---(.*?)^---/ms;
+export function extractFrontmatter(code: string): string | undefined {
+	return frontmatterRE.exec(code)?.[1];
+}
+
+export interface ParseFrontmatterOptions {
+	/**
+	 * How the frontmatter should be handled in the returned `content` string.
+	 * - `preserve`: Keep the frontmatter.
+	 * - `remove`: Remove the frontmatter.
+	 * - `empty-with-spaces`: Replace the frontmatter with empty spaces. (preserves sourcemap line/col/offset)
+	 * - `empty-with-lines`: Replace the frontmatter with empty line breaks. (preserves sourcemap line/col)
+	 *
+	 * @default 'remove'
+	 */
+	frontmatter: 'preserve' | 'remove' | 'empty-with-spaces' | 'empty-with-lines';
+}
+
+export interface ParseFrontmatterResult {
+	frontmatter: Record<string, any>;
+	rawFrontmatter: string;
+	content: string;
+}
+
+export function parseFrontmatter(
+	code: string,
+	options?: ParseFrontmatterOptions,
+): ParseFrontmatterResult {
+	const rawFrontmatter = extractFrontmatter(code);
+
+	if (rawFrontmatter == null) {
+		return { frontmatter: {}, rawFrontmatter: '', content: code };
+	}
+
+	const parsed = yaml.load(rawFrontmatter);
+	const frontmatter = (parsed && typeof parsed === 'object' ? parsed : {}) as Record<string, any>;
+
+	let content: string;
+	switch (options?.frontmatter ?? 'remove') {
+		case 'preserve':
+			content = code;
+			break;
+		case 'remove':
+			content = code.replace(`---${rawFrontmatter}---`, '');
+			break;
+		case 'empty-with-spaces':
+			content = code.replace(
+				`---${rawFrontmatter}---`,
+				`   ${rawFrontmatter.replace(/[^\r\n]/g, ' ')}   `,
+			);
+			break;
+		case 'empty-with-lines':
+			content = code.replace(`---${rawFrontmatter}---`, rawFrontmatter.replace(/[^\r\n]/g, ''));
+			break;
+	}
+
+	return {
+		frontmatter,
+		rawFrontmatter,
+		content,
+	};
+}

packages/markdown/remark/src/index.ts

@@ -20,7 +20,13 @@ export { rehypeHeadingIds } from './rehype-collect-headings.js';
 export { remarkCollectImages } from './remark-collect-images.js';
 export { rehypePrism } from './rehype-prism.js';
 export { rehypeShiki } from './rehype-shiki.js';
-export { isFrontmatterValid } from './frontmatter.js';
+export {
+	isFrontmatterValid,
+	extractFrontmatter,
+	parseFrontmatter,
+	type ParseFrontmatterOptions,
+	type ParseFrontmatterResult,
+} from './frontmatter.js';
 export {
 	createShikiHighlighter,
 	type ShikiHighlighter,