feat: add markdown rendering to content layer

benchmark/bench/memory.js

@@ -18,7 +18,7 @@ export async function run(projectDir, outputFile) {
 	const outputFilePath = fileURLToPath(outputFile);
 
 	console.log('Building and benchmarking...');
-	await execaCommand(`node --expose-gc --max_old_space_size=256 ${astroBin} build`, {
+	await execaCommand(`node --expose-gc --max_old_space_size=10000 ${astroBin} build --silent`, {
 		cwd: root,
 		stdio: 'inherit',
 		env: {

benchmark/make-project/markdown-cc1.js

@@ -0,0 +1,63 @@
+import fs from 'node:fs/promises';
+import { loremIpsumMd } from './_util.js';
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./src/content/blog', projectDir), { recursive: true });
+	await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./src/image.jpg', projectDir));
+
+	const promises = [];
+
+
+	for (let i = 0; i < 10000; i++) {
+		const content = `\
+# Article ${i}
+
+${loremIpsumMd}
+
+![image ${i}](../../image.jpg)
+
+
+`;
+		promises.push(
+			fs.writeFile(new URL(`./src/content/blog/article-${i}.md`, projectDir), content, 'utf-8')
+		);
+	}
+
+
+	await fs.writeFile(
+		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
+		`\
+---
+import { getCollection } from 'astro:content';
+export async function getStaticPaths() {
+  const blogEntries = await getCollection('blog');
+  return blogEntries.map(entry => ({
+    params: { slug: entry.slug }, props: { entry },
+  }));
+}
+const { entry } = Astro.props;
+const { Content } = await entry.render();
+---
+<h1>{entry.data.title}</h1>
+<Content />
+`,
+		'utf-8'
+	);
+
+	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+});`,
+		'utf-8'
+	);
+}

benchmark/make-project/markdown-cc2.js

@@ -0,0 +1,78 @@
+import fs from 'node:fs/promises';
+import { loremIpsumMd } from './_util.js';
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+	await fs.rm(projectDir, { recursive: true, force: true });
+	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./data/blog', projectDir), { recursive: true });
+	await fs.mkdir(new URL('./src/content', projectDir), { recursive: true });
+	await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./image.jpg', projectDir));
+
+	const promises = [];
+
+	for (let i = 0; i < 10000; i++) {
+		const content = `\
+# Article ${i}
+
+${loremIpsumMd}
+
+![image ${i}](../../image.jpg)
+
+`;
+		promises.push(
+			fs.writeFile(new URL(`./data/blog/article-${i}.md`, projectDir), content, 'utf-8')
+		);
+	}
+
+	await fs.writeFile(
+		new URL(`./src/content/config.ts`, projectDir),
+		/*ts */ `
+		import { defineCollection, z } from 'astro:content';
+		import { glob } from 'astro/loaders';
+
+		const blog = defineCollection({
+			type: 'experimental_data',
+			loader: glob({ pattern: '*', base: './data/blog' }),
+		});
+
+		export const collections = { blog }
+
+		`
+	);
+
+	await fs.writeFile(
+		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
+		`\
+---
+import { getCollection } from 'astro:content';
+export async function getStaticPaths() {
+  const blogEntries = await getCollection('blog');
+  return blogEntries.map(entry => ({
+    params: { slug: entry.id }, props: { entry },
+  }));
+}
+const { entry } = Astro.props;
+const { Content } = await entry.render();
+
+---
+<h1>{entry.data.title}</h1>
+<Content />
+`,
+		'utf-8'
+	);
+
+	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+
+export default defineConfig({
+});`,
+		'utf-8'
+	);
+}

benchmark/package.json

@@ -16,6 +16,7 @@
     "markdown-table": "^3.0.3",
     "mri": "^1.2.0",
     "port-authority": "^2.0.1",
-    "pretty-bytes": "^6.1.1"
+    "pretty-bytes": "^6.1.1",
+    "sharp": "^0.33.3"
   }
 }

packages/astro/package.json

@@ -170,6 +170,7 @@
     "magic-string": "^0.30.10",
     "micromatch": "^4.0.7",
     "mrmime": "^2.0.0",
+    "neotraverse": "^0.6.9",
     "ora": "^8.0.1",
     "p-limit": "^6.1.0",
     "p-queue": "^8.0.1",
@@ -187,6 +188,7 @@
     "vite": "^5.3.4",
     "vitefu": "^0.2.5",
     "which-pm": "^3.0.0",
+    "xxhash-wasm": "^1.0.2",
     "yargs-parser": "^21.1.1",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.1",

packages/astro/src/@types/astro.ts

@@ -17,6 +17,7 @@ import type {
 	ActionInputSchema,
 } from '../actions/runtime/virtual/server.js';
 import type { RemotePattern } from '../assets/utils/remotePattern.js';
+import type { DataEntry, RenderedContent } from '../content/data-store.js';
 import type { AssetsPrefix, SSRManifest, SerializedSSRManifest } from '../core/app/types.js';
 import type { PageBuildData } from '../core/build/types.js';
 import type { AstroConfigType } from '../core/config/index.js';
@@ -2377,6 +2378,8 @@ export type DataEntryModule = {
 	};
 };
 
+export type ContentEntryRenderFuction = (entry: DataEntry) => Promise<RenderedContent>;
+
 export interface ContentEntryType {
 	extensions: string[];
 	getEntryInfo(params: {
@@ -2392,6 +2395,8 @@ export interface ContentEntryType {
 		}
 	): rollup.LoadResult | Promise<rollup.LoadResult>;
 	contentModuleTypes?: string;
+	getRenderFunction?(settings: AstroSettings): Promise<ContentEntryRenderFuction>;
+
 	/**
 	 * Handle asset propagation for rendered content to avoid bleed.
 	 * Ex. MDX content can import styles and scripts, so `handlePropagation` should be true.

packages/astro/src/assets/utils/resolveImports.ts

@@ -0,0 +1,40 @@
+import { isRemotePath, removeBase } from '@astrojs/internal-helpers/path';
+import { CONTENT_IMAGE_FLAG, IMAGE_IMPORT_PREFIX } from '../../content/consts.js';
+import { shorthash } from '../../runtime/server/shorthash.js';
+import { VALID_INPUT_FORMATS } from '../consts.js';
+
+/**
+ * Resolves an image src from a content file (such as markdown) to a module ID or import that can be resolved by Vite.
+ *
+ * @param imageSrc The src attribute of an image tag
+ * @param filePath The path to the file that contains the imagem relative to the site root
+ * @returns A module id of the image that can be rsolved by Vite, or undefined if it is not a local image
+ */
+export function imageSrcToImportId(imageSrc: string, filePath: string): string | undefined {
+	// If the import is coming from the data store it will have a special prefix to identify it
+	// as an image import. We remove this prefix so that we can resolve the image correctly.
+	imageSrc = removeBase(imageSrc, IMAGE_IMPORT_PREFIX);
+
+	// We only care about local imports
+	if (isRemotePath(imageSrc) || imageSrc.startsWith('/')) {
+		return;
+	}
+	// We only care about images
+	const ext = imageSrc.split('.').at(-1) as (typeof VALID_INPUT_FORMATS)[number] | undefined;
+	if (!ext || !VALID_INPUT_FORMATS.includes(ext)) {
+		return;
+	}
+
+	// The import paths are relative to the content (md) file, but when it's actually resolved it will
+	// be in a single assets file, so relative paths will no longer work. To deal with this we use
+	// a query parameter to store the original path to the file and append a query param flag.
+	// This allows our Vite plugin to intercept the import and resolve the path relative to the
+	// importer and get the correct full path for the imported image.
+
+	const params = new URLSearchParams(CONTENT_IMAGE_FLAG);
+	params.set('importer', filePath);
+	return `${imageSrc}?${params.toString()}`;
+}
+
+export const importIdToSymbolName = (importId: string) =>
+	`__ASTRO_IMAGE_IMPORT_${shorthash(importId)}`;

packages/astro/src/content/consts.ts

@@ -2,22 +2,28 @@ export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets';
 export const CONTENT_RENDER_FLAG = 'astroRenderContent';
 export const CONTENT_FLAG = 'astroContentCollectionEntry';
 export const DATA_FLAG = 'astroDataCollectionEntry';
+export const CONTENT_IMAGE_FLAG = 'astroContentImageFlag';
 
 export const VIRTUAL_MODULE_ID = 'astro:content';
 export const RESOLVED_VIRTUAL_MODULE_ID = '\0' + VIRTUAL_MODULE_ID;
 export const DATA_STORE_VIRTUAL_ID = 'astro:data-layer-content';
 export const RESOLVED_DATA_STORE_VIRTUAL_ID = '\0' + DATA_STORE_VIRTUAL_ID;
+export const ASSET_IMPORTS_VIRTUAL_ID = 'astro:asset-imports';
+export const ASSET_IMPORTS_RESOLVED_STUB_ID = '\0' + ASSET_IMPORTS_VIRTUAL_ID;
 export const LINKS_PLACEHOLDER = '@@ASTRO-LINKS@@';
 export const STYLES_PLACEHOLDER = '@@ASTRO-STYLES@@';
 export const SCRIPTS_PLACEHOLDER = '@@ASTRO-SCRIPTS@@';
+export const IMAGE_IMPORT_PREFIX = '__ASTRO_IMAGE_';
 
 export const CONTENT_FLAGS = [
 	CONTENT_FLAG,
 	CONTENT_RENDER_FLAG,
 	DATA_FLAG,
 	PROPAGATED_ASSET_FLAG,
+	CONTENT_IMAGE_FLAG,
 ] as const;
 
 export const CONTENT_TYPES_FILE = 'types.d.ts';
 
 export const DATA_STORE_FILE = 'data-store.json';
+export const ASSET_IMPORTS_FILE = 'assets.mjs';