Data collections and references (#6850)

* feat: add generated lookup-map

* feat: wire up fast getEntryBySlug() lookup

* fix: consider frontmatter slugs

* chore: changeset

* chore: lint no-shadow

* fix: revert bad rootRelativePath change

* chore: better var name

* refactor: generated `.json` to in-memory map

* chore: removed unneeded await

Co-authored-by: Bjorn Lu <bjornlu.dev@gmail.com>

* chore: removed unneeded await

Co-authored-by: Bjorn Lu <bjornlu.dev@gmail.com>

* Revert "chore: removed unneeded await"

This reverts commit 1b0a8b0.

* fix: bad `GetEntryImport` type

* chore: remove unused variable

* refactor: for -> Promise.all

* refactor: replace duplicate parseSlug

* refactor: add cache layer

* Revert "refactor: add cache layer"

This reverts commit 1c3bfdc.

* feat: json collection POC

* wip: add test json file

* wip: playing with api ideas

* refactor: extract getCollectionName

* feat: add defineDataCollection

* refactor: variable destructure

* wip: basic data entry pipeline

* chore: revert fixture playing

* wip: basic entry array parser

* feat: basic data type gen

* chore: add with-data playground

* feat: add error when `defineDataCollection()` isn't used

* fix: missing error message

* feat: data collections are here!

* wip: play with data query APIs

* feat: reference() util!

* fix: Markdoc `$entry` variable

* play: add reference util with markdoc

* chore: delete console logs

* feat: `src/data/`!

* feat: reference() errors

* fix: handle hoisted schema parse errors

* fix: reload config and invalid on collection changes

* feat: separate maps for content and data entries

* feat: new `reference()` API that fixes type inference

* feat: support `defineCollection()` for data config

* fix: defineCollection `type` inferenenceπinference

* chore: lock

* feat: getCollection() for everything!

* feat: get full entry access from reference()

* chore: changeset

* wip: type error on acorn?

* chore: lint

* chore: add slugger to data ID processing

* chore: astro/zod -> zod

* chore: example version

* chore: remove slugifier from data id

* chore: remove dead getDataCollection

* chore: remove dead defineDataCollection

* fix: bad collection import

* chore: lock

* feat: add data collections to lookup map

* refactor: stop resolving data from reference

* feat: introduce getEntry and new reference()

* fix: update config loader

* fix: reference() type

* feat: test self references (they work 🎉)

* fix: use `slug` for content references

* fix: bad getEntry content type

* chroe: remove console logs

* fix: strict null checks on with-data

* feat: add getEntries for ref arrays

* chore: fix type hints for reference strings

* chore: change to type never for clarity

* play: try getEntries

* Return to "everything goes in `src/content/`

This reverts commit cc637ec.

* fix: remove old function

* chore: update to AstroErrors

* chore: remove unused fixture files

* play: names

* deps: js-yaml

* feat: data collection YAML with error handling

* refactor: remove console log

* refactor: code cleanup

* fix: allow mixed content to pass through glob imports

* chore: move lookupMap util to virtual-mod

* refactor: new lookupMap logic, better errors

* chore: change MixedContent title

* refactor: remove unneeded try / catch

* fix: use `ws.send` for type gen errors

* fix: bubble `ws.send` errors from astro sync

* refactor: revert verbose astroContentCollectionEntry

* fix: bad with-data package name

* fix: bad virtual mod flag

* chore: remove with-data playground

* test: data collection authors

* test: translations data collection

* fix: add `.yml` support

* refactor: mix in `.yaml` just for fun

* refactor: i18n -> translations

* chore: content-collection-references fixture

* chore: bad lockfile

* fix: bad ContentLookupMap import

* chore: revert back to astroContentCollectionEntry

* test: collection references

* fix: bad error code override

* chore: remove unused asset

* test: sync errors

* chore: remove stray console log

* chore: lock

* chore: revert with-markdoc changes

* chore: doc error states, remove bad merge code

* chore: remove bad `as any`

* chore: lint

* chore: inline ContentLookupMap comments

* chore: settings -> config

* fix: put back `defineCollection()`

* fix: entry.slug for get content collection

* chore: update get-entry-type tests

* docs: totally shorten "missing a `type`"

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

* docs: truncate share a `schema`

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

* chore: add `test:unit` and `test:unit:match`to base

* chore:  update changeset

* refactor: cleanup runtime types and inline comments

* nit: [0] instead of shift()

* refactor: `getRelativeEntryPath()` util

* chore: capitalized Collections for test:match

* nit: ?? viteId on split

* nit: separate Params obj

* chore: add try / catch on readFile

* nit: `const data`

* chore: clean up data collection exceptions

* nit: `?? ''` for search params

* chore: remove TODO on hoisted error

---------

Co-authored-by: Bjorn Lu <bjornlu.dev@gmail.com>
Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

.changeset/early-eyes-bow.md

@@ -0,0 +1,6 @@
+---
+'astro': minor
+'@astrojs/markdoc': minor
+---
+
+Content collections now support data formats including JSON and YAML. You can also create relationships, or references, between collections to pull information from one collection entry into another. Learn more on our [updated Content Collections docs](https://docs.astro.build/en/guides/content-collections/).

package.json

@@ -18,6 +18,8 @@
     "format:imports": "organize-imports-cli ./packages/*/tsconfig.json ./packages/*/*/tsconfig.json",
     "test": "turbo run test --concurrency=1 --filter=astro --filter=create-astro --filter=\"@astrojs/*\"",
     "test:match": "cd packages/astro && pnpm run test:match",
+    "test:unit": "cd packages/astro && pnpm run test:unit",
+    "test:unit:match": "cd packages/astro && pnpm run test:unit:match",
     "test:smoke": "pnpm test:smoke:example && pnpm test:smoke:docs",
     "test:smoke:example": "turbo run build --concurrency=100% --filter=\"@example/*\"",
     "test:smoke:docs": "turbo run build --filter=docs",

packages/astro/package.json

@@ -144,6 +144,7 @@
     "github-slugger": "^2.0.0",
     "gray-matter": "^4.0.3",
     "html-escaper": "^3.0.3",
+    "js-yaml": "^4.1.0",
     "kleur": "^4.1.4",
     "magic-string": "^0.27.0",
     "mime": "^3.0.0",
@@ -181,6 +182,7 @@
     "@types/estree": "^0.0.51",
     "@types/hast": "^2.3.4",
     "@types/html-escaper": "^3.0.0",
+    "@types/js-yaml": "^4.0.5",
     "@types/mime": "^2.0.3",
     "@types/mocha": "^9.1.1",
     "@types/prettier": "^2.6.3",

packages/astro/src/@types/astro.ts

@@ -1250,12 +1250,22 @@ export type ContentEntryModule = {
 	};
 };
 
+export type DataEntryModule = {
+	id: string;
+	collection: string;
+	data: Record<string, unknown>;
+	_internal: {
+		rawData: string;
+		filePath: string;
+	};
+};
+
 export interface ContentEntryType {
 	extensions: string[];
 	getEntryInfo(params: {
 		fileUrl: URL;
 		contents: string;
-	}): GetEntryInfoReturnType | Promise<GetEntryInfoReturnType>;
+	}): GetContentEntryInfoReturnType | Promise<GetContentEntryInfoReturnType>;
 	getRenderModule?(
 		this: rollup.PluginContext,
 		params: {
@@ -1266,7 +1276,7 @@ export interface ContentEntryType {
 	contentModuleTypes?: string;
 }
 
-type GetEntryInfoReturnType = {
+type GetContentEntryInfoReturnType = {
 	data: Record<string, unknown>;
 	/**
 	 * Used for error hints to point to correct line and location
@@ -1278,12 +1288,23 @@ type GetEntryInfoReturnType = {
 	slug: string;
 };
 
+export interface DataEntryType {
+	extensions: string[];
+	getEntryInfo(params: {
+		fileUrl: URL;
+		contents: string;
+	}): GetDataEntryInfoReturnType | Promise<GetDataEntryInfoReturnType>;
+}
+
+export type GetDataEntryInfoReturnType = { data: Record<string, unknown>; rawData?: string };
+
 export interface AstroSettings {
 	config: AstroConfig;
 	adapter: AstroAdapter | undefined;
 	injectedRoutes: InjectedRoute[];
 	pageExtensions: string[];
 	contentEntryTypes: ContentEntryType[];
+	dataEntryTypes: DataEntryType[];
 	renderers: AstroRenderer[];
 	scripts: {
 		stage: InjectedScriptStage;

packages/astro/src/content/consts.ts

@@ -1,5 +1,8 @@
 export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets';
-export const CONTENT_FLAG = 'astroContent';
+export const CONTENT_FLAG = 'astroContentCollectionEntry';
+export const DATA_FLAG = 'astroDataCollectionEntry';
+export const CONTENT_FLAGS = [CONTENT_FLAG, DATA_FLAG, PROPAGATED_ASSET_FLAG] as const;
+
 export const VIRTUAL_MODULE_ID = 'astro:content';
 export const LINKS_PLACEHOLDER = '@@ASTRO-LINKS@@';
 export const STYLES_PLACEHOLDER = '@@ASTRO-STYLES@@';

packages/astro/src/content/runtime-assets.ts

@@ -4,7 +4,7 @@ import type { AstroSettings } from '../@types/astro.js';
 import { emitESMImage } from '../assets/index.js';
 
 export function createImage(
-	settings: AstroSettings,
+	settings: Pick<AstroSettings, 'config'>,
 	pluginContext: PluginContext,
 	entryFilePath: string
 ) {

packages/astro/src/content/runtime.ts

@@ -1,6 +1,6 @@
 import { AstroError, AstroErrorData } from '../core/errors/index.js';
 import { prependForwardSlash } from '../core/path.js';
-
+import { ZodIssueCode, string as zodString, type z } from 'zod';
 import {
 	createComponent,
 	createHeadAndContent,
@@ -9,7 +9,10 @@ import {
 	renderTemplate,
 	renderUniqueStylesheet,
 	unescapeHTML,
+	type AstroComponentFactory,
 } from '../runtime/server/index.js';
+import type { ContentLookupMap } from './utils.js';
+import type { MarkdownHeading } from '@astrojs/markdown-remark';
 
 type LazyImport = () => Promise<any>;
 type GlobResult = Record<string, LazyImport>;
@@ -37,14 +40,31 @@ export function createCollectionToGlobResultMap({
 
 const cacheEntriesByCollection = new Map<string, any[]>();
 export function createGetCollection({
-	collectionToEntryMap,
+	contentCollectionToEntryMap,
+	dataCollectionToEntryMap,
 	getRenderEntryImport,
 }: {
-	collectionToEntryMap: CollectionToEntryMap;
+	contentCollectionToEntryMap: CollectionToEntryMap;
+	dataCollectionToEntryMap: CollectionToEntryMap;
 	getRenderEntryImport: GetEntryImport;
 }) {
 	return async function getCollection(collection: string, filter?: (entry: any) => unknown) {
-		const lazyImports = Object.values(collectionToEntryMap[collection] ?? {});
+		let type: 'content' | 'data';
+		if (collection in contentCollectionToEntryMap) {
+			type = 'content';
+		} else if (collection in dataCollectionToEntryMap) {
+			type = 'data';
+		} else {
+			throw new AstroError({
+				...AstroErrorData.CollectionDoesNotExistError,
+				message: AstroErrorData.CollectionDoesNotExistError.message(collection),
+			});
+		}
+		const lazyImports = Object.values(
+			type === 'content'
+				? contentCollectionToEntryMap[collection]
+				: dataCollectionToEntryMap[collection]
+		);
 		let entries: any[] = [];
 		// Cache `getCollection()` calls in production only
 		// prevents stale cache in development
@@ -54,20 +74,26 @@ export function createGetCollection({
 			entries = await Promise.all(
 				lazyImports.map(async (lazyImport) => {
 					const entry = await lazyImport();
-					return {
-						id: entry.id,
-						slug: entry.slug,
-						body: entry.body,
-						collection: entry.collection,
-						data: entry.data,
-						async render() {
-							return render({
+					return type === 'content'
+						? {
+								id: entry.id,
+								slug: entry.slug,
+								body: entry.body,
 								collection: entry.collection,
+								data: entry.data,
+								async render() {
+									return render({
+										collection: entry.collection,
+										id: entry.id,
+										renderEntryImport: await getRenderEntryImport(collection, entry.slug),
+									});
+								},
+						  }
+						: {
 								id: entry.id,
-								renderEntryImport: await getRenderEntryImport(collection, entry.slug),
-							});
-						},
-					};
+								collection: entry.collection,
+								data: entry.data,
+						  };
 				})
 			);
 			cacheEntriesByCollection.set(collection, entries);
@@ -110,6 +136,121 @@ export function createGetEntryBySlug({
 	};
 }
 
+export function createGetDataEntryById({
+	dataCollectionToEntryMap,
+}: {
+	dataCollectionToEntryMap: CollectionToEntryMap;
+}) {
+	return async function getDataEntryById(collection: string, id: string) {
+		const lazyImport =
+			dataCollectionToEntryMap[collection]?.[/*TODO: filePathToIdMap*/ id + '.json'];
+
+		// TODO: AstroError
+		if (!lazyImport) throw new Error(`Entry ${collection} → ${id} was not found.`);
+		const entry = await lazyImport();
+
+		return {
+			id: entry.id,
+			collection: entry.collection,
+			data: entry.data,
+		};
+	};
+}
+
+type ContentEntryResult = {
+	id: string;
+	slug: string;
+	body: string;
+	collection: string;
+	data: Record<string, any>;
+	render(): Promise<RenderResult>;
+};
+
+type DataEntryResult = {
+	id: string;
+	collection: string;
+	data: Record<string, any>;
+};
+
+type EntryLookupObject = { collection: string; id: string } | { collection: string; slug: string };
+
+export function createGetEntry({
+	getEntryImport,
+	getRenderEntryImport,
+}: {
+	getEntryImport: GetEntryImport;
+	getRenderEntryImport: GetEntryImport;
+}) {
+	return async function getEntry(
+		// Can either pass collection and identifier as 2 positional args,
+		// Or pass a single object with the collection and identifier as properties.
+		// This means the first positional arg can have different shapes.
+		collectionOrLookupObject: string | EntryLookupObject,
+		_lookupId?: string
+	): Promise<ContentEntryResult | DataEntryResult | undefined> {
+		let collection: string, lookupId: string;
+		if (typeof collectionOrLookupObject === 'string') {
+			collection = collectionOrLookupObject;
+			if (!_lookupId)
+				throw new AstroError({
+					...AstroErrorData.UnknownContentCollectionError,
+					message: '`getEntry()` requires an entry identifier as the second argument.',
+				});
+			lookupId = _lookupId;
+		} else {
+			collection = collectionOrLookupObject.collection;
+			// Identifier could be `slug` for content entries, or `id` for data entries
+			lookupId =
+				'id' in collectionOrLookupObject
+					? collectionOrLookupObject.id
+					: collectionOrLookupObject.slug;
+		}
+
+		const entryImport = await getEntryImport(collection, lookupId);
+		if (typeof entryImport !== 'function') return undefined;
+
+		const entry = await entryImport();
+
+		if (entry._internal.type === 'content') {
+			return {
+				id: entry.id,
+				slug: entry.slug,
+				body: entry.body,
+				collection: entry.collection,
+				data: entry.data,
+				async render() {
+					return render({
+						collection: entry.collection,
+						id: entry.id,
+						renderEntryImport: await getRenderEntryImport(collection, lookupId),
+					});
+				},
+			};
+		} else if (entry._internal.type === 'data') {
+			return {
+				id: entry.id,
+				collection: entry.collection,
+				data: entry.data,
+			};
+		}
+		return undefined;
+	};
+}
+
+export function createGetEntries(getEntry: ReturnType<typeof createGetEntry>) {
+	return async function getEntries(
+		entries: { collection: string; id: string }[] | { collection: string; slug: string }[]
+	) {
+		return Promise.all(entries.map((e) => getEntry(e)));
+	};
+}
+
+type RenderResult = {
+	Content: AstroComponentFactory;
+	headings: MarkdownHeading[];
+	remarkPluginFrontmatter: Record<string, any>;
+};
+
 async function render({
 	collection,
 	id,
@@ -118,7 +259,7 @@ async function render({
 	collection: string;
 	id: string;
 	renderEntryImport?: LazyImport;
-}) {
+}): Promise<RenderResult> {
 	const UnexpectedRenderError = new AstroError({
 		...AstroErrorData.UnknownContentCollectionError,
 		message: `Unexpected error while rendering ${String(collection)} → ${String(id)}.`,
@@ -186,3 +327,38 @@ async function render({
 		remarkPluginFrontmatter: mod.frontmatter ?? {},
 	};
 }
+
+export function createReference({ lookupMap }: { lookupMap: ContentLookupMap }) {
+	return function reference(collection: string) {
+		return zodString().transform((lookupId: string, ctx) => {
+			const flattenedErrorPath = ctx.path.join('.');
+			if (!lookupMap[collection]) {
+				ctx.addIssue({
+					code: ZodIssueCode.custom,
+					message: `**${flattenedErrorPath}:** Reference to ${collection} invalid. Collection does not exist or is empty.`,
+				});
+				return;
+			}
+
+			const { type, entries } = lookupMap[collection];
+			const entry = entries[lookupId];
+
+			if (!entry) {
+				ctx.addIssue({
+					code: ZodIssueCode.custom,
+					message: `**${flattenedErrorPath}**: Reference to ${collection} invalid. Expected ${Object.keys(
+						entries
+					)
+						.map((c) => JSON.stringify(c))
+						.join(' | ')}. Received ${JSON.stringify(lookupId)}.`,
+				});
+				return;
+			}
+			// Content is still identified by slugs, so map to a `slug` key for consistency.
+			if (type === 'content') {
+				return { slug: lookupId, collection };
+			}
+			return { id: lookupId, collection };
+		});
+	};
+}