feat(content-layer): support references in content layer

packages/astro/src/content/runtime.ts

@@ -1,7 +1,7 @@
 import type { MarkdownHeading } from '@astrojs/markdown-remark';
 import { Traverse } from 'neotraverse/modern';
 import pLimit from 'p-limit';
-import { ZodIssueCode, string as zodString } from 'zod';
+import { ZodIssueCode, z } from 'zod';
 import type { GetImageResult, ImageMetadata } from '../@types/astro.js';
 import { imageSrcToImportId } from '../assets/utils/resolveImports.js';
 import { AstroError, AstroErrorData } from '../core/errors/index.js';
@@ -147,9 +147,11 @@ export function createGetCollection({
 export function createGetEntryBySlug({
 	getEntryImport,
 	getRenderEntryImport,
+	collectionNames,
 }: {
 	getEntryImport: GetEntryImport;
 	getRenderEntryImport: GetEntryImport;
+	collectionNames: Set<string>;
 }) {
 	return async function getEntryBySlug(collection: string, slug: string) {
 		const store = await globalDataStore.get();
@@ -168,6 +170,11 @@ export function createGetEntryBySlug({
 				render: () => renderEntry(entry),
 			};
 		}
+		if (!collectionNames.has(collection)) {
+			// eslint-disable-next-line no-console
+			console.warn(`The collection ${JSON.stringify(collection)} does not exist.`);
+			return undefined;
+		}
 
 		const entryImport = await getEntryImport(collection, slug);
 		if (typeof entryImport !== 'function') return undefined;
@@ -191,7 +198,10 @@ export function createGetEntryBySlug({
 	};
 }
 
-export function createGetDataEntryById({ getEntryImport }: { getEntryImport: GetEntryImport }) {
+export function createGetDataEntryById({
+	getEntryImport,
+	collectionNames,
+}: { getEntryImport: GetEntryImport; collectionNames: Set<string> }) {
 	return async function getDataEntryById(collection: string, id: string) {
 		const store = await globalDataStore.get();
 
@@ -208,6 +218,11 @@ export function createGetDataEntryById({ getEntryImport }: { getEntryImport: Get
 				collection,
 			};
 		}
+		if (!collectionNames.has(collection)) {
+			// eslint-disable-next-line no-console
+			console.warn(`The collection ${JSON.stringify(collection)} does not exist.`);
+			return undefined;
+		}
 
 		const lazyImport = await getEntryImport(collection, id);
 
@@ -243,9 +258,11 @@ type EntryLookupObject = { collection: string; id: string } | { collection: stri
 export function createGetEntry({
 	getEntryImport,
 	getRenderEntryImport,
+	collectionNames,
 }: {
 	getEntryImport: GetEntryImport;
 	getRenderEntryImport: GetEntryImport;
+	collectionNames: Set<string>;
 }) {
 	return async function getEntry(
 		// Can either pass collection and identifier as 2 positional args,
@@ -277,7 +294,9 @@ export function createGetEntry({
 		if (store.hasCollection(collection)) {
 			const entry = store.get<DataEntry>(collection, lookupId);
 			if (!entry) {
-				throw new Error(`Entry ${collection} → ${lookupId} was not found.`);
+				// eslint-disable-next-line no-console
+				console.warn(`Entry ${collection} → ${lookupId} was not found.`);
+				return;
 			}
 
 			if (entry.filePath) {
@@ -292,6 +311,12 @@ export function createGetEntry({
 			} as DataEntryResult | ContentEntryResult;
 		}
 
+		if (!collectionNames.has(collection)) {
+			// eslint-disable-next-line no-console
+			console.warn(`The collection ${JSON.stringify(collection)} does not exist.`);
+			return undefined;
+		}
+
 		const entryImport = await getEntryImport(collection, lookupId);
 		if (typeof entryImport !== 'function') return undefined;
 
@@ -514,36 +539,92 @@ async function render({
 
 export function createReference({ lookupMap }: { lookupMap: ContentLookupMap }) {
 	return function reference(collection: string) {
-		return zodString().transform((lookupId: string, ctx) => {
-			const flattenedErrorPath = ctx.path.join('.');
-			if (!lookupMap[collection]) {
-				ctx.addIssue({
-					code: ZodIssueCode.custom,
-					message: `**${flattenedErrorPath}:** Reference to ${collection} invalid. Collection does not exist or is empty.`,
-				});
-				return;
-			}
-
-			const { type, entries } = lookupMap[collection];
-			const entry = entries[lookupId];
-
-			if (!entry) {
-				ctx.addIssue({
-					code: ZodIssueCode.custom,
-					message: `**${flattenedErrorPath}**: Reference to ${collection} invalid. Expected ${Object.keys(
-						entries
-					)
-						.map((c) => JSON.stringify(c))
-						.join(' | ')}. Received ${JSON.stringify(lookupId)}.`,
-				});
-				return;
-			}
-			// Content is still identified by slugs, so map to a `slug` key for consistency.
-			if (type === 'content') {
-				return { slug: lookupId, collection };
-			}
-			return { id: lookupId, collection };
-		});
+		return z
+			.union([
+				z.string(),
+				z.object({
+					id: z.string(),
+					collection: z.string(),
+				}),
+				z.object({
+					slug: z.string(),
+					collection: z.string(),
+				}),
+			])
+			.transform(
+				async (
+					lookup:
+						| string
+						| { id: string; collection: string }
+						| { slug: string; collection: string },
+					ctx
+				) => {
+					const flattenedErrorPath = ctx.path.join('.');
+					const store = await globalDataStore.get();
+					const collectionIsInStore = store.hasCollection(collection);
+
+					if (typeof lookup === 'object') {
+						// If these don't match then something is wrong with the reference
+						if (lookup.collection !== collection) {
+							ctx.addIssue({
+								code: ZodIssueCode.custom,
+								message: `**${flattenedErrorPath}**: Reference to ${collection} invalid. Expected ${collection}. Received ${lookup.collection}.`,
+							});
+							return;
+						}
+
+						// A reference object might refer to an invalid collection, because when we convert it we don't have access to the store.
+						// If it is an object then we're validating later in the pipeline, so we can check the collection at that point.
+						if (!lookupMap[collection] && !collectionIsInStore) {
+							ctx.addIssue({
+								code: ZodIssueCode.custom,
+								message: `**${flattenedErrorPath}:** Reference to ${collection} invalid. Collection does not exist or is empty.`,
+							});
+							return;
+						}
+						return lookup;
+					}
+
+					if (collectionIsInStore) {
+						const entry = store.get(collection, lookup);
+						if (!entry) {
+							ctx.addIssue({
+								code: ZodIssueCode.custom,
+								message: `**${flattenedErrorPath}**: Reference to ${collection} invalid. Entry ${lookup} does not exist.`,
+							});
+							return;
+						}
+						return { id: lookup, collection };
+					}
+
+					if (!lookupMap[collection] && store.collections().size === 0) {
+						// If the collection is not in the lookup map or store, it may be a content layer collection and the store may not yet be populated.
+						// For now, we can't validate this reference, so we'll optimistically convert it to a reference object which we'll validate
+						// later in the pipeline when we do have access to the store.
+						return { id: lookup, collection };
+					}
+
+					const { type, entries } = lookupMap[collection];
+					const entry = entries[lookup];
+
+					if (!entry) {
+						ctx.addIssue({
+							code: ZodIssueCode.custom,
+							message: `**${flattenedErrorPath}**: Reference to ${collection} invalid. Expected ${Object.keys(
+								entries
+							)
+								.map((c) => JSON.stringify(c))
+								.join(' | ')}. Received ${JSON.stringify(lookup)}.`,
+						});
+						return;
+					}
+					// Content is still identified by slugs, so map to a `slug` key for consistency.
+					if (type === 'content') {
+						return { slug: lookup, collection };
+					}
+					return { id: lookup, collection };
+				}
+			);
 	};
 }
 

packages/astro/templates/content/module.mjs

@@ -33,6 +33,8 @@ const collectionToEntryMap = createCollectionToGlobResultMap({
 let lookupMap = {};
 /* @@LOOKUP_MAP_ASSIGNMENT@@ */
 
+const collectionNames = new Set(Object.keys(lookupMap));
+
 function createGlobLookup(glob) {
 	return async (collection, lookupId) => {
 		const filePath = lookupMap[collection]?.entries[lookupId];
@@ -59,15 +61,18 @@ export const getCollection = createGetCollection({
 export const getEntryBySlug = createGetEntryBySlug({
 	getEntryImport: createGlobLookup(contentCollectionToEntryMap),
 	getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap),
+	collectionNames,
 });
 
 export const getDataEntryById = createGetDataEntryById({
 	getEntryImport: createGlobLookup(dataCollectionToEntryMap),
+	collectionNames,
 });
 
 export const getEntry = createGetEntry({
 	getEntryImport: createGlobLookup(collectionToEntryMap),
 	getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap),
+	collectionNames,
 });
 
 export const getEntries = createGetEntries(getEntry);

packages/astro/test/content-layer.test.js

@@ -112,6 +112,27 @@ describe('Content Layer', () => {
 				},
 			});
 		});
+
+		it('transforms a reference id to a reference object', async () => {
+			assert.ok(json.hasOwnProperty('entryWithReference'));
+			assert.deepEqual(json.entryWithReference.data.cat, { collection: 'cats', id: 'tabby' });
+		});
+
+		it('returns a referenced entry', async () => {
+			assert.ok(json.hasOwnProperty('referencedEntry'));
+			assert.deepEqual(json.referencedEntry, {
+				collection: 'cats',
+				data: {
+					breed: 'Tabby',
+					id: 'tabby',
+					size: 'Medium',
+					origin: 'Egypt',
+					lifespan: '15 years',
+					temperament: ['Curious', 'Playful', 'Independent'],
+				},
+				id: 'tabby',
+			});
+		});
 	});
 
 	describe('Dev', () => {

packages/astro/test/fixtures/content-layer/content-outside-src/columbia-copy.md

@@ -3,6 +3,7 @@ title: More Columbia
 description: 'Learn about the Columbia NASA space shuttle.'
 publishedDate: 'Sat May 21 2022 00:00:00 GMT-0400 (Eastern Daylight Time)'
 tags: [space, 90s]
+cat: tabby
 ---
 
 **Source:** [Wikipedia](https://en.wikipedia.org/wiki/Space_Shuttle_Endeavour)

packages/astro/test/fixtures/content-layer/src/content/config.ts

@@ -1,4 +1,4 @@
-import { defineCollection, z } from 'astro:content';
+import { defineCollection, z, reference } from 'astro:content';
 import { file, glob } from 'astro/loaders';
 import { loader } from '../loaders/post-loader.js';
 import { fileURLToPath } from 'node:url';
@@ -82,6 +82,7 @@ const spacecraft = defineCollection({
 		publishedDate: z.string(),
 		tags: z.array(z.string()),
 		heroImage: image().optional(),
+		cat: reference('cats').optional(),
 	}),
 });
 

packages/astro/test/fixtures/content-layer/src/pages/collections.json.js

@@ -4,9 +4,12 @@ export async function GET() {
 	const customLoader = (await getCollection('blog')).slice(0, 10);
 	const fileLoader = await getCollection('dogs');
 
-	const dataEntry= await getEntry('dogs', 'beagle');
+	const dataEntry = await getEntry('dogs', 'beagle');
 
 	const simpleLoader = await getCollection('cats');
 
-	return Response.json({ customLoader, fileLoader, dataEntry, simpleLoader })
+	const entryWithReference = await getEntry('spacecraft', 'columbia-copy')
+	const referencedEntry = await getEntry(entryWithReference.data.cat)
+
+	return Response.json({ customLoader, fileLoader, dataEntry, simpleLoader, entryWithReference, referencedEntry });
 }

packages/astro/test/fixtures/content-layer/src/pages/spacecraft/[slug].astro

@@ -1,6 +1,6 @@
 ---
 import type { GetStaticPaths } from "astro";
-import { getCollection } from "astro:content"
+import { getCollection, getEntry } from "astro:content"
 import { Image } from "astro:assets"
 
 export const getStaticPaths = (async () => {
@@ -21,11 +21,13 @@ export const getStaticPaths = (async () => {
 
 const { craft } = Astro.props as any
 
+let cat = craft.data.cat ? await getEntry(craft.data.cat) : undefined
 const { Content } = await craft.render()
 
 ---
-
+<meta charset="utf-8">
 <h1>{craft.data.title}</h1>
+{cat? <p>🐈: {cat.data.breed}</p> : undefined}
 {craft.data.heroImage ? <Image src={craft.data.heroImage} alt={craft.data.title} width="100" height="100"/> : undefined}
 <Content />
 </ul>