diff --git a/packages/astro/src/content/consts.ts b/packages/astro/src/content/consts.ts index 70826c8984147..c4ace33d11c90 100644 --- a/packages/astro/src/content/consts.ts +++ b/packages/astro/src/content/consts.ts @@ -19,3 +19,5 @@ export const CONTENT_FLAGS = [ ] as const; export const CONTENT_TYPES_FILE = 'types.d.ts'; + +export const DATA_STORE_FILE = 'data-store.json'; diff --git a/packages/astro/src/content/data-store.ts b/packages/astro/src/content/data-store.ts index 65cb4298cbd66..07971562d2268 100644 --- a/packages/astro/src/content/data-store.ts +++ b/packages/astro/src/content/data-store.ts @@ -7,11 +7,19 @@ export class DataStore { get(collectionName: string, key: string) { return this.#collections.get(collectionName)?.get(String(key)); } - entries(collectionName: string): IterableIterator<[id: string, any]> { + entries(collectionName: string): Array<[id: string, any]> { const collection = this.#collections.get(collectionName) ?? new Map(); - return collection.entries(); + return [...collection.entries()]; } - set(collectionName: string, key: string, value: any) { + values(collectionName: string): Array { + const collection = this.#collections.get(collectionName) ?? new Map(); + return [...collection.values()]; + } + keys(collectionName: string): Array { + const collection = this.#collections.get(collectionName) ?? new Map(); + return [...collection.keys()]; + } + set(collectionName: string, key: string, value: unknown) { const collection = this.#collections.get(collectionName) ?? new Map(); collection.set(String(key), value); this.#collections.set(collectionName, collection); @@ -46,6 +54,8 @@ export class DataStore { return { get: (key: string) => this.get(collectionName, key), entries: () => this.entries(collectionName), + values: () => this.values(collectionName), + keys: () => this.keys(collectionName), set: (key: string, value: any) => this.set(collectionName, key, value), delete: (key: string) => this.delete(collectionName, key), clear: () => this.clear(collectionName), @@ -54,7 +64,7 @@ export class DataStore { } metaStore(collectionName: string): MetaStore { - return this.scopedStore(`meta:${collectionName}`); + return this.scopedStore(`meta:${collectionName}`) as MetaStore; } toString() { @@ -66,7 +76,11 @@ export class DataStore { } async writeToDisk(filePath: PathLike) { - await fs.writeFile(filePath, this.toString()); + try { + await fs.writeFile(filePath, this.toString()); + } catch { + throw new Error(`Failed to save data store to disk`); + } } static async fromDisk(filePath: PathLike) { @@ -103,14 +117,20 @@ export class DataStore { } export interface ScopedDataStore { - get: (key: string) => any; - entries: () => IterableIterator<[id: string, any]>; - set: (key: string, value: any) => void; + get: (key: string) => unknown; + entries: () => Array<[id: string, unknown]>; + set: (key: string, value: unknown) => void; + values: () => Array; + keys: () => Array; delete: (key: string) => void; clear: () => void; has: (key: string) => boolean; } +/** + * A key-value store for metadata strings. Useful for storing things like sync tokens. + */ + export interface MetaStore { get: (key: string) => string | undefined; set: (key: string, value: string) => void; diff --git a/packages/astro/src/content/file.ts b/packages/astro/src/content/file.ts index 4fa1ac21b6d55..cde24b53d94c4 100644 --- a/packages/astro/src/content/file.ts +++ b/packages/astro/src/content/file.ts @@ -9,7 +9,8 @@ import { promises as fs, existsSync } from 'fs'; */ export function file(fileName: string): Loader { if (fileName.includes('*')) { - throw new Error('Glob patterns are not supported in file loader. Use `glob` loader instead.'); + // TODO: AstroError + throw new Error('Glob patterns are not supported in `file` loader. Use `glob` loader instead.'); } return { name: 'file-loader', @@ -22,8 +23,16 @@ export function file(fileName: string): Loader { return; } - const data = await fs.readFile(url, 'utf-8'); - const json = JSON.parse(data); + let json: Array>; + + try { + const data = await fs.readFile(url, 'utf-8'); + json = JSON.parse(data); + } catch (error: any) { + logger.error(`Error reading data from ${fileName}`); + logger.debug(error.message); + return; + } const filePath = fileURLToPath(url); @@ -32,7 +41,11 @@ export function file(fileName: string): Loader { logger.warn(`No items found in ${fileName}`); } for (const rawItem of json) { - const id = rawItem.id ?? rawItem.slug; + const id = (rawItem.id ?? rawItem.slug)?.toString(); + if (!id) { + logger.error(`Item in ${fileName} is missing an id or slug field.`); + continue; + } const item = await parseData({ id, data: rawItem, filePath }); store.set(id, item); } diff --git a/packages/astro/src/content/loaders.ts b/packages/astro/src/content/loaders.ts index b966d13d47bcf..934ef99e3e30c 100644 --- a/packages/astro/src/content/loaders.ts +++ b/packages/astro/src/content/loaders.ts @@ -4,17 +4,19 @@ import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js'; import { DataStore, globalDataStore, type MetaStore, type ScopedDataStore } from './data-store.js'; import { getEntryData, globalContentConfigObserver } from './utils.js'; import { promises as fs, existsSync } from 'fs'; +import { DATA_STORE_FILE } from './consts.js'; export interface ParseDataOptions { /** The ID of the entry. Unique per collection */ id: string; /** The raw, unvalidated data of the entry */ data: Record; - /** An optional file path, where the entry represents a local file */ + /** An optional file path, where the entry represents a local file. */ filePath?: string; } export interface LoaderContext { + /** The unique name of the collection */ collection: string; /** A database abstraction to store the actual data */ store: ScopedDataStore; @@ -24,7 +26,7 @@ export interface LoaderContext { settings: AstroSettings; - /** Validates and parses the data according to the schema */ + /** Validates and parses the data according to the collection schema */ parseData = Record>( props: ParseDataOptions ): T; @@ -39,14 +41,20 @@ export interface Loader { schema?: S | Promise | (() => S | Promise); render?: (entry: any) => any; } -export async function syncDataLayer({ + +/** + * Run the `load()` method of each collection's loader, which will load the data and save it in the data store. + * The loader itself is responsible for deciding whether this will clear and reload the full collection, or + * perform an incremental update. After the data is loaded, the data store is written to disk. + */ +export async function syncContentLayer({ settings, logger: globalLogger, store, }: { settings: AstroSettings; logger: Logger; store?: DataStore }) { const logger = globalLogger.forkIntegrationLogger('content'); if (!store) { - store = await DataStore.fromDisk(new URL('data-store.json', settings.config.cacheDir)); + store = await DataStore.fromDisk(new URL(DATA_STORE_FILE, settings.config.cacheDir)); globalDataStore.set(store); } const contentConfig = globalContentConfigObserver.get(); @@ -106,7 +114,7 @@ export async function syncDataLayer({ }); }) ); - const cacheFile = new URL('data-store.json', settings.config.cacheDir); + const cacheFile = new URL(DATA_STORE_FILE, settings.config.cacheDir); if (!existsSync(settings.config.cacheDir)) { await fs.mkdir(settings.config.cacheDir, { recursive: true }); } diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index 9af1d4a7ec0ec..6308c47757e3b 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -16,6 +16,7 @@ import { CONTENT_FLAG, CONTENT_RENDER_FLAG, DATA_FLAG, + DATA_STORE_FILE, DATA_STORE_VIRTUAL_ID, RESOLVED_DATA_STORE_VIRTUAL_ID, RESOLVED_VIRTUAL_MODULE_ID, @@ -45,7 +46,7 @@ export function astroContentVirtualModPlugin({ }: AstroContentVirtualModPluginParams): Plugin { let IS_DEV = false; const IS_SERVER = isServerLikeOutput(settings.config); - const dataStoreFile = new URL('data-store.json', settings.config.cacheDir); + const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir); return { name: 'astro-content-virtual-mod-plugin', enforce: 'pre', diff --git a/packages/astro/src/core/build/index.ts b/packages/astro/src/core/build/index.ts index 1f1fdf56a348c..b318ffb5b4b2f 100644 --- a/packages/astro/src/core/build/index.ts +++ b/packages/astro/src/core/build/index.ts @@ -11,7 +11,7 @@ import type { RuntimeMode, } from '../../@types/astro.js'; import { injectImageEndpoint } from '../../assets/endpoint/config.js'; -import { syncDataLayer } from '../../content/loaders.js'; +import { syncContentLayer } from '../../content/loaders.js'; import { telemetry } from '../../events/index.js'; import { eventCliSession } from '../../events/session.js'; import { @@ -152,7 +152,7 @@ class AstroBuilder { const dataStore = await DataStore.fromModule(); globalDataStore.set(dataStore); - await syncDataLayer({ settings: this.settings, logger: logger }); + await syncContentLayer({ settings: this.settings, logger: logger }); return { viteConfig }; } diff --git a/packages/astro/src/core/sync/index.ts b/packages/astro/src/core/sync/index.ts index 1533e793ccf0b..e399ee516a02b 100644 --- a/packages/astro/src/core/sync/index.ts +++ b/packages/astro/src/core/sync/index.ts @@ -6,7 +6,7 @@ import { type HMRPayload, createServer } from 'vite'; import type { AstroConfig, AstroInlineConfig, AstroSettings } from '../../@types/astro.js'; import { getPackage } from '../../cli/install-package.js'; import { createContentTypesGenerator } from '../../content/index.js'; -import { syncDataLayer } from '../../content/loaders.js'; +import { syncContentLayer } from '../../content/loaders.js'; import { globalContentConfigObserver } from '../../content/utils.js'; import { syncAstroEnv } from '../../env/sync.js'; import { telemetry } from '../../events/index.js'; @@ -87,7 +87,7 @@ export default async function sync( if (exitCode !== 0) return exitCode; syncAstroEnv(settings, options?.fs); - await syncDataLayer({ settings, logger }); + await syncContentLayer({ settings, logger }); logger.info(null, `Types generated ${dim(getTimeStat(timerStart, performance.now()))}`); return 0; diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 0262d4d06bda5..6b079e299818b 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -2,88 +2,82 @@ import assert from 'node:assert/strict'; import { before, describe, it } from 'node:test'; import { loadFixture } from './test-utils.js'; -describe('Content Collections', () => { - describe('Query', () => { - let fixture; - before(async () => { - fixture = await loadFixture({ root: './fixtures/content-layer/' }); - await fixture.build(); - }); +describe('Content Layer', () => { + let fixture; + let json; - describe('Collection', () => { - let json; - before(async () => { - const rawJson = await fixture.readFile('/collections.json'); - json = JSON.parse(rawJson); - }); + before(async () => { + fixture = await loadFixture({ root: './fixtures/content-layer/' }); + await fixture.build(); + const rawJson = await fixture.readFile('/collections.json'); + json = JSON.parse(rawJson); + }); - it('Returns custom loader collection', async () => { - assert.ok(json.hasOwnProperty('customLoader')); - assert.ok(Array.isArray(json.customLoader)); + it('Returns custom loader collection', async () => { + assert.ok(json.hasOwnProperty('customLoader')); + assert.ok(Array.isArray(json.customLoader)); - const item = json.customLoader[0]; - assert.deepEqual(item, { - id: '1', - collection: 'blog', - data: { - userId: 1, - id: 1, - title: 'sunt aut facere repellat provident occaecati excepturi optio reprehenderit', - body: 'quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto', - }, - type: 'experimental_data', - }); - }); + const item = json.customLoader[0]; + assert.deepEqual(item, { + id: '1', + collection: 'blog', + data: { + userId: 1, + id: 1, + title: 'sunt aut facere repellat provident occaecati excepturi optio reprehenderit', + body: 'quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto', + }, + type: 'experimental_data', + }); + }); - it('Returns file loader collection', async () => { - assert.ok(json.hasOwnProperty('fileLoader')); - assert.ok(Array.isArray(json.fileLoader)); + it('Returns `file()` loader collection', async () => { + assert.ok(json.hasOwnProperty('fileLoader')); + assert.ok(Array.isArray(json.fileLoader)); - const ids = json.fileLoader.map((item) => item.data.id); - assert.deepEqual(ids, [ - 'labrador-retriever', - 'german-shepherd', - 'golden-retriever', - 'french-bulldog', - 'bulldog', - 'beagle', - 'poodle', - 'rottweiler', - 'german-shorthaired-pointer', - 'yorkshire-terrier', - 'boxer', - 'dachshund', - 'siberian-husky', - 'great-dane', - 'doberman-pinscher', - 'australian-shepherd', - 'miniature-schnauzer', - 'cavalier-king-charles-spaniel', - 'shih-tzu', - 'boston-terrier', - 'bernese-mountain-dog', - 'pomeranian', - 'havanese', - 'english-springer-spaniel', - 'shetland-sheepdog', - ]); - }); + const ids = json.fileLoader.map((item) => item.data.id); + assert.deepEqual(ids, [ + 'labrador-retriever', + 'german-shepherd', + 'golden-retriever', + 'french-bulldog', + 'bulldog', + 'beagle', + 'poodle', + 'rottweiler', + 'german-shorthaired-pointer', + 'yorkshire-terrier', + 'boxer', + 'dachshund', + 'siberian-husky', + 'great-dane', + 'doberman-pinscher', + 'australian-shepherd', + 'miniature-schnauzer', + 'cavalier-king-charles-spaniel', + 'shih-tzu', + 'boston-terrier', + 'bernese-mountain-dog', + 'pomeranian', + 'havanese', + 'english-springer-spaniel', + 'shetland-sheepdog', + ]); + }); - it('Returns data entry by id', async () => { - assert.ok(json.hasOwnProperty('dataEntryById')); - assert.deepEqual(json.dataEntryById, { - id: 'beagle', - collection: 'dogs', - data: { - breed: 'Beagle', - id: 'beagle', - size: 'Small to Medium', - origin: 'England', - lifespan: '12-15 years', - temperament: ['Friendly', 'Curious', 'Merry'], - }, - }); - }); + it('Returns data entry by id', async () => { + assert.ok(json.hasOwnProperty('dataEntryById')); + assert.deepEqual(json.dataEntryById, { + id: 'beagle', + collection: 'dogs', + data: { + breed: 'Beagle', + id: 'beagle', + size: 'Small to Medium', + origin: 'England', + lifespan: '12-15 years', + temperament: ['Friendly', 'Curious', 'Merry'], + }, }); }); });