From 7221d92a5458417b7f06455842edad6681ac51a4 Mon Sep 17 00:00:00 2001 From: Michael Cooper Date: Wed, 21 Feb 2024 15:22:32 -0800 Subject: [PATCH 01/30] typescript level conversion --- package.json | 11 ++- src/brandedFs.ts | 100 +++++++++++++++++++++ src/brandedPath.ts | 98 +++++++++++++++++++++ src/build.ts | 100 ++++++++++++--------- src/commandInstruction.ts | 2 +- src/config.ts | 40 +++++---- src/convert.ts | 28 +++--- src/create.ts | 65 +++++++------- src/dataloader.ts | 82 +++++++++--------- src/deploy.ts | 29 +++++-- src/fileWatchers.ts | 11 ++- src/files.ts | 51 ++++++----- src/javascript.ts | 15 ++-- src/javascript/features.ts | 11 +-- src/javascript/imports.ts | 133 +++++++++++++++------------- src/libraries.ts | 10 ++- src/markdown.ts | 41 ++++----- src/observableApiClient.ts | 11 +-- src/observableApiConfig.ts | 50 +++++------ src/pager.ts | 16 ++-- src/preview.ts | 143 +++++++++++++++++-------------- src/render.ts | 69 ++++++++------- src/rollup.ts | 73 +++++++++++----- src/search.ts | 12 +-- src/telemetry.ts | 6 +- src/theme.ts | 33 +++---- src/tty.ts | 4 +- src/url.ts | 31 ++++--- test/build-test.ts | 42 ++++----- test/config-test.ts | 9 +- test/create-test.ts | 9 +- test/dataloaders-test.ts | 33 +++---- test/deploy-test.ts | 3 +- test/fileWatchers-test.ts | 78 +++++++++-------- test/files-test.ts | 17 ++-- test/isLocalImport-test.ts | 35 ++++---- test/javascript-test.ts | 43 +++++----- test/javascript/features-test.ts | 3 +- test/javascript/imports-test.ts | 108 +++++++++++++++-------- test/markdown-test.ts | 36 ++++---- test/observableApiConfig-test.ts | 20 +++-- test/pager-test.ts | 112 +++++++++++++++--------- test/telemetry-test.ts | 2 +- test/url-test.ts | 63 +++++++------- yarn.lock | 129 ++++++++++++++++++++++++++-- 45 files changed, 1272 insertions(+), 745 deletions(-) create mode 100644 src/brandedFs.ts create mode 100644 src/brandedPath.ts diff --git a/package.json b/package.json index d2c0b9fbb..3fba9af61 100644 --- a/package.json +++ b/package.json @@ -22,13 +22,14 @@ "observable": "bin/observable-init.js" }, "scripts": { - "dev": "rm -f docs/themes.md docs/theme/*.md && (tsx watch docs/theme/generate-themes.ts & tsx watch --no-warnings=ExperimentalWarning ./bin/observable.ts preview --no-open)", - "build": "yarn rebuild-themes && rm -rf dist && tsx --no-warnings=ExperimentalWarning ./bin/observable.ts build", + "dev": "rimraf --glob docs/themes.md docs/theme/*.md && (tsx watch docs/theme/generate-themes.ts & tsx watch --no-warnings=ExperimentalWarning ./bin/observable.ts preview --no-open)", + "build": "yarn rebuild-themes && rimraf dist && tsx --no-warnings=ExperimentalWarning ./bin/observable.ts build", "deploy": "yarn rebuild-themes && tsx --no-warnings=ExperimentalWarning ./bin/observable.ts deploy", - "rebuild-themes": "rm -f docs/themes.md docs/theme/*.md && tsx docs/theme/generate-themes.ts", + "rebuild-themes": "rimraf --glob docs/themes.md docs/theme/*.md && tsx docs/theme/generate-themes.ts", "test": "yarn test:mocha && yarn test:tsc && yarn test:lint && yarn test:prettier", "test:coverage": "c8 yarn test:mocha", - "test:mocha": "rm -rf test/.observablehq/cache test/input/build/*/.observablehq/cache && OBSERVABLE_TELEMETRY_DISABLE=1 TZ=America/Los_Angeles tsx --no-warnings=ExperimentalWarning ./node_modules/.bin/mocha 'test/**/*-test.*'", + "test:mocha": "rimraf --glob test/.observablehq/cache test/input/build/*/.observablehq/cache && cross-env OBSERVABLE_TELEMETRY_DISABLE=1 TZ=America/Los_Angeles tsx --no-warnings=ExperimentalWarning ./node_modules/mocha/bin/mocha.js 'test/**/*-test.*'", + "test:mocha-debug": "rimraf --glob test/.observablehq/cache test/input/build/*/.observablehq/cache && cross-env OBSERVABLE_TELEMETRY_DISABLE=1 TZ=America/Los_Angeles tsx --no-warnings=ExperimentalWarning --inspect-brk ./node_modules/mocha/bin/mocha.js --timeout 3600000 'test/**/*-test.*'", "test:lint": "eslint src test --max-warnings=0", "test:prettier": "prettier --check src test", "test:tsc": "tsc --noEmit", @@ -92,6 +93,7 @@ "c8": "^8.0.1", "chai": "^4.3.10", "chai-http": "^4.4.0", + "cross-env": "^7.0.3", "d3-array": "^3.2.4", "d3-dsv": "^3.0.1", "eslint": "^8.50.0", @@ -100,6 +102,7 @@ "eslint-plugin-import": "^2.29.0", "mocha": "^10.2.0", "prettier": "^3.0.3 <3.1", + "rimraf": "^5.0.5", "typescript": "^5.2.2", "undici": "^5.27.2" }, diff --git a/src/brandedFs.ts b/src/brandedFs.ts new file mode 100644 index 000000000..9cc901399 --- /dev/null +++ b/src/brandedFs.ts @@ -0,0 +1,100 @@ +import fs, {type MakeDirectoryOptions, type Stats, type WatchListener, type WriteFileOptions} from "node:fs"; +import fsp, {type FileHandle} from "node:fs/promises"; +import {FilePath, unFilePath} from "./brandedPath.js"; + +export const constants = fsp.constants; + +export function access(path: FilePath, mode?: number): Promise { + return fsp.access(unFilePath(path), mode); +} + +export function accessSync(path: FilePath, mode?: number): void { + return fs.accessSync(unFilePath(path), mode); +} + +export function copyFile(source: FilePath, destination: FilePath, flags?: number): Promise { + return fsp.copyFile(unFilePath(source), unFilePath(destination), flags); +} + +export function readFile(path: FilePath, encoding?: undefined): Promise; +export function readFile(path: FilePath, encoding: BufferEncoding): Promise; +export function readFile(path: FilePath, encoding?: BufferEncoding | undefined): Promise { + return fsp.readFile(unFilePath(path), encoding); +} + +export function readFileSync(path: FilePath, encoding: BufferEncoding): string { + return fs.readFileSync(unFilePath(path), encoding); +} + +export function writeFile(path: FilePath, data: string | Buffer, options?: WriteFileOptions): Promise { + return fsp.writeFile(unFilePath(path), data, options); +} + +export function writeFileSync(path: FilePath, data: string | Buffer, options?: WriteFileOptions): void { + return fs.writeFileSync(unFilePath(path), data, options); +} + +export async function mkdir(path: FilePath, options?: MakeDirectoryOptions): Promise { + const rv = await fsp.mkdir(unFilePath(path), options); + return rv ? FilePath(rv) : undefined; +} + +export function readdir(path: FilePath): Promise { + return fsp.readdir(unFilePath(path)); +} + +export function stat(path: FilePath): Promise { + return fsp.stat(unFilePath(path)); +} + +export function open(path: FilePath, flags: string | number, mode?: number): Promise { + return fsp.open(unFilePath(path), flags, mode); +} + +export function rename(oldPath: FilePath, newPath: FilePath): Promise { + return fsp.rename(unFilePath(oldPath), unFilePath(newPath)); +} + +export function renameSync(oldPath: FilePath, newPath: FilePath): void { + return fs.renameSync(unFilePath(oldPath), unFilePath(newPath)); +} + +export function unlink(path: FilePath): Promise { + return fsp.unlink(unFilePath(path)); +} + +export function unlinkSync(path: FilePath): void { + return fs.unlinkSync(unFilePath(path)); +} + +export function existsSync(path: FilePath): boolean { + return fs.existsSync(unFilePath(path)); +} + +export function readdirSync(path: FilePath): FilePath[] { + return fs.readdirSync(unFilePath(path)) as unknown as FilePath[]; +} + +export function statSync(path: FilePath): Stats { + return fs.statSync(unFilePath(path)); +} + +export function watch(path: FilePath, listener?: WatchListener): fs.FSWatcher { + return fs.watch(unFilePath(path), listener); +} + +export function utimes(path: FilePath, atime: number | Date, mtime: number | Date): Promise { + return fsp.utimes(unFilePath(path), atime, mtime); +} + +export function utimesSync(path: FilePath, atime: number | Date, mtime: number | Date): void { + return fs.utimesSync(unFilePath(path), atime, mtime); +} + +export function createReadStream(path: FilePath): fs.ReadStream { + return fs.createReadStream(unFilePath(path)); +} + +export function rm(path: FilePath, options?: {force?: boolean; recursive?: boolean}): Promise { + return fsp.rm(unFilePath(path), options); +} diff --git a/src/brandedPath.ts b/src/brandedPath.ts new file mode 100644 index 000000000..912691dd7 --- /dev/null +++ b/src/brandedPath.ts @@ -0,0 +1,98 @@ +import osPath from "node:path"; +import posixPath from "node:path/posix"; + +// including "unknown &" here improves type error message +type BrandedString = unknown & + Omit & {__type: T} & { + replace: (search: string | RegExp, replace: string) => BrandedString; + slice: (start?: number, end?: number) => BrandedString; + }; + +export type FilePath = BrandedString<"FilePath">; +export type UrlPath = BrandedString<"UrlPath">; + +export function FilePath(path: string | FilePath): FilePath { + if (osPath.sep === "\\") { + path = path.replaceAll("/", osPath.sep); + } else if (osPath.sep === "/") { + path = path.replaceAll("\\", osPath.sep); + } + return path as unknown as FilePath; +} + +export function UrlPath(path: string | UrlPath): UrlPath { + path = path.replaceAll("\\", posixPath.sep); + return path as unknown as UrlPath; +} + +export function unFilePath(path: FilePath | string): string { + return path as unknown as string; +} + +export function unUrlPath(path: UrlPath | string): string { + return path as unknown as string; +} + +export function filePathToUrlPath(path: FilePath): UrlPath { + if (osPath.sep === "/") return path as unknown as UrlPath; + return urlJoin(...(path.split(osPath.sep) as string[])); +} + +export function urlPathToFilePath(path: UrlPath): FilePath { + if (osPath.sep === "/") return path as unknown as FilePath; + return fileJoin(...(path.split(posixPath.sep) as string[])); +} + +// Implemenations of node:path functions: + +export function urlJoin(...paths: (string | UrlPath)[]): UrlPath { + return posixPath.join(...(paths as string[])) as unknown as UrlPath; +} + +export function fileJoin(...paths: (string | FilePath)[]): FilePath { + return osPath.join(...(paths as string[])) as unknown as FilePath; +} + +export function fileRelative(from: string | FilePath, to: string | FilePath): FilePath { + return FilePath(osPath.relative(unFilePath(from), unFilePath(to))); +} + +export function fileDirname(path: string | FilePath): FilePath { + return FilePath(osPath.dirname(unFilePath(path))); +} + +export function urlDirname(path: string | UrlPath): UrlPath { + return UrlPath(posixPath.dirname(unUrlPath(path))); +} + +export function fileNormalize(path: string | FilePath): FilePath { + return FilePath(osPath.normalize(unFilePath(path))); +} + +export function urlNormalize(path: string | UrlPath): UrlPath { + return UrlPath(osPath.normalize(unUrlPath(path))); +} + +export function fileBasename(path: string | FilePath, suffix?: string): string { + return osPath.basename(unFilePath(path), suffix); +} + +export function urlBasename(path: string | UrlPath, suffix?: string): string { + return osPath.basename(unUrlPath(path), suffix); +} + +export function fileExtname(path: string | FilePath | string): string { + return osPath.extname(unFilePath(path)); +} + +export function urlExtname(path: string | UrlPath | string): string { + return osPath.extname(unUrlPath(path)); +} + +export function fileResolve(...paths: (string | FilePath)[]): FilePath { + return FilePath(osPath.resolve(...(paths as string[]))); +} + +export function urlResolve(...paths: (string | UrlPath)[]): UrlPath { + return UrlPath(osPath.resolve(...(paths as string[]))); +} diff --git a/src/build.ts b/src/build.ts index 539776d3f..6225a5c14 100644 --- a/src/build.ts +++ b/src/build.ts @@ -1,7 +1,19 @@ -import {existsSync} from "node:fs"; -import {access, constants, copyFile, readFile, writeFile} from "node:fs/promises"; -import {basename, dirname, join} from "node:path"; import {fileURLToPath} from "node:url"; +import {existsSync} from "./brandedFs.js"; +import {access, constants, copyFile, readFile, writeFile} from "./brandedFs.js"; +import { + FilePath, + fileBasename, + fileDirname, + fileJoin, + filePathToUrlPath, + unFilePath, + unUrlPath, + urlBasename, + urlDirname, + urlJoin, + urlPathToFilePath +} from "./brandedPath.js"; import type {Config, Style} from "./config.js"; import {mergeStyle} from "./config.js"; import {Loader} from "./dataloader.js"; @@ -16,10 +28,10 @@ import {Telemetry} from "./telemetry.js"; import {faint} from "./tty.js"; import {resolvePath} from "./url.js"; -const EXTRA_FILES = new Map([ +const EXTRA_FILES: Map = new Map([ [ - join(fileURLToPath(import.meta.resolve("@observablehq/runtime")), "../../dist/runtime.js"), - "_observablehq/runtime.js" + fileJoin(fileURLToPath(import.meta.resolve("@observablehq/runtime")), "..", "..", "dist", "runtime.js"), + fileJoin("_observablehq", "runtime.js") ] ]); @@ -32,18 +44,20 @@ export interface BuildOptions { export interface BuildEffects { logger: Logger; output: Writer; + existsSync: (path: FilePath) => boolean; + readFile(path: FilePath, encoding: "utf-8"): Promise; /** * @param outputPath The path of this file relative to the outputRoot. For * example, in a local build this should be relative to the dist directory. */ - copyFile(sourcePath: string, outputPath: string): Promise; + copyFile(sourcePath: FilePath, outputPath: FilePath): Promise; /** * @param outputPath The path of this file relative to the outputRoot. For * example, in a local build this should be relative to the dist directory. */ - writeFile(outputPath: string, contents: Buffer | string): Promise; + writeFile(outputPath: FilePath, contents: Buffer | string): Promise; } export async function build( @@ -56,34 +70,36 @@ export async function build( // Make sure all files are readable before starting to write output files. let pageCount = 0; for await (const sourceFile of visitMarkdownFiles(root)) { - await access(join(root, sourceFile), constants.R_OK); + await access(fileJoin(root, sourceFile), constants.R_OK); pageCount++; } if (!pageCount) throw new CliError(`Nothing to build: no page files found in your ${root} directory.`); effects.logger.log(`${faint("found")} ${pageCount} ${faint(`page${pageCount === 1 ? "" : "s"} in`)} ${root}`); // Render .md files, building a list of file attachments as we go. - const files: string[] = []; - const imports: string[] = []; + const files: FilePath[] = []; + const imports: FilePath[] = []; const styles: Style[] = []; for await (const sourceFile of visitMarkdownFiles(root)) { - const sourcePath = join(root, sourceFile); - const outputPath = join(dirname(sourceFile), basename(sourceFile, ".md") + ".html"); + const sourcePath = fileJoin(root, sourceFile); + const outputPath = fileJoin(fileDirname(sourceFile), fileBasename(sourceFile, ".md") + ".html"); effects.output.write(`${faint("render")} ${sourcePath} ${faint("→")} `); - const path = join("/", dirname(sourceFile), basename(sourceFile, ".md")); - const render = await renderServerless(sourcePath, {path, ...config}); + const urlSourceFile = filePathToUrlPath(sourceFile); + const urlPath = urlJoin("/", urlDirname(urlSourceFile), urlBasename(urlSourceFile, ".md")); + const filePath = fileJoin(fileDirname(sourceFile), fileBasename(sourceFile, ".md")); + const render = await renderServerless(sourcePath, {path: urlPath, ...config}); const resolveFile = ({name}) => resolvePath(sourceFile, name); files.push(...render.files.map(resolveFile)); imports.push(...render.imports.filter((i) => i.type === "local").map(resolveFile)); await effects.writeFile(outputPath, render.html); - const style = mergeStyle(path, render.data?.style, render.data?.theme, config.style); + const style = mergeStyle(filePath, render.data?.style, render.data?.theme, config.style); if (style && !styles.some((s) => styleEquals(s, style))) styles.push(style); } // Add imported local scripts. for (const script of config.scripts) { - if (!/^\w+:/.test(script.src)) { - imports.push(script.src); + if (!/^\w+:/.test(unUrlPath(script.src))) { + imports.push(urlPathToFilePath(script.src)); } } @@ -106,8 +122,8 @@ export async function build( ["./src/client/stdlib/zip.js", "stdlib/zip.js"], ...(config.search ? [["./src/client/search.js", "search.js"]] : []) ]) { - const clientPath = getClientPath(entry); - const outputPath = join("_observablehq", name); + const clientPath = getClientPath(FilePath(entry)); + const outputPath = fileJoin("_observablehq", name); effects.output.write(`${faint("bundle")} ${clientPath} ${faint("→")} `); const code = await (entry.endsWith(".css") ? bundleStyles({path: clientPath}) @@ -115,20 +131,20 @@ export async function build( await effects.writeFile(outputPath, code); } if (config.search) { - const outputPath = join("_observablehq", "minisearch.json"); + const outputPath = fileJoin("_observablehq", "minisearch.json"); const code = await searchIndex(config, effects); effects.output.write(`${faint("search")} ${faint("→")} `); await effects.writeFile(outputPath, code); } for (const style of styles) { if ("path" in style) { - const outputPath = join("_import", style.path); - const sourcePath = join(root, style.path); + const outputPath = fileJoin("_import", urlPathToFilePath(style.path)); + const sourcePath = fileJoin(root, urlPathToFilePath(style.path)); effects.output.write(`${faint("style")} ${sourcePath} ${faint("→")} `); const code = await bundleStyles({path: sourcePath}); await effects.writeFile(outputPath, code); } else { - const outputPath = join("_observablehq", `theme-${style.theme}.css`); + const outputPath = fileJoin("_observablehq", `theme-${style.theme}.css`); effects.output.write(`${faint("bundle")} theme-${style.theme}.css ${faint("→")} `); const code = await bundleStyles({theme: style.theme}); await effects.writeFile(outputPath, code); @@ -138,16 +154,16 @@ export async function build( // Copy over the referenced files. for (const file of files) { - let sourcePath = join(root, file); - const outputPath = join("_file", file); - if (!existsSync(sourcePath)) { - const loader = Loader.find(root, join("/", file), {useStale: true}); + let sourcePath = fileJoin(root, file); + const outputPath = fileJoin("_file", file); + if (!effects.existsSync(sourcePath)) { + const loader = Loader.find(root, file, {useStale: true}); if (!loader) { effects.logger.error("missing referenced file", sourcePath); continue; } try { - sourcePath = join(root, await loader.load(effects)); + sourcePath = fileJoin(root, await loader.load(effects)); } catch (error) { if (!isEnoent(error)) throw error; continue; @@ -160,14 +176,14 @@ export async function build( // Copy over the imported modules. const importResolver = createImportResolver(root); for (const file of imports) { - const sourcePath = join(root, file); - const outputPath = join("_import", file); - if (!existsSync(sourcePath)) { + const sourcePath = fileJoin(root, file); + const outputPath = fileJoin("_import", file); + if (!effects.existsSync(sourcePath)) { effects.logger.error("missing referenced file", sourcePath); continue; } effects.output.write(`${faint("copy")} ${sourcePath} ${faint("→")} `); - const contents = await rewriteModule(await readFile(sourcePath, "utf-8"), file, importResolver); + const contents = await rewriteModule(await effects.readFile(sourcePath, "utf-8"), file, importResolver); await effects.writeFile(outputPath, contents); } @@ -182,11 +198,11 @@ export async function build( } export class FileBuildEffects implements BuildEffects { - private readonly outputRoot: string; + private readonly outputRoot: FilePath; readonly logger: Logger; readonly output: Writer; constructor( - outputRoot: string, + outputRoot: FilePath, {logger = console, output = process.stdout}: {logger?: Logger; output?: Writer} = {} ) { if (!outputRoot) throw new Error("missing outputRoot"); @@ -194,14 +210,20 @@ export class FileBuildEffects implements BuildEffects { this.output = output; this.outputRoot = outputRoot; } - async copyFile(sourcePath: string, outputPath: string): Promise { - const destination = join(this.outputRoot, outputPath); + existsSync(path: FilePath) { + return existsSync(unFilePath(path)); + } + readFile(path: FilePath, encoding: "utf-8"): Promise { + return readFile(path, encoding); + } + async copyFile(sourcePath: FilePath, outputPath: FilePath): Promise { + const destination = fileJoin(this.outputRoot, outputPath); this.logger.log(destination); await prepareOutput(destination); await copyFile(sourcePath, destination); } - async writeFile(outputPath: string, contents: string | Buffer): Promise { - const destination = join(this.outputRoot, outputPath); + async writeFile(outputPath: FilePath, contents: string | Buffer): Promise { + const destination = fileJoin(this.outputRoot, outputPath); this.logger.log(destination); await prepareOutput(destination); await writeFile(destination, contents); diff --git a/src/commandInstruction.ts b/src/commandInstruction.ts index 19eadf7c2..b41aed2a8 100644 --- a/src/commandInstruction.ts +++ b/src/commandInstruction.ts @@ -9,7 +9,7 @@ export function commandInstruction( env = process.env }: {color?: TtyColor | null; env?: Record} = {} ): string { - if (!color) color = (s) => s; + if (!color) color = (s) => `${s}`; const prefix = env["npm_config_user_agent"]?.includes("yarn/") ? "yarn observable" diff --git a/src/config.ts b/src/config.ts index a1cf671a9..c9c6488b9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,5 @@ -import {basename, dirname, join} from "node:path"; +import type {UrlPath} from "./brandedPath.js"; +import {FilePath, fileJoin, filePathToUrlPath, urlBasename, urlDirname, urlJoin} from "./brandedPath.js"; import {visitMarkdownFiles} from "./files.js"; import {formatIsoDate, formatLocaleDate} from "./format.js"; import {parseMarkdown} from "./markdown.js"; @@ -7,7 +8,7 @@ import {resolvePath} from "./url.js"; export interface Page { name: string; - path: string; + path: UrlPath; } export interface Section { @@ -22,19 +23,19 @@ export interface TableOfContents { } export type Style = - | {path: string} // custom stylesheet + | {path: UrlPath} // custom stylesheet | {theme: string[]}; // zero or more named theme export interface Script { - src: string; + src: UrlPath; async: boolean; type: string | null; } export interface Config { - root: string; // defaults to docs - output: string; // defaults to dist - base: string; // defaults to "/" + root: FilePath; // defaults to docs + output: FilePath; // defaults to dist + base: UrlPath; // defaults to "/" title?: string; sidebar: boolean; // defaults to true if pages isn’t empty pages: (Page | Section)[]; @@ -49,16 +50,16 @@ export interface Config { search: boolean; // default to false } -export async function readConfig(configPath?: string, root?: string): Promise { +export async function readConfig(configPath?: FilePath, root?: FilePath): Promise { if (configPath === undefined) return readDefaultConfig(root); - const importPath = join(process.cwd(), root ?? ".", configPath); + const importPath = "file:///" + filePathToUrlPath(fileJoin(process.cwd(), root ?? ".", configPath)); return normalizeConfig((await import(importPath)).default, root); } -export async function readDefaultConfig(root?: string): Promise { +export async function readDefaultConfig(root?: FilePath): Promise { for (const ext of [".js", ".ts"]) { try { - return await readConfig("observablehq.config" + ext, root); + return await readConfig(FilePath("observablehq.config" + ext), root); } catch (error: any) { if (error.code !== "ERR_MODULE_NOT_FOUND") throw error; continue; @@ -67,13 +68,14 @@ export async function readDefaultConfig(root?: string): Promise { return normalizeConfig(undefined, root); } -async function readPages(root: string): Promise { +async function readPages(root: FilePath): Promise { const pages: Page[] = []; for await (const file of visitMarkdownFiles(root)) { - if (file === "index.md" || file === "404.md") continue; - const parsed = await parseMarkdown(join(root, file), {root, path: file}); - const name = basename(file, ".md"); - const page = {path: join("/", dirname(file), name), name: parsed.title ?? "Untitled"}; + const urlFile = filePathToUrlPath(file); + if (file === FilePath("index.md") || file === FilePath("404.md")) continue; + const parsed = await parseMarkdown(fileJoin(root, file), {root, path: urlFile}); + const name = urlBasename(urlFile, ".md"); + const page = {path: urlJoin("/", urlDirname(urlFile), name), name: parsed.title ?? "Untitled"}; if (name === "index") pages.unshift(page); else pages.push(page); } @@ -86,7 +88,7 @@ export function setCurrentDate(date = new Date()): void { currentDate = date; } -export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Promise { +export async function normalizeConfig(spec: any = {}, defaultRoot = FilePath("docs")): Promise { let { root = defaultRoot, output = "dist", @@ -178,12 +180,12 @@ export function mergeToc(spec: any, toc: TableOfContents): TableOfContents { return {label, show}; } -export function mergeStyle(path: string, style: any, theme: any, defaultStyle: null | Style): null | Style { +export function mergeStyle(path: FilePath, style: any, theme: any, defaultStyle: null | Style): null | Style { return style === undefined && theme === undefined ? defaultStyle : style === null ? null // disable : style !== undefined - ? {path: resolvePath(path, style)} + ? {path: filePathToUrlPath(resolvePath(path, style))} : {theme: normalizeTheme(theme)}; } diff --git a/src/convert.ts b/src/convert.ts index 372d3ec3b..d10cd973e 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -1,8 +1,8 @@ -import {existsSync} from "node:fs"; -import {utimes, writeFile} from "node:fs/promises"; -import {join} from "node:path"; import * as clack from "@clack/prompts"; import wrapAnsi from "wrap-ansi"; +import {utimes, writeFile} from "./brandedFs.js"; +import {existsSync} from "./brandedFs.js"; +import {type FilePath, fileJoin} from "./brandedPath.js"; import type {ClackEffects} from "./clack.js"; import {CliError} from "./error.js"; import {prepareOutput} from "./files.js"; @@ -11,25 +11,25 @@ import {type TtyEffects, bold, cyan, faint, inverse, link, reset, defaultEffects export interface ConvertEffects extends TtyEffects { clack: ClackEffects; - prepareOutput(outputPath: string): Promise; - existsSync(outputPath: string): boolean; - writeFile(outputPath: string, contents: Buffer | string): Promise; - touch(outputPath: string, date: Date | string | number): Promise; + prepareOutput(outputPath: FilePath): Promise; + existsSync(outputPath: FilePath): boolean; + writeFile(outputPath: FilePath, contents: Buffer | string): Promise; + touch(outputPath: FilePath, date: Date | string | number): Promise; } const defaultEffects: ConvertEffects = { ...ttyEffects, clack, - async prepareOutput(outputPath: string): Promise { + async prepareOutput(outputPath: FilePath): Promise { await prepareOutput(outputPath); }, - existsSync(outputPath: string): boolean { + existsSync(outputPath: FilePath): boolean { return existsSync(outputPath); }, - async writeFile(outputPath: string, contents: Buffer | string): Promise { + async writeFile(outputPath: FilePath, contents: Buffer | string): Promise { await writeFile(outputPath, contents); }, - async touch(outputPath: string, date: Date | string | number): Promise { + async touch(outputPath: FilePath, date: Date | string | number): Promise { await utimes(outputPath, (date = new Date(date)), date); } }; @@ -47,7 +47,7 @@ export async function convert( let s = clack.spinner(); const url = resolveInput(input); const name = inferFileName(url); - const path = join(output, name); + const path = fileJoin(output, name); if (await maybeFetch(path, force, effects)) { s.start(`Downloading ${bold(path)}`); const response = await fetch(url); @@ -60,7 +60,7 @@ export async function convert( n++; if (includeFiles) { for (const file of files) { - const path = join(output, file.name); + const path = fileJoin(output, file.name); if (await maybeFetch(path, force, effects)) { start = Date.now(); s = clack.spinner(); @@ -94,7 +94,7 @@ export async function convert( ); } -async function maybeFetch(path: string, force: boolean, effects: ConvertEffects): Promise { +async function maybeFetch(path: FilePath, force: boolean, effects: ConvertEffects): Promise { const {clack} = effects; if (effects.existsSync(path) && !force) { const choice = await clack.confirm({message: `${bold(path)} already exists; replace?`, initialValue: false}); diff --git a/src/create.ts b/src/create.ts index 9de792bb6..ef3cc8ace 100644 --- a/src/create.ts +++ b/src/create.ts @@ -1,13 +1,13 @@ import {exec} from "node:child_process"; -import {accessSync, existsSync, readdirSync, statSync} from "node:fs"; -import {constants, copyFile, mkdir, readFile, readdir, stat, writeFile} from "node:fs/promises"; -import {basename, dirname, join, normalize, resolve} from "node:path"; import {setTimeout as sleep} from "node:timers/promises"; import {fileURLToPath} from "node:url"; import {promisify} from "node:util"; import * as clack from "@clack/prompts"; import untildify from "untildify"; import {version} from "../package.json"; +import {accessSync, existsSync, readdirSync, statSync} from "./brandedFs.js"; +import {constants, copyFile, mkdir, readFile, readdir, stat, writeFile} from "./brandedFs.js"; +import {FilePath, fileBasename, fileDirname, fileJoin, fileNormalize, fileResolve, unFilePath} from "./brandedPath.js"; import type {ClackEffects} from "./clack.js"; import {cyan, faint, inverse, link, reset} from "./tty.js"; @@ -15,9 +15,9 @@ export interface CreateEffects { clack: ClackEffects; sleep: (delay?: number) => Promise; log(output: string): void; - mkdir(outputPath: string, options?: {recursive?: boolean}): Promise; - copyFile(sourcePath: string, outputPath: string): Promise; - writeFile(outputPath: string, contents: string): Promise; + mkdir(outputPath: FilePath, options?: {recursive?: boolean}): Promise; + copyFile(sourcePath: FilePath, outputPath: FilePath): Promise; + writeFile(outputPath: FilePath, contents: string): Promise; } const defaultEffects: CreateEffects = { @@ -26,13 +26,13 @@ const defaultEffects: CreateEffects = { log(output: string): void { console.log(output); }, - async mkdir(outputPath: string, options): Promise { + async mkdir(outputPath: FilePath, options): Promise { await mkdir(outputPath, options); }, - async copyFile(sourcePath: string, outputPath: string): Promise { + async copyFile(sourcePath: FilePath, outputPath: FilePath): Promise { await copyFile(sourcePath, outputPath); }, - async writeFile(outputPath: string, contents: string): Promise { + async writeFile(outputPath: FilePath, contents: string): Promise { await writeFile(outputPath, contents); } }; @@ -46,23 +46,25 @@ const defaultEffects: CreateEffects = { export async function create(options = {}, effects: CreateEffects = defaultEffects): Promise { const {clack} = effects; clack.intro(`${inverse(" observable create ")} ${faint(`v${version}`)}`); - const defaultRootPath = "./hello-framework"; + const defaultRootPath = FilePath("./hello-framework"); const defaultRootPathError = validateRootPath(defaultRootPath); await clack.group( { rootPath: () => clack.text({ message: "Where to create your project?", - placeholder: defaultRootPath, - defaultValue: defaultRootPathError ? undefined : defaultRootPath, - validate: (input) => validateRootPath(input, defaultRootPathError) + placeholder: unFilePath(defaultRootPath), + defaultValue: defaultRootPathError ? undefined : unFilePath(defaultRootPath), + validate: (input) => validateRootPath(FilePath(input), defaultRootPathError) }), - projectTitle: ({results: {rootPath}}) => + projectTitle: ({results: {rootPath: rootPathStr}}) => { + const rootPath = FilePath(rootPathStr!); clack.text({ message: "What to title your project?", placeholder: inferTitle(rootPath!), defaultValue: inferTitle(rootPath!) - }), + }); + }, includeSampleFiles: () => clack.select({ message: "Include sample files to help you get started?", @@ -87,17 +89,16 @@ export async function create(options = {}, effects: CreateEffects = defaultEffec message: "Initialize git repository?" }), installing: async ({results: {rootPath, projectTitle, includeSampleFiles, packageManager, initializeGit}}) => { - rootPath = untildify(rootPath!); const s = clack.spinner(); s.start("Copying template files"); const template = includeSampleFiles ? "default" : "empty"; - const templateDir = resolve(fileURLToPath(import.meta.url), "..", "..", "templates", template); + const templateDir = fileResolve(fileURLToPath(import.meta.url), "..", "..", "templates", template); const runCommand = packageManager === "yarn" ? "yarn" : `${packageManager ?? "npm"} run`; const installCommand = `${packageManager ?? "npm"} install`; await effects.sleep(1000); await recursiveCopyTemplate( templateDir, - rootPath!, + FilePath(untildify(rootPath!)), { runCommand, installCommand, @@ -133,9 +134,9 @@ export async function create(options = {}, effects: CreateEffects = defaultEffec ); } -function validateRootPath(rootPath: string, defaultError?: string): string | undefined { - if (rootPath === "") return defaultError; // accept default value - rootPath = normalize(rootPath); +function validateRootPath(rootPath: FilePath, defaultError?: string): string | undefined { + if (rootPath === FilePath("")) return defaultError; // accept default value + rootPath = fileNormalize(rootPath); if (!canWriteRecursive(rootPath)) return "Path is not writable."; if (!existsSync(rootPath)) return; if (!statSync(rootPath).isDirectory()) return "File already exists."; @@ -143,14 +144,14 @@ function validateRootPath(rootPath: string, defaultError?: string): string | und if (readdirSync(rootPath).length !== 0) return "Directory is not empty."; } -function inferTitle(rootPath: string): string { - return basename(rootPath!) +function inferTitle(rootPath: FilePath): string { + return fileBasename(rootPath!) .split(/[-_\s]/) .map(([c, ...rest]) => c.toUpperCase() + rest.join("")) .join(" "); } -function canWrite(path: string): boolean { +function canWrite(path: FilePath): boolean { try { accessSync(path, constants.W_OK); return true; @@ -159,9 +160,9 @@ function canWrite(path: string): boolean { } } -function canWriteRecursive(path: string): boolean { +function canWriteRecursive(path: FilePath): boolean { while (true) { - const dir = dirname(path); + const dir = fileDirname(path); if (canWrite(dir)) return true; if (dir === path) break; path = dir; @@ -170,15 +171,15 @@ function canWriteRecursive(path: string): boolean { } async function recursiveCopyTemplate( - inputRoot: string, - outputRoot: string, + inputRoot: FilePath, + outputRoot: FilePath, context: Record, effects: CreateEffects, - stepPath: string = "." + stepPath: FilePath = FilePath(".") ) { - const templatePath = join(inputRoot, stepPath); + const templatePath = fileJoin(inputRoot, stepPath); const templateStat = await stat(templatePath); - let outputPath = join(outputRoot, stepPath); + let outputPath = fileJoin(outputRoot, stepPath); if (templateStat.isDirectory()) { try { await effects.mkdir(outputPath, {recursive: true}); @@ -186,7 +187,7 @@ async function recursiveCopyTemplate( // that's ok } for (const entry of await readdir(templatePath)) { - await recursiveCopyTemplate(inputRoot, outputRoot, context, effects, join(stepPath, entry)); + await recursiveCopyTemplate(inputRoot, outputRoot, context, effects, fileJoin(stepPath, entry)); } } else { if (templatePath.endsWith(".DS_Store")) return; diff --git a/src/dataloader.ts b/src/dataloader.ts index 6c80a7fc9..6e12df0c0 100644 --- a/src/dataloader.ts +++ b/src/dataloader.ts @@ -1,15 +1,17 @@ import {spawn} from "node:child_process"; -import {type WriteStream, createReadStream, existsSync, statSync} from "node:fs"; -import {mkdir, open, readFile, rename, unlink} from "node:fs/promises"; -import {dirname, extname, join} from "node:path"; +import {type WriteStream} from "node:fs"; import {createGunzip} from "node:zlib"; import JSZip from "jszip"; import {extract} from "tar-stream"; +import {createReadStream, existsSync, statSync} from "./brandedFs.js"; +import {mkdir, open, readFile, rename, unlink} from "./brandedFs.js"; +import {FilePath, fileExtname} from "./brandedPath.js"; +import {fileDirname, fileJoin, unFilePath} from "./brandedPath.js"; import {maybeStat, prepareOutput} from "./files.js"; import type {Logger, Writer} from "./logger.js"; import {cyan, faint, green, red, yellow} from "./tty.js"; -const runningCommands = new Map>(); +const runningCommands = new Map>(); const languages = { ".js": ["node", "--no-warnings=ExperimentalWarning"], @@ -34,9 +36,9 @@ const defaultEffects: LoadEffects = { }; export interface LoaderOptions { - path: string; - sourceRoot: string; - targetPath: string; + path: FilePath; + sourceRoot: FilePath; + targetPath: FilePath; useStale: boolean; } @@ -46,19 +48,19 @@ export abstract class Loader { * directory. This is exposed so that clients can check which file to watch to * see if the loader is edited (and in which case it needs to be re-run). */ - readonly path: string; + readonly path: FilePath; /** * The source root relative to the current working directory, such as docs. */ - readonly sourceRoot: string; + readonly sourceRoot: FilePath; /** * The path to the loader script’s output relative to the destination root. * This is where the loader’s output is served, but the loader generates the * file in the .observablehq/cache directory within the source root. */ - readonly targetPath: string; + readonly targetPath: FilePath; /** * Should the loader use a stale cache. true when building. @@ -79,23 +81,23 @@ export abstract class Loader { * abort if we find a matching folder or reach the source root; for example, * if docs/data exists, we won’t look for a docs/data.zip. */ - static find(sourceRoot: string, targetPath: string, {useStale = false} = {}): Loader | undefined { + static find(sourceRoot: FilePath, targetPath: FilePath, {useStale = false} = {}): Loader | undefined { const exact = this.findExact(sourceRoot, targetPath, {useStale}); if (exact) return exact; - let dir = dirname(targetPath); - for (let parent: string; true; dir = parent) { - parent = dirname(dir); + let dir = fileDirname(targetPath); + for (let parent: FilePath; true; dir = parent) { + parent = fileDirname(dir); if (parent === dir) return; // reached source root - if (existsSync(join(sourceRoot, dir))) return; // found folder - if (existsSync(join(sourceRoot, parent))) break; // found parent + if (existsSync(fileJoin(sourceRoot, dir))) return; // found folder + if (existsSync(fileJoin(sourceRoot, parent))) break; // found parent } for (const [ext, Extractor] of extractors) { - const archive = dir + ext; - if (existsSync(join(sourceRoot, archive))) { + const archive = FilePath(dir + ext); + if (existsSync(fileJoin(sourceRoot, archive))) { return new Extractor({ preload: async () => archive, inflatePath: targetPath.slice(archive.length - ext.length + 1), - path: join(sourceRoot, archive), + path: fileJoin(sourceRoot, archive), sourceRoot, targetPath, useStale @@ -115,17 +117,17 @@ export abstract class Loader { } } - private static findExact(sourceRoot: string, targetPath: string, {useStale}): Loader | undefined { + private static findExact(sourceRoot: FilePath, targetPath: FilePath, {useStale}): Loader | undefined { for (const [ext, [command, ...args]] of Object.entries(languages)) { - if (!existsSync(join(sourceRoot, targetPath + ext))) continue; - if (extname(targetPath) === "") { + if (!existsSync(fileJoin(sourceRoot, targetPath + ext))) continue; + if (fileExtname(targetPath) === "") { console.warn(`invalid data loader path: ${targetPath + ext}`); return; } - const path = join(sourceRoot, targetPath + ext); + const path = fileJoin(sourceRoot, targetPath + ext); return new CommandLoader({ command: command ?? path, - args: command == null ? args : [...args, path], + args: command == null ? args : [...args, unFilePath(path)], path, sourceRoot, targetPath, @@ -139,13 +141,13 @@ export abstract class Loader { * to the source root; this is within the .observablehq/cache folder within * the source root. */ - async load(effects = defaultEffects): Promise { - const key = join(this.sourceRoot, this.targetPath); + async load(effects = defaultEffects): Promise { + const key = fileJoin(this.sourceRoot, this.targetPath); let command = runningCommands.get(key); if (!command) { command = (async () => { - const outputPath = join(".observablehq", "cache", this.targetPath); - const cachePath = join(this.sourceRoot, outputPath); + const outputPath = fileJoin(".observablehq", "cache", this.targetPath); + const cachePath = fileJoin(this.sourceRoot, outputPath); const loaderStat = await maybeStat(this.path); const cacheStat = await maybeStat(cachePath); if (!cacheStat) effects.output.write(faint("[missing] ")); @@ -153,8 +155,8 @@ export abstract class Loader { if (this.useStale) return effects.output.write(faint("[using stale] ")), outputPath; else effects.output.write(faint("[stale] ")); } else return effects.output.write(faint("[fresh] ")), outputPath; - const tempPath = join(this.sourceRoot, ".observablehq", "cache", `${this.targetPath}.${process.pid}`); - const errorPath = tempPath + ".err"; + const tempPath = fileJoin(this.sourceRoot, ".observablehq", "cache", `${this.targetPath}.${process.pid}`); + const errorPath = FilePath(tempPath + ".err"); const errorStat = await maybeStat(errorPath); if (errorStat) { if (errorStat.mtimeMs > loaderStat!.mtimeMs && errorStat.mtimeMs > -1000 + Date.now()) @@ -165,7 +167,7 @@ export abstract class Loader { const tempFd = await open(tempPath, "w"); try { await this.exec(tempFd.createWriteStream({highWaterMark: 1024 * 1024}), effects); - await mkdir(dirname(cachePath), {recursive: true}); + await mkdir(fileDirname(cachePath), {recursive: true}); await rename(tempPath, cachePath); } catch (error) { await rename(tempPath, errorPath); @@ -183,7 +185,7 @@ export abstract class Loader { command.then( (path) => { effects.logger.log( - `${green("success")} ${cyan(formatSize(statSync(join(this.sourceRoot, path)).size))} ${faint( + `${green("success")} ${cyan(formatSize(statSync(fileJoin(this.sourceRoot, path)).size))} ${faint( `in ${formatElapsed(start)}` )}` ); @@ -239,12 +241,12 @@ class CommandLoader extends Loader { interface ZipExtractorOptions extends LoaderOptions { preload: Loader["load"]; - inflatePath: string; + inflatePath: FilePath; } class ZipExtractor extends Loader { private readonly preload: Loader["load"]; - private readonly inflatePath: string; + private readonly inflatePath: FilePath; constructor({preload, inflatePath, ...options}: ZipExtractorOptions) { super(options); @@ -253,8 +255,8 @@ class ZipExtractor extends Loader { } async exec(output: WriteStream, effects?: LoadEffects): Promise { - const archivePath = join(this.sourceRoot, await this.preload(effects)); - const file = (await JSZip.loadAsync(await readFile(archivePath))).file(this.inflatePath); + const archivePath = fileJoin(this.sourceRoot, await this.preload(effects)); + const file = (await JSZip.loadAsync(await readFile(archivePath))).file(unFilePath(this.inflatePath)); if (!file) throw Object.assign(new Error("file not found"), {code: "ENOENT"}); const pipe = file.nodeStream().pipe(output); await new Promise((resolve, reject) => pipe.on("error", reject).on("finish", resolve)); @@ -263,13 +265,13 @@ class ZipExtractor extends Loader { interface TarExtractorOptions extends LoaderOptions { preload: Loader["load"]; - inflatePath: string; + inflatePath: FilePath; gunzip?: boolean; } class TarExtractor extends Loader { private readonly preload: Loader["load"]; - private readonly inflatePath: string; + private readonly inflatePath: FilePath; private readonly gunzip: boolean; constructor({preload, inflatePath, gunzip = false, ...options}: TarExtractorOptions) { @@ -280,12 +282,12 @@ class TarExtractor extends Loader { } async exec(output: WriteStream, effects?: LoadEffects): Promise { - const archivePath = join(this.sourceRoot, await this.preload(effects)); + const archivePath = fileJoin(this.sourceRoot, await this.preload(effects)); const tar = extract(); const input = createReadStream(archivePath); (this.gunzip ? input.pipe(createGunzip()) : input).pipe(tar); for await (const entry of tar) { - if (entry.header.name === this.inflatePath) { + if (FilePath(entry.header.name) === this.inflatePath) { const pipe = entry.pipe(output); await new Promise((resolve, reject) => pipe.on("error", reject).on("finish", resolve)); return; diff --git a/src/deploy.ts b/src/deploy.ts index 57acc1c8c..c72ba54da 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -1,6 +1,7 @@ -import {join} from "node:path"; import * as clack from "@clack/prompts"; import wrapAnsi from "wrap-ansi"; +import {existsSync, readFile} from "./brandedFs.js"; +import {type FilePath, fileJoin, filePathToUrlPath} from "./brandedPath.js"; import type {BuildEffects} from "./build.js"; import {build} from "./build.js"; import type {ClackEffects} from "./clack.js"; @@ -39,8 +40,8 @@ export interface DeployOptions { } export interface DeployEffects extends ConfigEffects, TtyEffects, AuthEffects { - getDeployConfig: (sourceRoot: string) => Promise; - setDeployConfig: (sourceRoot: string, config: DeployConfig) => Promise; + getDeployConfig: (sourceRoot: FilePath) => Promise; + setDeployConfig: (sourceRoot: FilePath, config: DeployConfig) => Promise; clack: ClackEffects; logger: Logger; input: NodeJS.ReadableStream; @@ -78,14 +79,16 @@ export async function deploy( if (deployConfig.workspaceLogin && !deployConfig.workspaceLogin.match(/^@?[a-z0-9-]+$/)) { throw new CliError( - `Found invalid workspace login in ${join(config.root, ".observablehq", "deploy.json")}: ${ + `Found invalid workspace login in ${fileJoin(config.root, ".observablehq", "deploy.json")}: ${ deployConfig.workspaceLogin }.` ); } if (deployConfig.projectSlug && !deployConfig.projectSlug.match(/^[a-z0-9-]+$/)) { throw new CliError( - `Found invalid project slug in ${join(config.root, ".observablehq", "deploy.json")}: ${deployConfig.projectSlug}.` + `Found invalid project slug in ${fileJoin(config.root, ".observablehq", "deploy.json")}: ${ + deployConfig.projectSlug + }.` ); } @@ -348,10 +351,10 @@ class DeployBuildEffects implements BuildEffects { this.logger = effects.logger; this.output = effects.output; } - async copyFile(sourcePath: string, outputPath: string) { + async copyFile(sourcePath: FilePath, outputPath: FilePath) { this.logger.log(outputPath); try { - await this.apiClient.postDeployFile(this.deployId, sourcePath, outputPath); + await this.apiClient.postDeployFile(this.deployId, sourcePath, filePathToUrlPath(outputPath)); } catch (error) { if (isApiError(error) && error.details.errors.some((e) => e.code === "FILE_QUOTA_EXCEEDED")) { throw new CliError("You have reached the total file size limit.", {cause: error}); @@ -364,10 +367,10 @@ class DeployBuildEffects implements BuildEffects { throw error; } } - async writeFile(outputPath: string, content: Buffer | string) { + async writeFile(outputPath: FilePath, content: Buffer | string) { this.logger.log(outputPath); try { - await this.apiClient.postDeployFileContents(this.deployId, content, outputPath); + await this.apiClient.postDeployFileContents(this.deployId, content, filePathToUrlPath(outputPath)); } catch (error) { if (isApiError(error) && error.details.errors.some((e) => e.code === "FILE_QUOTA_EXCEEDED")) { throw new CliError("You have reached the total file size limit.", {cause: error}); @@ -375,6 +378,14 @@ class DeployBuildEffects implements BuildEffects { throw error; } } + existsSync(path: FilePath) { + return existsSync(path); + } + readFile(path: FilePath): Promise; + readFile(path: FilePath, encoding: BufferEncoding): Promise; + readFile(path: FilePath, encoding?: BufferEncoding): Promise { + return encoding ? readFile(path, encoding) : readFile(path); + } } // export for testing diff --git a/src/fileWatchers.ts b/src/fileWatchers.ts index 96fd1a564..d5cec18ac 100644 --- a/src/fileWatchers.ts +++ b/src/fileWatchers.ts @@ -1,4 +1,6 @@ -import {type FSWatcher, existsSync, watch} from "node:fs"; +import {type FSWatcher} from "node:fs"; +import {existsSync, watch} from "./brandedFs.js"; +import {type FilePath, filePathToUrlPath} from "./brandedPath.js"; import {Loader} from "./dataloader.js"; import {isEnoent} from "./error.js"; import {maybeStat} from "./files.js"; @@ -7,10 +9,11 @@ import {resolvePath} from "./url.js"; export class FileWatchers { private readonly watchers: FSWatcher[] = []; - static async of(root: string, path: string, names: string[], callback: (name: string) => void) { + static async of(root: FilePath, path: FilePath, names: FilePath[], callback: (name: FilePath) => void) { const that = new FileWatchers(); const {watchers} = that; - for (const name of new Set(names)) { + for (const fileName of new Set(names)) { + const name = filePathToUrlPath(fileName); const exactPath = resolvePath(root, path, name); const watchPath = existsSync(exactPath) ? exactPath : Loader.find(root, resolvePath(path, name))?.path; if (!watchPath) continue; @@ -36,7 +39,7 @@ export class FileWatchers { // Ignore if the file was truncated or not modified. if (currentStat?.mtimeMs === newStat?.mtimeMs || newStat?.size === 0) return; currentStat = newStat; - callback(name); + callback(fileName); }); } catch (error) { if (!isEnoent(error)) throw error; diff --git a/src/files.ts b/src/files.ts index 2da58e669..23e872594 100644 --- a/src/files.ts +++ b/src/files.ts @@ -1,65 +1,70 @@ -import {type Stats, existsSync} from "node:fs"; -import {mkdir, readdir, stat} from "node:fs/promises"; -import {dirname, extname, join, normalize, relative} from "node:path"; +import {type Stats} from "node:fs"; import {cwd} from "node:process"; import {fileURLToPath} from "node:url"; import mime from "mime"; +import {existsSync, mkdir, readdir, stat} from "./brandedFs.js"; +import {UrlPath, urlJoin, urlPathToFilePath} from "./brandedPath.js"; +import {FilePath, fileDirname, fileExtname, fileJoin, fileNormalize, fileRelative, unUrlPath} from "./brandedPath.js"; import {isEnoent} from "./error.js"; import type {FileReference} from "./javascript.js"; import {relativeUrl, resolvePath} from "./url.js"; // A path is local if it doesn’t go outside the the root. -export function getLocalPath(sourcePath: string, name: string): string | null { - if (/^\w+:/.test(name)) return null; // URL +export function getLocalPath(sourcePath: UrlPath, name: UrlPath): FilePath | null { + if (/^\w+:/.test(unUrlPath(name))) return null; // URL if (name.startsWith("#")) return null; // anchor tag - const path = resolvePath(sourcePath, name); + const path = resolvePath(urlPathToFilePath(sourcePath), name); if (path.startsWith("../")) return null; // goes above root return path; } -export function getClientPath(entry: string): string { - const path = relative(cwd(), join(dirname(fileURLToPath(import.meta.url)), "..", entry)); +export function getClientPath(entry: FilePath): FilePath { + const path = fileRelative( + FilePath(cwd()), + fileJoin(fileDirname(FilePath(fileURLToPath(import.meta.url))), "..", entry) + ); + // TODO this should use the effect version of existsSync to be more type safe if (path.endsWith(".js") && !existsSync(path)) { - const tspath = path.slice(0, -".js".length) + ".ts"; + const tspath = FilePath(path.slice(0, -".js".length) + ".ts"); if (existsSync(tspath)) return tspath; } return path; } -export function fileReference(name: string, sourcePath: string): FileReference { +export function fileReference(name: UrlPath, sourcePath: UrlPath): FileReference { return { - name: relativeUrl(sourcePath, name), - mimeType: mime.getType(name), - path: relativeUrl(sourcePath, join("_file", name)) + name: relativeUrl(sourcePath, UrlPath(name)), + mimeType: mime.getType(unUrlPath(name)), + path: relativeUrl(sourcePath, urlJoin("_file", name)) }; } -export async function* visitMarkdownFiles(root: string): AsyncGenerator { +export async function* visitMarkdownFiles(root: FilePath): AsyncGenerator { for await (const file of visitFiles(root)) { - if (extname(file) !== ".md") continue; + if (fileExtname(file) !== ".md") continue; yield file; } } -export async function* visitFiles(root: string): AsyncGenerator { +export async function* visitFiles(root: FilePath): AsyncGenerator { const visited = new Set(); - const queue: string[] = [(root = normalize(root))]; + const queue: FilePath[] = [(root = fileNormalize(root))]; for (const path of queue) { const status = await stat(path); if (status.isDirectory()) { if (visited.has(status.ino)) continue; // circular symlink visited.add(status.ino); for (const entry of await readdir(path)) { - queue.push(join(path, entry)); + queue.push(fileJoin(path, entry)); } } else { - yield relative(root, path); + yield fileRelative(root, path); } } } // Like fs.stat, but returns undefined instead of throwing ENOENT if not found. -export async function maybeStat(path: string): Promise { +export async function maybeStat(path: FilePath): Promise { try { return await stat(path); } catch (error) { @@ -67,8 +72,8 @@ export async function maybeStat(path: string): Promise { } } -export async function prepareOutput(outputPath: string): Promise { - const outputDir = dirname(outputPath); - if (outputDir === ".") return; +export async function prepareOutput(outputPath: FilePath): Promise { + const outputDir = fileDirname(outputPath); + if (outputDir === FilePath(".")) return; await mkdir(outputDir, {recursive: true}); } diff --git a/src/javascript.ts b/src/javascript.ts index 3e4192f70..eace7add0 100644 --- a/src/javascript.ts +++ b/src/javascript.ts @@ -1,5 +1,6 @@ import {Parser, tokTypes} from "acorn"; import type {Expression, Identifier, Node, Options, Program} from "acorn"; +import {type FilePath, type UrlPath, filePathToUrlPath} from "./brandedPath.js"; import {fileReference} from "./files.js"; import {findAssignments} from "./javascript/assignments.js"; import {findAwaits} from "./javascript/awaits.js"; @@ -14,21 +15,21 @@ import {red} from "./tty.js"; export interface FileReference { /** The relative path from the page to the original file (e.g., "./test.txt"). */ - name: string; + name: UrlPath; /** The MIME type, if known; derived from the file extension. */ mimeType: string | null; /** The relative path from the page to the file in _file (e.g., "../_file/sub/test.txt"). */ - path: string; + path: UrlPath; } export interface ImportReference { - name: string; + name: UrlPath; type: "global" | "local"; } export interface Feature { type: "FileAttachment"; - name: string; + name: UrlPath; } export interface BaseTranspile { @@ -51,8 +52,8 @@ export interface Transpile extends BaseTranspile { export interface ParseOptions { id: string; - root: string; - sourcePath: string; + root: FilePath; + sourcePath: FilePath; inline?: boolean; sourceLine?: number; globals?: Set; @@ -65,7 +66,7 @@ export function transpileJavaScript(input: string, options: ParseOptions): Pendi const node = parseJavaScript(input, options); const files = node.features .filter((f) => f.type === "FileAttachment") - .map(({name}) => fileReference(name, sourcePath)); + .map(({name}) => fileReference(name, filePathToUrlPath(sourcePath))); const inputs = Array.from(new Set(node.references.map((r) => r.name))); const implicitDisplay = node.expression && !inputs.includes("display") && !inputs.includes("view"); if (implicitDisplay) inputs.push("display"), (node.async = true); diff --git a/src/javascript/features.ts b/src/javascript/features.ts index 5e969567c..6c9ef50be 100644 --- a/src/javascript/features.ts +++ b/src/javascript/features.ts @@ -5,8 +5,9 @@ import type {Feature} from "../javascript.js"; import {defaultGlobals} from "./globals.js"; import {findReferences} from "./references.js"; import {syntaxError} from "./syntaxError.js"; +import {FilePath, UrlPath, filePathToUrlPath, unFilePath} from "../brandedPath.js"; -export function findFeatures(node: Node, path: string, references: Identifier[], input: string): Feature[] { +export function findFeatures(node: Node, path: FilePath, references: Identifier[], input: string): Feature[] { const featureMap = getFeatureReferenceMap(node); const features: Feature[] = []; @@ -82,7 +83,7 @@ export function getFeatureReferenceMap(node: Node): Map(urlPaths.map((urlPath) => [urlPath, urlPathToFilePath(urlPath)])); - for (const path of set) { - imports.push({type: "local", name: path}); + for (const [urlPath, filePath] of map.entries()) { + imports.push({type: "local", name: urlPath}); try { - const input = readFileSync(join(root, path), "utf-8"); + const input = readFileSync(fileJoin(root, filePath), "utf-8"); const body = Parser.parse(input, parseOptions); simple( @@ -117,22 +118,23 @@ export function parseLocalImports(root: string, paths: string[]): ImportsAndFeat ExportNamedDeclaration: findImport }, undefined, - path + filePath ); - features.push(...findImportFeatures(body, path, input)); + features.push(...findImportFeatures(body, urlPath, input)); } catch (error) { if (!isEnoent(error) && !(error instanceof SyntaxError)) throw error; } } - function findImport(node: ImportNode | ExportNode, path: string) { + function findImport(node: ImportNode | ExportNode, path: FilePath) { if (isStringLiteral(node.source)) { const value = getStringLiteralValue(node.source); if (isLocalImport(value, path)) { - set.add(resolvePath(path, value)); + const filePath = resolvePath(path, UrlPath(value)); + map.set(filePathToUrlPath(filePath), filePath); } else { - imports.push({name: value, type: "global"}); + imports.push({name: UrlPath(value), type: "global"}); // non-local imports don't need to be traversed } } @@ -141,14 +143,14 @@ export function parseLocalImports(root: string, paths: string[]): ImportsAndFeat return {imports, features}; } -export function findImportFeatures(node: Node, path: string, input: string): Feature[] { +export function findImportFeatures(node: Node, path: UrlPath, input: string): Feature[] { const featureMap = getFeatureReferenceMap(node); const features: Feature[] = []; simple(node, { CallExpression(node) { const type = featureMap.get(node.callee as Identifier); - if (type) features.push(getFeature(type, node, path, input)); + if (type) features.push(getFeature(type, node, urlPathToFilePath(path), input)); } }); @@ -156,7 +158,7 @@ export function findImportFeatures(node: Node, path: string, input: string): Fea } /** Rewrites import specifiers and FileAttachment calls in the specified ES module source. */ -export async function rewriteModule(input: string, path: string, resolver: ImportResolver): Promise { +export async function rewriteModule(input: string, path: FilePath, resolver: ImportResolver): Promise { const body = Parser.parse(input, parseOptions); const featureMap = getFeatureReferenceMap(body); const output = new Sourcemap(input); @@ -173,7 +175,9 @@ export async function rewriteModule(input: string, path: string, resolver: Impor const feature = getFeature(type, node, path, input); // validate syntax if (feature.type === "FileAttachment") { const arg = node.arguments[0]; - const result = JSON.stringify(relativeUrl(join("_import", path), feature.name)); + const result = JSON.stringify( + relativeUrl(urlJoin("_import", filePathToUrlPath(path)), UrlPath(feature.name)) + ); output.replaceLeft(arg.start, arg.end, `${result}, import.meta.url`); } } @@ -218,7 +222,7 @@ export function findImportDeclarations(cell: JavaScriptNode): ImportDeclaration[ export async function rewriteImports( output: Sourcemap, cell: JavaScriptNode, - sourcePath: string, + sourcePath: FilePath, resolver: ImportResolver ): Promise { const expressions: ImportExpression[] = []; @@ -264,35 +268,39 @@ export async function rewriteImports( } } -export type ImportResolver = (path: string, specifier: string) => Promise; +export type ImportResolver = (path: FilePath, specifier: string) => Promise; -export function createImportResolver(root: string, base: "." | "_import" = "."): ImportResolver { +export function createImportResolver(root: FilePath, base: "." | "_import" = "."): ImportResolver { return async (path, specifier) => { + const urlPath = filePathToUrlPath(path); return isLocalImport(specifier, path) - ? relativeUrl(path, resolvePath(base, path, resolveImportHash(root, path, specifier))) + ? relativeUrl( + urlPath, + filePathToUrlPath(resolvePath(FilePath(base), path, resolveImportHash(root, urlPath, specifier))) + ) : specifier === "npm:@observablehq/runtime" - ? resolveBuiltin(base, path, "runtime.js") + ? resolveBuiltin(base, urlPath, "runtime.js") : specifier === "npm:@observablehq/stdlib" - ? resolveBuiltin(base, path, "stdlib.js") + ? resolveBuiltin(base, urlPath, "stdlib.js") : specifier === "npm:@observablehq/dot" - ? resolveBuiltin(base, path, "stdlib/dot.js") // TODO publish to npm + ? resolveBuiltin(base, urlPath, "stdlib/dot.js") // TODO publish to npm : specifier === "npm:@observablehq/duckdb" - ? resolveBuiltin(base, path, "stdlib/duckdb.js") // TODO publish to npm + ? resolveBuiltin(base, urlPath, "stdlib/duckdb.js") // TODO publish to npm : specifier === "npm:@observablehq/inputs" - ? resolveBuiltin(base, path, "stdlib/inputs.js") // TODO publish to npm + ? resolveBuiltin(base, urlPath, "stdlib/inputs.js") // TODO publish to npm : specifier === "npm:@observablehq/mermaid" - ? resolveBuiltin(base, path, "stdlib/mermaid.js") // TODO publish to npm + ? resolveBuiltin(base, urlPath, "stdlib/mermaid.js") // TODO publish to npm : specifier === "npm:@observablehq/tex" - ? resolveBuiltin(base, path, "stdlib/tex.js") // TODO publish to npm + ? resolveBuiltin(base, urlPath, "stdlib/tex.js") // TODO publish to npm : specifier === "npm:@observablehq/sqlite" - ? resolveBuiltin(base, path, "stdlib/sqlite.js") // TODO publish to npm + ? resolveBuiltin(base, urlPath, "stdlib/sqlite.js") // TODO publish to npm : specifier === "npm:@observablehq/xlsx" - ? resolveBuiltin(base, path, "stdlib/xlsx.js") // TODO publish to npm + ? resolveBuiltin(base, urlPath, "stdlib/xlsx.js") // TODO publish to npm : specifier === "npm:@observablehq/zip" - ? resolveBuiltin(base, path, "stdlib/zip.js") // TODO publish to npm + ? resolveBuiltin(base, urlPath, "stdlib/zip.js") // TODO publish to npm : specifier.startsWith("npm:") ? await resolveNpmImport(specifier.slice("npm:".length)) - : specifier; + : UrlPath(specifier); }; } @@ -313,14 +321,14 @@ function formatNpmSpecifier({name, range, path}: {name: string; range?: string; // Like import, don’t fetch the same package more than once to ensure // consistency; restart the server if you want to clear the cache. -const fetchCache = new Map>(); +const fetchCache = new Map>(); -async function cachedFetch(href: string): Promise<{headers: Headers; body: any}> { +async function cachedFetch(href: UrlPath): Promise<{headers: Headers; body: any}> { if (!remoteModulePreloadEnabled) throw new Error("remote module preload is not enabled"); let promise = fetchCache.get(href); if (promise) return promise; promise = (async () => { - const response = await fetch(href); + const response = await fetch(unUrlPath(href)); if (!response.ok) throw new Error(`unable to fetch: ${href}`); const json = /^application\/json(;|$)/.test(response.headers.get("content-type")!); const body = await (json ? response.json() : response.text()); @@ -336,24 +344,25 @@ async function resolveNpmVersion({name, range}: {name: string; range?: string}): if (range && /^\d+\.\d+\.\d+([-+].*)?$/.test(range)) return range; // exact version specified const specifier = formatNpmSpecifier({name, range}); const search = range ? `?specifier=${range}` : ""; - const {version} = (await cachedFetch(`https://data.jsdelivr.com/v1/packages/npm/${name}/resolved${search}`)).body; + const {version} = (await cachedFetch(UrlPath(`https://data.jsdelivr.com/v1/packages/npm/${name}/resolved${search}`))) + .body; if (!version) throw new Error(`unable to resolve version: ${specifier}`); return version; } -export async function resolveNpmImport(specifier: string): Promise { +export async function resolveNpmImport(specifier: string): Promise { let {name, range, path = "+esm"} = parseNpmSpecifier(specifier); // eslint-disable-line prefer-const if (name === "@duckdb/duckdb-wasm" && !range) range = "1.28.0"; // https://github.com/duckdb/duckdb-wasm/issues/1561 if (name === "apache-arrow" && !range) range = "13.0.0"; // https://github.com/observablehq/framework/issues/750 if (name === "parquet-wasm" && !range) range = "0.5.0"; // https://github.com/observablehq/framework/issues/733 try { - return `https://cdn.jsdelivr.net/npm/${name}@${await resolveNpmVersion({name, range})}/${path}`; + return UrlPath(`https://cdn.jsdelivr.net/npm/${name}@${await resolveNpmVersion({name, range})}/${path}`); } catch { - return `https://cdn.jsdelivr.net/npm/${name}${range ? `@${range}` : ""}/${path}`; + return UrlPath(`https://cdn.jsdelivr.net/npm/${name}${range ? `@${range}` : ""}/${path}`); } } -const preloadCache = new Map | undefined>>(); +const preloadCache = new Map | undefined>>(); /** * Fetches the module at the specified URL and returns a promise to any @@ -362,7 +371,7 @@ const preloadCache = new Map | undefined>>(); * are considered, and the fetched module must be have immutable public caching; * dynamic imports may not be used and hence are not preloaded. */ -async function fetchModulePreloads(href: string): Promise | undefined> { +async function fetchModulePreloads(href: UrlPath): Promise | undefined> { let promise = preloadCache.get(href); if (promise) return promise; promise = (async () => { @@ -375,7 +384,7 @@ async function fetchModulePreloads(href: string): Promise | undefine const {headers, body} = response; const cache = headers.get("cache-control")?.split(/\s*,\s*/); if (!cache?.some((c) => c === "immutable") || !cache?.some((c) => c === "public")) return; - const imports = new Set(); + const imports = new Set(); let program: Program; try { program = Parser.parse(body, parseOptions); @@ -391,7 +400,7 @@ async function fetchModulePreloads(href: string): Promise | undefine function findImport(node: ImportNode | ExportNode) { if (isStringLiteral(node.source)) { const value = getStringLiteralValue(node.source); - if (isPathImport(value)) imports.add(String(new URL(value, href))); + if (isPathImport(value)) imports.add(UrlPath(String(new URL(value, unUrlPath(href))))); } } // TODO integrityCache.set(href, `sha384-${createHash("sha384").update(body).digest("base64")}`); @@ -409,10 +418,10 @@ const integrityCache = new Map(); * externally-hosted modules to compute the transitively-imported modules; also * precomputes the subresource integrity hash for each fetched module. */ -export async function resolveModulePreloads(hrefs: Set): Promise { +export async function resolveModulePreloads(hrefs: Set): Promise { if (!remoteModulePreloadEnabled) return; let resolve: () => void; - const visited = new Set(); + const visited = new Set(); const queue = new Set>(); for (const href of hrefs) { @@ -421,7 +430,7 @@ export async function resolveModulePreloads(hrefs: Set): Promise { } } - function enqueue(href: string) { + function enqueue(href: UrlPath) { if (visited.has(href)) return; visited.add(href); const promise = (async () => { @@ -450,16 +459,21 @@ export function resolveModuleIntegrity(href: string): string | undefined { return integrityCache.get(href); } -function resolveBuiltin(base: "." | "_import", path: string, specifier: string): string { - return relativeUrl(join(base === "." ? "_import" : ".", path), join("_observablehq", specifier)); +function resolveBuiltin(base: "." | "_import", path: UrlPath, specifier: string): UrlPath { + return relativeUrl(urlJoin(base === "." ? "_import" : ".", path), urlJoin("_observablehq", specifier)); } /** * Given the specified local import, applies the ?sha query string based on the * content hash of the imported module and its transitively imported modules. */ -function resolveImportHash(root: string, path: string, specifier: string): string { - return `${specifier}?sha=${getModuleHash(root, resolvePath(path, specifier))}`; +function resolveImportHash(root: FilePath, path: UrlPath, specifier: string): UrlPath { + return UrlPath( + `${specifier}?sha=${getModuleHash( + root, + filePathToUrlPath(resolvePath(urlPathToFilePath(path), UrlPath(specifier))) + )}` + ); } /** @@ -467,10 +481,11 @@ function resolveImportHash(root: string, path: string, specifier: string): strin * given source root. This involves parsing the specified module to process * transitive imports. */ -function getModuleHash(root: string, path: string): string { +function getModuleHash(root: FilePath, path: UrlPath): string { const hash = createHash("sha256"); + const filePath = urlPathToFilePath(path); try { - hash.update(readFileSync(join(root, path), "utf-8")); + hash.update(readFileSync(fileJoin(root, filePath), "utf-8")); } catch (error) { if (!isEnoent(error)) throw error; } @@ -479,7 +494,7 @@ function getModuleHash(root: string, path: string): string { for (const i of [...imports, ...features]) { if (i.type === "local" || i.type === "FileAttachment") { try { - hash.update(readFileSync(join(root, i.name), "utf-8")); + hash.update(readFileSync(fileJoin(root, urlPathToFilePath(UrlPath(i.name))), "utf-8")); } catch (error) { if (!isEnoent(error)) throw error; continue; @@ -501,8 +516,8 @@ export function isPathImport(specifier: string): boolean { return ["./", "../", "/"].some((prefix) => specifier.startsWith(prefix)); } -export function isLocalImport(specifier: string, path: string): boolean { - return isPathImport(specifier) && !resolvePath(path, specifier).startsWith("../"); +export function isLocalImport(specifier: string, path: FilePath): boolean { + return isPathImport(specifier) && !resolvePath(path, UrlPath(specifier)).startsWith("../"); } function isNamespaceSpecifier(node) { diff --git a/src/libraries.ts b/src/libraries.ts index 54da0f3c7..29b007c24 100644 --- a/src/libraries.ts +++ b/src/libraries.ts @@ -1,3 +1,4 @@ +import {UrlPath} from "./brandedPath.js"; import {resolveNpmImport} from "./javascript/imports.js"; export function getImplicitSpecifiers(inputs: Set): Set { @@ -25,12 +26,15 @@ export function addImplicitSpecifiers(specifiers: Set, inputs: Set): Promise> { +export async function getImplicitStylesheets(specifiers: Set): Promise> { return addImplicitStylesheets(new Set(), specifiers); } -export async function addImplicitStylesheets(stylesheets: Set, specifiers: Set): Promise> { - if (specifiers.has("npm:@observablehq/inputs")) stylesheets.add("observablehq:stdlib/inputs.css"); +export async function addImplicitStylesheets( + stylesheets: Set, + specifiers: Set +): Promise> { + if (specifiers.has("npm:@observablehq/inputs")) stylesheets.add(UrlPath("observablehq:stdlib/inputs.css")); if (specifiers.has("npm:katex")) stylesheets.add(await resolveNpmImport("katex/dist/katex.min.css")); if (specifiers.has("npm:leaflet")) stylesheets.add(await resolveNpmImport("leaflet/dist/leaflet.css")); if (specifiers.has("npm:mapbox-gl")) stylesheets.add(await resolveNpmImport("mapbox-gl/dist/mapbox-gl.css")); diff --git a/src/markdown.ts b/src/markdown.ts index 570acb452..6fa453474 100644 --- a/src/markdown.ts +++ b/src/markdown.ts @@ -1,5 +1,4 @@ import {createHash} from "node:crypto"; -import {readFile} from "node:fs/promises"; import {type Patch, type PatchItem, getPatch} from "fast-array-diff"; import equal from "fast-deep-equal"; import matter from "gray-matter"; @@ -10,6 +9,8 @@ import {type RuleCore} from "markdown-it/lib/parser_core.js"; import {type RuleInline} from "markdown-it/lib/parser_inline.js"; import {type RenderRule, type default as Renderer} from "markdown-it/lib/renderer.js"; import MarkdownItAnchor from "markdown-it-anchor"; +import {readFile} from "./brandedFs.js"; +import {type FilePath, UrlPath, filePathToUrlPath, unUrlPath, urlPathToFilePath} from "./brandedPath.js"; import {isEnoent} from "./error.js"; import {fileReference, getLocalPath} from "./files.js"; import {computeHash} from "./hash.js"; @@ -98,7 +99,7 @@ function getLiveSource(content: string, tag: string): string | undefined { : undefined; } -function makeFenceRenderer(root: string, baseRenderer: RenderRule, sourcePath: string): RenderRule { +function makeFenceRenderer(root: FilePath, baseRenderer: RenderRule, sourcePath: FilePath): RenderRule { return (tokens, idx, options, context: ParseContext, self) => { const token = tokens[idx]; const {tag, attributes} = parseInfo(token.info); @@ -262,7 +263,7 @@ const transformPlaceholderCore: RuleCore = (state) => { state.tokens = output; }; -function makePlaceholderRenderer(root: string, sourcePath: string): RenderRule { +function makePlaceholderRenderer(root: FilePath, sourcePath: FilePath): RenderRule { return (tokens, idx, options, context: ParseContext) => { const id = uniqueCodeId(context, tokens[idx].content); const token = tokens[idx]; @@ -296,7 +297,7 @@ function extendPiece(context: ParseContext, extend: Partial) { }; } -function renderIntoPieces(renderer: Renderer, root: string, sourcePath: string): Renderer["render"] { +function renderIntoPieces(renderer: Renderer, root: FilePath, sourcePath: FilePath): Renderer["render"] { return (tokens, options, context: ParseContext) => { const rules = renderer.rules; for (let i = 0, len = tokens.length; i < len; i++) { @@ -334,15 +335,15 @@ const SUPPORTED_PROPERTIES: readonly {query: string; src: "href" | "src" | "srcs {query: "video source[src]", src: "src"} ]); -export function normalizePieceHtml(html: string, sourcePath: string, context: ParseContext): string { +export function normalizePieceHtml(html: string, sourcePath: FilePath, context: ParseContext): string { const {document} = parseHTML(html); // Extracting references to files (such as from linked stylesheets). const filePaths = new Set(); - const resolvePath = (source: string): FileReference | undefined => { - const path = getLocalPath(sourcePath, source); + const resolvePath = (source: UrlPath): FileReference | undefined => { + const path = getLocalPath(filePathToUrlPath(sourcePath), source); if (!path) return; - const file = fileReference(path, sourcePath); + const file = fileReference(filePathToUrlPath(path), filePathToUrlPath(sourcePath)); if (!filePaths.has(file.path)) { filePaths.add(file.path); context.files.push(file); @@ -357,16 +358,16 @@ export function normalizePieceHtml(html: string, sourcePath: string, context: Pa .split(",") .map((p) => { const parts = p.trim().split(/\s+/); - const source = decodeURIComponent(parts[0]); + const source = UrlPath(decodeURIComponent(parts[0])); const file = resolvePath(source); return file ? `${file.path} ${parts.slice(1).join(" ")}`.trim() : parts.join(" "); }) .filter((p) => !!p); if (paths && paths.length > 0) element.setAttribute(src, paths.join(", ")); } else { - const source = decodeURIComponent(element.getAttribute(src)!); + const source = UrlPath(decodeURIComponent(element.getAttribute(src)!)); const file = resolvePath(source); - if (file) element.setAttribute(src, file.path); + if (file) element.setAttribute(src, unUrlPath(file.path)); } } } @@ -413,21 +414,21 @@ async function toParseCells(pieces: RenderPiece[]): Promise { } export interface ParseOptions { - root: string; - path: string; + root: FilePath; + path: UrlPath; } -export async function parseMarkdown(sourcePath: string, {root, path}: ParseOptions): Promise { +export async function parseMarkdown(sourcePath: FilePath, {root, path}: ParseOptions): Promise { const source = await readFile(sourcePath, "utf-8"); const parts = matter(source, {}); const md = MarkdownIt({html: true}); md.use(MarkdownItAnchor, {permalink: MarkdownItAnchor.permalink.headerLink({class: "observablehq-header-anchor"})}); md.inline.ruler.push("placeholder", transformPlaceholderInline); md.core.ruler.before("linkify", "placeholder", transformPlaceholderCore); - md.renderer.rules.placeholder = makePlaceholderRenderer(root, path); - md.renderer.rules.fence = makeFenceRenderer(root, md.renderer.rules.fence!, path); + md.renderer.rules.placeholder = makePlaceholderRenderer(root, urlPathToFilePath(path)); + md.renderer.rules.fence = makeFenceRenderer(root, md.renderer.rules.fence!, urlPathToFilePath(path)); md.renderer.rules.softbreak = makeSoftbreakRenderer(md.renderer.rules.softbreak!); - md.renderer.render = renderIntoPieces(md.renderer, root, path); + md.renderer.render = renderIntoPieces(md.renderer, root, urlPathToFilePath(path)); const context: ParseContext = {files: [], imports: [], pieces: [], startLine: 0, currentLine: 0}; const tokens = md.parse(parts.content, context); const html = md.renderer.render(tokens, md.options, context); // Note: mutates context.pieces, context.files! @@ -439,14 +440,14 @@ export async function parseMarkdown(sourcePath: string, {root, path}: ParseOptio imports: context.imports, pieces: toParsePieces(context.pieces), cells: await toParseCells(context.pieces), - hash: await computeMarkdownHash(source, root, path, context.imports) + hash: await computeMarkdownHash(source, root, urlPathToFilePath(path), context.imports) }; } async function computeMarkdownHash( contents: string, - root: string, - path: string, + root: FilePath, + path: FilePath, imports: ImportReference[] ): Promise { const hash = createHash("sha256").update(contents); diff --git a/src/observableApiClient.ts b/src/observableApiClient.ts index 0ba23f62c..d8f21b0e8 100644 --- a/src/observableApiClient.ts +++ b/src/observableApiClient.ts @@ -1,5 +1,6 @@ -import fs from "node:fs/promises"; import packageJson from "../package.json"; +import {readFile} from "./brandedFs.js"; +import {unUrlPath, type FilePath, type UrlPath} from "./brandedPath.js"; import {CliError, HttpError, isApiError} from "./error.js"; import type {ApiKey} from "./observableApiConfig.js"; import {faint, red} from "./tty.js"; @@ -145,18 +146,18 @@ export class ObservableApiClient { return data.id; } - async postDeployFile(deployId: string, filePath: string, relativePath: string): Promise { - const buffer = await fs.readFile(filePath); + async postDeployFile(deployId: string, filePath: FilePath, relativePath: UrlPath): Promise { + const buffer = await readFile(filePath); return await this.postDeployFileContents(deployId, buffer, relativePath); } - async postDeployFileContents(deployId: string, contents: Buffer | string, relativePath: string): Promise { + async postDeployFileContents(deployId: string, contents: Buffer | string, relativePath: UrlPath): Promise { if (typeof contents === "string") contents = Buffer.from(contents); const url = new URL(`/cli/deploy/${deployId}/file`, this._apiOrigin); const body = new FormData(); const blob = new Blob([contents]); body.append("file", blob); - body.append("client_name", relativePath); + body.append("client_name", unUrlPath(relativePath)); await this._fetch(url, {method: "POST", body}); } diff --git a/src/observableApiConfig.ts b/src/observableApiConfig.ts index 54738bd0d..48285a41c 100644 --- a/src/observableApiConfig.ts +++ b/src/observableApiConfig.ts @@ -1,26 +1,26 @@ -import fs from "node:fs/promises"; import os from "node:os"; -import path from "node:path"; +import {mkdir, readFile, writeFile} from "./brandedFs.js"; +import {FilePath, fileDirname, fileJoin, fileResolve} from "./brandedPath.js"; import {CliError, isEnoent} from "./error.js"; export interface ConfigEffects { - readFile: (path: string, encoding: "utf8") => Promise; - writeFile: (path: string, contents: string) => Promise; + readFile: (path: FilePath, encoding: "utf8") => Promise; + writeFile: (path: FilePath, contents: string) => Promise; env: typeof process.env; - cwd: typeof process.cwd; - mkdir: (path: string, options?: {recursive?: boolean}) => Promise; - homedir: typeof os.homedir; + cwd: () => FilePath; + mkdir: (path: FilePath, options?: {recursive?: boolean}) => Promise; + homedir: () => FilePath; } export const defaultEffects: ConfigEffects = { - readFile: (path, encoding) => fs.readFile(path, encoding), - writeFile: fs.writeFile, + readFile: (path, encoding) => readFile(path, encoding), + writeFile, mkdir: async (path, options) => { - await fs.mkdir(path, options); + await mkdir(path, options); }, env: process.env, - cwd: process.cwd, - homedir: os.homedir + cwd: () => FilePath(process.cwd()), + homedir: () => FilePath(os.homedir()) }; const userConfigName = ".observablehq"; @@ -38,7 +38,7 @@ export interface DeployConfig { } export type ApiKey = - | {source: "file"; filePath: string; key: string} + | {source: "file"; filePath: FilePath; key: string} | {source: "env"; envVar: string; key: string} | {source: "test"; key: string} | {source: "login"; key: string}; @@ -66,10 +66,10 @@ export async function setObservableApiKey(info: null | {id: string; key: string} } export async function getDeployConfig( - sourceRoot: string, + sourceRoot: FilePath, effects: ConfigEffects = defaultEffects ): Promise { - const deployConfigPath = path.join(effects.cwd(), sourceRoot, ".observablehq", "deploy.json"); + const deployConfigPath = fileJoin(effects.cwd(), sourceRoot, ".observablehq", "deploy.json"); let config: object | null = null; try { const content = await effects.readFile(deployConfigPath, "utf8"); @@ -89,12 +89,12 @@ export async function getDeployConfig( } export async function setDeployConfig( - sourceRoot: string, + sourceRoot: FilePath, newConfig: DeployConfig, effects: ConfigEffects = defaultEffects ): Promise { - const dir = path.join(effects.cwd(), sourceRoot, ".observablehq"); - const deployConfigPath = path.join(dir, "deploy.json"); + const dir = fileJoin(effects.cwd(), sourceRoot, ".observablehq"); + const deployConfigPath = fileJoin(dir, "deploy.json"); const oldConfig = (await getDeployConfig(sourceRoot)) || {}; const merged = {...oldConfig, ...newConfig}; await effects.mkdir(dir, {recursive: true}); @@ -103,14 +103,14 @@ export async function setDeployConfig( export async function loadUserConfig( effects: ConfigEffects = defaultEffects -): Promise<{configPath: string; config: UserConfig}> { - const homeConfigPath = path.join(effects.homedir(), userConfigName); +): Promise<{configPath: FilePath; config: UserConfig}> { + const homeConfigPath = fileJoin(effects.homedir(), userConfigName); - function* pathsToTry(): Generator { - let cursor = path.resolve(effects.cwd()); + function* pathsToTry(): Generator { + let cursor = fileResolve(effects.cwd()); while (true) { - yield path.join(cursor, userConfigName); - const nextCursor = path.dirname(cursor); + yield fileJoin(cursor, userConfigName); + const nextCursor = fileDirname(cursor); if (nextCursor === cursor) break; cursor = nextCursor; } @@ -138,7 +138,7 @@ export async function loadUserConfig( } async function writeUserConfig( - {configPath, config}: {configPath: string; config: UserConfig}, + {configPath, config}: {configPath: FilePath; config: UserConfig}, effects: ConfigEffects = defaultEffects ): Promise { await effects.writeFile(configPath, JSON.stringify(config, null, 2)); diff --git a/src/pager.ts b/src/pager.ts index 6a82c5860..3b9dc780b 100644 --- a/src/pager.ts +++ b/src/pager.ts @@ -1,3 +1,4 @@ +import {UrlPath} from "./brandedPath.js"; import type {Config, Page} from "./config.js"; export type PageLink = @@ -6,17 +7,20 @@ export type PageLink = | {prev: Page; next: undefined}; // last page // Pager links in the footer are computed once for a given navigation. -const linkCache = new WeakMap>(); +const linkCache = new WeakMap>(); -export function normalizePath(path: string): string { +/** + * Strip URL query string and hash fragment. + */ +export function normalizePath(path: UrlPath): UrlPath { return path.replace(/[?#].*$/, ""); } -export function findLink(path: string, options: Pick = {pages: []}): PageLink | undefined { +export function findLink(path: UrlPath, options: Pick = {pages: []}): PageLink | undefined { const {pages, title} = options; let links = linkCache.get(pages); if (!links) { - links = new Map(); + links = new Map(); let prev: Page | undefined; for (const page of walk(pages, title)) { const path = normalizePath(page.path); @@ -40,8 +44,8 @@ export function findLink(path: string, options: Pick // Walks the unique pages in the site so as to avoid creating cycles. Implicitly // adds a link at the beginning to the home page (/index). -function* walk(pages: Config["pages"], title = "Home", visited = new Set()): Generator { - if (!visited.has("/index")) yield (visited.add("/index"), {name: title, path: "/index"}); +function* walk(pages: Config["pages"], title = "Home", visited = new Set()): Generator { + if (!visited.has(UrlPath("/index"))) yield (visited.add(UrlPath("/index")), {name: title, path: UrlPath("/index")}); for (const page of pages) { if ("pages" in page) yield* walk(page.pages, title, visited); else if (!visited.has(page.path)) yield (visited.add(page.path), page); diff --git a/src/preview.ts b/src/preview.ts index 93185dedf..7e615f5d3 100644 --- a/src/preview.ts +++ b/src/preview.ts @@ -1,16 +1,31 @@ import {createHash} from "node:crypto"; -import {watch} from "node:fs"; import type {FSWatcher, WatchEventType} from "node:fs"; -import {access, constants, readFile, stat} from "node:fs/promises"; import {createServer} from "node:http"; import type {IncomingMessage, RequestListener, Server, ServerResponse} from "node:http"; -import {basename, dirname, extname, join, normalize} from "node:path"; import {fileURLToPath} from "node:url"; import {difference} from "d3-array"; import openBrowser from "open"; import send from "send"; import {type WebSocket, WebSocketServer} from "ws"; import {version} from "../package.json"; +import {access, constants, readFile, stat, watch} from "./brandedFs.js"; +import { + FilePath, + UrlPath, + fileBasename, + fileDirname, + fileExtname, + fileJoin, + fileNormalize, + filePathToUrlPath, + unFilePath, + unUrlPath, + urlBasename, + urlDirname, + urlJoin, + urlNormalize, + urlPathToFilePath +} from "./brandedPath.js"; import type {Config} from "./config.js"; import {mergeStyle} from "./config.js"; import {Loader} from "./dataloader.js"; @@ -28,7 +43,7 @@ import {Telemetry} from "./telemetry.js"; import {bold, faint, green, link, red} from "./tty.js"; import {relativeUrl} from "./url.js"; -const publicRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "public"); +const publicRoot = fileJoin(fileDirname(FilePath(fileURLToPath(import.meta.url))), "..", "public"); export interface PreviewOptions { config: Config; @@ -91,15 +106,15 @@ export class PreviewServer { if (this._verbose) console.log(faint(req.method!), req.url); try { const url = new URL(req.url!, "http://localhost"); - let pathname = decodeURIComponent(url.pathname); + let pathname = UrlPath(decodeURIComponent(url.pathname)); let match: RegExpExecArray | null; - if (pathname === "/_observablehq/runtime.js") { - const root = join(fileURLToPath(import.meta.resolve("@observablehq/runtime")), "../../"); - send(req, "/dist/runtime.js", {root}).pipe(res); + if (pathname === UrlPath("/_observablehq/runtime.js")) { + const root = fileJoin(fileURLToPath(import.meta.resolve("@observablehq/runtime")), "../../"); + send(req, "/dist/runtime.js", {root: unFilePath(root)}).pipe(res); } else if (pathname.startsWith("/_observablehq/stdlib.js")) { - end(req, res, await rollupClient(getClientPath("./src/client/stdlib.js")), "text/javascript"); + end(req, res, await rollupClient(getClientPath(FilePath("./src/client/stdlib.js"))), "text/javascript"); } else if (pathname.startsWith("/_observablehq/stdlib/")) { - const path = getClientPath("./src/client/" + pathname.slice("/_observablehq/".length)); + const path = getClientPath(FilePath("./src/client/" + pathname.slice("/_observablehq/".length))); if (pathname.endsWith(".js")) { end(req, res, await rollupClient(path), "text/javascript"); } else if (pathname.endsWith(".css")) { @@ -107,19 +122,18 @@ export class PreviewServer { } else { throw new HttpError(`Not found: ${pathname}`, 404); } - } else if (pathname === "/_observablehq/client.js") { - end(req, res, await rollupClient(getClientPath("./src/client/preview.js")), "text/javascript"); - } else if (pathname === "/_observablehq/search.js") { - end(req, res, await rollupClient(getClientPath("./src/client/search.js")), "text/javascript"); - } else if (pathname === "/_observablehq/minisearch.json") { + } else if (pathname === UrlPath("/_observablehq/client.js")) { + end(req, res, await rollupClient(getClientPath(FilePath("./src/client/preview.js"))), "text/javascript"); + } else if (pathname === UrlPath("/_observablehq/search.js")) { + end(req, res, await rollupClient(getClientPath(FilePath("./src/client/search.js"))), "text/javascript"); + } else if (pathname === UrlPath("/_observablehq/minisearch.json")) { end(req, res, await searchIndex(config), "application/json"); - } else if ((match = /^\/_observablehq\/theme-(?[\w-]+(,[\w-]+)*)?\.css$/.exec(pathname))) { + } else if ((match = /^\/_observablehq\/theme-(?[\w-]+(,[\w-]+)*)?\.css$/.exec(unUrlPath(pathname)))) { end(req, res, await bundleStyles({theme: match.groups!.theme?.split(",") ?? []}), "text/css"); } else if (pathname.startsWith("/_observablehq/")) { - send(req, pathname.slice("/_observablehq".length), {root: publicRoot}).pipe(res); + send(req, unUrlPath(pathname).slice("/_observablehq".length), {root: unFilePath(publicRoot)}).pipe(res); } else if (pathname.startsWith("/_import/")) { - const path = pathname.slice("/_import".length); - const filepath = join(root, path); + const filepath = fileJoin(root, urlPathToFilePath(pathname)); try { if (pathname.endsWith(".css")) { await access(filepath, constants.R_OK); @@ -127,7 +141,7 @@ export class PreviewServer { return; } else if (pathname.endsWith(".js")) { const input = await readFile(filepath, "utf-8"); - const output = await rewriteModule(input, path, createImportResolver(root)); + const output = await rewriteModule(input, filepath, createImportResolver(root)); end(req, res, output, "text/javascript"); return; } @@ -136,11 +150,11 @@ export class PreviewServer { } throw new HttpError(`Not found: ${pathname}`, 404); } else if (pathname.startsWith("/_file/")) { - const path = pathname.slice("/_file".length); - const filepath = join(root, path); + const path = urlPathToFilePath(pathname.slice("/_file".length)); + const filepath = fileJoin(root, path); try { await access(filepath, constants.R_OK); - send(req, pathname.slice("/_file".length), {root}).pipe(res); + send(req, unUrlPath(pathname.slice("/_file".length)), {root: unFilePath(root)}).pipe(res); return; } catch (error) { if (!isEnoent(error)) throw error; @@ -150,7 +164,7 @@ export class PreviewServer { const loader = Loader.find(root, path); if (loader) { try { - send(req, await loader.load(), {root}).pipe(res); + send(req, unFilePath(await loader.load()), {root: unFilePath(root)}).pipe(res); return; } catch (error) { if (!isEnoent(error)) throw error; @@ -158,22 +172,22 @@ export class PreviewServer { } throw new HttpError(`Not found: ${pathname}`, 404); } else { - if ((pathname = normalize(pathname)).startsWith("..")) throw new Error("Invalid path: " + pathname); - let path = join(root, pathname); + if ((pathname = urlNormalize(pathname)).startsWith("..")) throw new Error("Invalid path: " + pathname); + let path = fileJoin(root, urlPathToFilePath(pathname)); // If this path is for /index, redirect to the parent directory for a // tidy path. (This must be done before implicitly adding /index below!) // Respect precedence of dir/index.md over dir.md in choosing between // dir/ and dir! - if (basename(path, ".html") === "index") { + if (fileBasename(path, ".html") === "index") { try { - await stat(join(dirname(path), "index.md")); - res.writeHead(302, {Location: join(dirname(pathname), "/") + url.search}); + await stat(fileJoin(fileDirname(path), "index.md")); + res.writeHead(302, {Location: urlJoin(urlDirname(pathname), "/") + url.search}); res.end(); return; } catch (error) { if (!isEnoent(error)) throw error; - res.writeHead(302, {Location: dirname(pathname) + url.search}); + res.writeHead(302, {Location: urlDirname(pathname) + url.search}); res.end(); return; } @@ -182,14 +196,14 @@ export class PreviewServer { // If this path resolves to a directory, then add an implicit /index to // the end of the path, assuming that the corresponding index.md exists. try { - if ((await stat(path)).isDirectory() && (await stat(join(path, "index.md"))).isFile()) { + if ((await stat(path)).isDirectory() && (await stat(fileJoin(path, "index.md"))).isFile()) { if (!pathname.endsWith("/")) { res.writeHead(302, {Location: pathname + "/" + url.search}); res.end(); return; } - pathname = join(pathname, "index"); - path = join(path, "index"); + pathname = urlJoin(pathname, "index"); + path = fileJoin(path, "index"); } } catch (error) { if (!isEnoent(error)) throw error; // internal error @@ -197,8 +211,8 @@ export class PreviewServer { // If this path ends with .html, then redirect to drop the .html. TODO: // Check for the existence of the .md file first. - if (extname(path) === ".html") { - res.writeHead(302, {Location: join(dirname(pathname), basename(pathname, ".html")) + url.search}); + if (fileExtname(path) === ".html") { + res.writeHead(302, {Location: urlJoin(urlDirname(pathname), urlBasename(pathname, ".html")) + url.search}); res.end(); return; } @@ -206,7 +220,7 @@ export class PreviewServer { // Otherwise, serve the corresponding Markdown file, if it exists. // Anything else should 404; static files should be matched above. try { - const {html} = await renderPreview(path + ".md", {path: pathname, ...config}); + const {html} = await renderPreview(FilePath(path + ".md"), {path: pathname, ...config}); end(req, res, html, "text/html"); } catch (error) { if (!isEnoent(error)) throw error; // internal error @@ -222,7 +236,7 @@ export class PreviewServer { } if (req.method === "GET" && res.statusCode === 404) { try { - const {html} = await renderPreview(join(root, "404.md"), {path: "/404", ...config}); + const {html} = await renderPreview(fileJoin(root, "404.md"), {path: UrlPath("/404"), ...config}); end(req, res, html, "text/html"); return; } catch { @@ -265,53 +279,56 @@ function end(req: IncomingMessage, res: ServerResponse, content: string, type: s } } -function getWatchPaths(parseResult: ParseResult): string[] { - const paths: string[] = []; +function getWatchPaths(parseResult: ParseResult): FilePath[] { + const paths: FilePath[] = []; const {files, imports} = parseResult; - for (const f of files) paths.push(f.name); - for (const i of imports) paths.push(i.name); + for (const f of files) paths.push(urlPathToFilePath(f.name)); + for (const i of imports) paths.push(urlPathToFilePath(i.name)); return paths; } -export function getPreviewStylesheet(path: string, data: ParseResult["data"], style: Config["style"]): string | null { +export function getPreviewStylesheet(path: UrlPath, data: ParseResult["data"], style: Config["style"]): UrlPath | null { + const filePath = urlPathToFilePath(path); try { - style = mergeStyle(path, data?.style, data?.theme, style); + style = mergeStyle(filePath, data?.style, data?.theme, style); } catch (error) { console.error(red(String(error))); - return relativeUrl(path, "/_observablehq/theme-.css"); + return relativeUrl(path, UrlPath("/_observablehq/theme-.css")); } return !style ? null : "path" in style - ? relativeUrl(path, `/_import/${style.path}`) - : relativeUrl(path, `/_observablehq/theme-${style.theme.join(",")}.css`); + ? relativeUrl(path, UrlPath(`/_import/${style.path}`)) + : relativeUrl(path, UrlPath(`/_observablehq/theme-${style.theme.join(",")}.css`)); } function handleWatch(socket: WebSocket, req: IncomingMessage, {root, style: defaultStyle}: Config) { - let path: string | null = null; + let path: FilePath | null = null; let current: ParseResult | null = null; - let stylesheets: Set | null = null; + let stylesheets: Set | null = null; let markdownWatcher: FSWatcher | null = null; let attachmentWatcher: FileWatchers | null = null; let emptyTimeout: ReturnType | null = null; console.log(faint("socket open"), req.url); - async function getStylesheets({cells, data}: ParseResult): Promise> { + async function getStylesheets({cells, data}: ParseResult): Promise> { const inputs = new Set(); + const urlPath = filePathToUrlPath(path!); for (const cell of cells) cell.inputs?.forEach(inputs.add, inputs); const stylesheets = await getImplicitStylesheets(getImplicitSpecifiers(inputs)); - const style = getPreviewStylesheet(path!, data, defaultStyle); + const style = getPreviewStylesheet(urlPath, data, defaultStyle); if (style) stylesheets.add(style); - return new Set(Array.from(stylesheets, (href) => resolveStylesheet(path!, href))); + return new Set(Array.from(stylesheets, (href) => resolveStylesheet(urlPath, href))); } - function refreshAttachment(name: string) { + function refreshAttachment(name: FilePath) { + const urlName = filePathToUrlPath(name); const {cells} = current!; - if (cells.some((cell) => cell.imports?.some((i) => i.name === name))) { + if (cells.some((cell) => cell.imports?.some((i) => i.name === urlName))) { watcher("change"); // trigger re-compilation of JavaScript to get new import hashes } else { - const affectedCells = cells.filter((cell) => cell.files?.some((f) => f.name === name)); + const affectedCells = cells.filter((cell) => cell.files?.some((f) => f.name === urlName)); if (affectedCells.length > 0) { send({type: "refresh", cellIds: affectedCells.map((cell) => cell.id)}); } @@ -324,7 +341,7 @@ function handleWatch(socket: WebSocket, req: IncomingMessage, {root, style: defa case "rename": { markdownWatcher?.close(); try { - markdownWatcher = watch(join(root, path), (event) => watcher(event)); + markdownWatcher = watch(fileJoin(root, path), (event) => watcher(event)); } catch (error) { if (!isEnoent(error)) throw error; console.error(`file no longer exists: ${path}`); @@ -335,7 +352,7 @@ function handleWatch(socket: WebSocket, req: IncomingMessage, {root, style: defa break; } case "change": { - const updated = await parseMarkdown(join(root, path), {root, path}); + const updated = await parseMarkdown(fileJoin(root, path), {root, path: filePathToUrlPath(path)}); // delay to avoid a possibly-empty file if (!force && updated.html === "") { if (!emptyTimeout) { @@ -364,17 +381,17 @@ function handleWatch(socket: WebSocket, req: IncomingMessage, {root, style: defa } } - async function hello({path: initialPath, hash: initialHash}: {path: string; hash: string}): Promise { + async function hello({path: initialPath, hash: initialHash}: {path: UrlPath; hash: string}): Promise { if (markdownWatcher || attachmentWatcher) throw new Error("already watching"); - path = initialPath; - if (!(path = normalize(path)).startsWith("/")) throw new Error("Invalid path: " + initialPath); - if (path.endsWith("/")) path += "index"; - path += ".md"; - current = await parseMarkdown(join(root, path), {root, path}); + path = urlPathToFilePath(initialPath); + if (!(path = fileNormalize(path)).startsWith("/")) throw new Error("Invalid path: " + initialPath); + if (path.endsWith("/")) path = FilePath(path + "index"); + path = FilePath(path + ".md"); + current = await parseMarkdown(fileJoin(root, path), {root, path: filePathToUrlPath(path)}); if (current.hash !== initialHash) return void send({type: "reload"}); stylesheets = await getStylesheets(current); attachmentWatcher = await FileWatchers.of(root, path, getWatchPaths(current), refreshAttachment); - markdownWatcher = watch(join(root, path), (event) => watcher(event)); + markdownWatcher = watch(fileJoin(root, path), (event) => watcher(event)); } socket.on("message", async (data) => { diff --git a/src/render.ts b/src/render.ts index 906de4a5f..fea550941 100644 --- a/src/render.ts +++ b/src/render.ts @@ -1,4 +1,5 @@ import {parseHTML} from "linkedom"; +import {FilePath, UrlPath, unUrlPath, urlPathToFilePath} from "./brandedPath.js"; import type {Config, Page, Script, Section} from "./config.js"; import {mergeToc} from "./config.js"; import {getClientPath} from "./files.js"; @@ -21,11 +22,11 @@ export interface Render { } export interface RenderOptions extends Config { - root: string; - path: string; + root: FilePath; + path: UrlPath; } -export async function renderPreview(sourcePath: string, options: RenderOptions): Promise { +export async function renderPreview(sourcePath: FilePath, options: RenderOptions): Promise { const parseResult = await parseMarkdown(sourcePath, options); return { html: await render(parseResult, {...options, preview: true}), @@ -35,7 +36,7 @@ export async function renderPreview(sourcePath: string, options: RenderOptions): }; } -export async function renderServerless(sourcePath: string, options: RenderOptions): Promise { +export async function renderServerless(sourcePath: FilePath, options: RenderOptions): Promise { const parseResult = await parseMarkdown(sourcePath, options); return { html: await render(parseResult, options), @@ -62,7 +63,7 @@ async function render(parseResult: ParseResult, options: RenderOptions & RenderI const sidebar = parseResult.data?.sidebar !== undefined ? Boolean(parseResult.data.sidebar) : options.sidebar; const toc = mergeToc(parseResult.data?.toc, options.toc); return String(html` -${path === "/404" ? html`\n` : ""} +${path === UrlPath("/404") ? html`\n` : ""} ${ parseResult.title || title @@ -71,7 +72,7 @@ ${ .join(" | ")}\n` : "" }${await renderHead(parseResult, options, path, createImportResolver(root, "_import"))}${ - path === "/404" + path === UrlPath("/404") ? html.unsafe(`\n` : "" } @@ -142,7 +143,7 @@ async function renderSidebar(title = "Home", pages: (Page | Section)[], path: st `; } @@ -177,34 +178,34 @@ function renderToc(headers: Header[], label: string): Html { `; } -function renderListItem(p: Page, path: string): Html { +function renderListItem(p: Page, path: UrlPath): Html { return html`\n
  • ${p.name}
    • `; } -function prettyPath(path: string): string { +function prettyPath(path: UrlPath): UrlPath { return path.replace(/\/index$/, "/") || "/"; } async function renderHead( parseResult: ParseResult, options: Pick, - path: string, + path: UrlPath, resolver: ImportResolver ): Promise { const scripts = options.scripts; const head = parseResult.data?.head !== undefined ? parseResult.data.head : options.head; - const stylesheets = new Set(["https://fonts.googleapis.com/css2?family=Source+Serif+Pro:ital,wght@0,400;0,600;0,700;1,400;1,600;1,700&display=swap"]); // prettier-ignore + const stylesheets = new Set([UrlPath("https://fonts.googleapis.com/css2?family=Source+Serif+Pro:ital,wght@0,400;0,600;0,700;1,400;1,600;1,700&display=swap")]); // prettier-ignore const style = getPreviewStylesheet(path, parseResult.data, options.style); if (style) stylesheets.add(style); const specifiers = new Set(["npm:@observablehq/runtime", "npm:@observablehq/stdlib"]); - for (const {name} of parseResult.imports) specifiers.add(name); + for (const {name} of parseResult.imports) specifiers.add(unUrlPath(name)); const inputs = new Set(parseResult.cells.flatMap((cell) => cell.inputs ?? [])); addImplicitSpecifiers(specifiers, inputs); await addImplicitStylesheets(stylesheets, specifiers); - const preloads = new Set([relativeUrl(path, "/_observablehq/client.js")]); - for (const specifier of specifiers) preloads.add(await resolver(path, specifier)); + const preloads = new Set([relativeUrl(path, UrlPath("/_observablehq/client.js"))]); + for (const specifier of specifiers) preloads.add(await resolver(urlPathToFilePath(path), specifier)); await resolveModulePreloads(preloads); return html`${ Array.from(stylesheets) @@ -221,28 +222,30 @@ async function renderHead( }${head ? html`\n${html.unsafe(head)}` : null}${html.unsafe(scripts.map((s) => renderScript(s, path)).join(""))}`; } -export function resolveStylesheet(path: string, href: string): string { +export function resolveStylesheet(path: UrlPath, href: UrlPath): UrlPath { return href.startsWith("observablehq:") - ? relativeUrl(path, `/_observablehq/${href.slice("observablehq:".length)}`) + ? relativeUrl(path, UrlPath(`/_observablehq/${href.slice("observablehq:".length)}`)) : href; } -function renderScript(script: Script, path: string): Html { +function renderScript(script: Script, path: UrlPath): Html { return html`\n`; } -function renderStylesheet(href: string): Html { - return html`\n`; +function renderStylesheet(href: UrlPath): Html { + return html`\n`; } -function renderStylesheetPreload(href: string): Html { - return html`\n`; +function renderStylesheetPreload(href: UrlPath): Html { + return html`\n`; } -function renderModulePreload(href: string): Html { - const integrity: string | undefined = resolveModuleIntegrity(href); +function renderModulePreload(href: UrlPath): Html { + const integrity: string | undefined = resolveModuleIntegrity(unUrlPath(href)); return html`\n`; } @@ -252,7 +255,7 @@ function renderHeader({header}: Pick, data: ParseResult["data" } function renderFooter( - path: string, + path: UrlPath, options: Pick, data: ParseResult["data"] ): Html | null { @@ -267,10 +270,10 @@ function renderFooter( : null; } -function renderPager(path: string, {prev, next}: PageLink): Html { +function renderPager(path: UrlPath, {prev, next}: PageLink): Html { return html`\n`; } -function renderRel(path: string, page: Page, rel: "prev" | "next"): Html { +function renderRel(path: UrlPath, page: Page, rel: "prev" | "next"): Html { return html`${page.name}`; } diff --git a/src/rollup.ts b/src/rollup.ts index 547a1126f..dd8b5060b 100644 --- a/src/rollup.ts +++ b/src/rollup.ts @@ -5,6 +5,7 @@ import {build} from "esbuild"; import type {AstNode, OutputChunk, Plugin, ResolveIdResult} from "rollup"; import {rollup} from "rollup"; import esbuild from "rollup-plugin-esbuild"; +import {FilePath, UrlPath, filePathToUrlPath, unFilePath, unUrlPath} from "./brandedPath.js"; import {getClientPath} from "./files.js"; import {getStringLiteralValue, isStringLiteral} from "./javascript/features.js"; import {isPathImport, resolveNpmImport} from "./javascript/imports.js"; @@ -13,8 +14,8 @@ import {Sourcemap} from "./sourcemap.js"; import {THEMES, renderTheme} from "./theme.js"; import {relativeUrl} from "./url.js"; -const STYLE_MODULES = { - "observablehq:default.css": getClientPath("./src/style/default.css"), +const STYLE_MODULES: Record = { + "observablehq:default.css": getClientPath(FilePath("./src/style/default.css")), ...Object.fromEntries(THEMES.map(({name, path}) => [`observablehq:theme-${name}.css`, path])) }; @@ -25,20 +26,21 @@ function rewriteInputsNamespace(code: string) { return code.replace(/\b__ns__\b/g, "inputs-3a86ea"); } -export async function bundleStyles({path, theme}: {path?: string; theme?: string[]}): Promise { +export async function bundleStyles({path, theme}: {path?: FilePath; theme?: string[]}): Promise { + const compatiblePath: string | undefined = path ? unFilePath(path) : undefined; const result = await build({ bundle: true, - ...(path ? {entryPoints: [path]} : {stdin: {contents: renderTheme(theme!), loader: "css"}}), + ...(compatiblePath ? {entryPoints: [compatiblePath]} : {stdin: {contents: renderTheme(theme!), loader: "css"}}), write: false, - alias: STYLE_MODULES + alias: STYLE_MODULES as unknown as Record }); const text = result.outputFiles[0].text; return rewriteInputsNamespace(text); // TODO only for inputs } -export async function rollupClient(clientPath: string, {minify = false} = {}): Promise { +export async function rollupClient(clientPath: FilePath, {minify = false} = {}): Promise { const bundle = await rollup({ - input: clientPath, + input: unFilePath(clientPath), external: [/^https:/], plugins: [ nodeResolve({resolveOnly: BUNDLED_MODULES}), @@ -76,43 +78,68 @@ function rewriteTypeScriptImports(code: string): string { return code.replace(/(?<=\bimport\(([`'"])[\w./]+)\.ts(?=\1\))/g, ".js"); } -function importResolve(clientPath: string): Plugin { +function importResolve(clientPath: FilePath): Plugin { + const urlClientPath = filePathToUrlPath(clientPath); return { name: "resolve-import", - resolveId: (specifier) => resolveImport(clientPath, specifier), - resolveDynamicImport: (specifier) => resolveImport(clientPath, specifier) + resolveId: (specifier) => resolveImport(urlClientPath, specifier), + resolveDynamicImport: (specifier) => resolveImport(urlClientPath, specifier) }; } // TODO Consolidate with createImportResolver. -async function resolveImport(source: string, specifier: string | AstNode): Promise { +async function resolveImport(source: UrlPath, specifier: string | AstNode): Promise { return typeof specifier !== "string" ? null : specifier.startsWith("observablehq:") - ? {id: relativeUrl(source, getClientPath(`./src/client/${specifier.slice("observablehq:".length)}.js`)), external: true} // prettier-ignore + ? {id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath(`./src/client/${specifier.slice("observablehq:".length)}.js`)))), external: true} // prettier-ignore : specifier === "npm:@observablehq/runtime" - ? {id: relativeUrl(source, getClientPath("./src/client/runtime.js")), external: true} + ? {id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath("./src/client/runtime.js")))), external: true} : specifier === "npm:@observablehq/stdlib" - ? {id: relativeUrl(source, getClientPath("./src/client/stdlib.js")), external: true} + ? {id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath("./src/client/stdlib.js")))), external: true} : specifier === "npm:@observablehq/dot" - ? {id: relativeUrl(source, getClientPath("./src/client/stdlib/dot.js")), external: true} // TODO publish to npm + ? { + id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath("./src/client/stdlib/dot.js")))), + external: true + } // TODO publish to npm : specifier === "npm:@observablehq/duckdb" - ? {id: relativeUrl(source, getClientPath("./src/client/stdlib/duckdb.js")), external: true} // TODO publish to npm + ? { + id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath("./src/client/stdlib/duckdb.js")))), + external: true + } // TODO publish to npm : specifier === "npm:@observablehq/inputs" - ? {id: relativeUrl(source, getClientPath("./src/client/stdlib/inputs.js")), external: true} // TODO publish to npm + ? { + id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath("./src/client/stdlib/inputs.js")))), + external: true + } // TODO publish to npm : specifier === "npm:@observablehq/mermaid" - ? {id: relativeUrl(source, getClientPath("./src/client/stdlib/mermaid.js")), external: true} // TODO publish to npm + ? { + id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath("./src/client/stdlib/mermaid.js")))), + external: true + } // TODO publish to npm : specifier === "npm:@observablehq/tex" - ? {id: relativeUrl(source, getClientPath("./src/client/stdlib/tex.js")), external: true} // TODO publish to npm + ? { + id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath("./src/client/stdlib/tex.js")))), + external: true + } // TODO publish to npm : specifier === "npm:@observablehq/sqlite" - ? {id: relativeUrl(source, getClientPath("./src/client/stdlib/sqlite.js")), external: true} // TODO publish to npm + ? { + id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath("./src/client/stdlib/sqlite.js")))), + external: true + } // TODO publish to npm : specifier === "npm:@observablehq/xlsx" - ? {id: relativeUrl(source, getClientPath("./src/client/stdlib/xlsx.js")), external: true} // TODO publish to npm + ? { + id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath("./src/client/stdlib/xlsx.js")))), + external: true + } // TODO publish to npm : specifier === "npm:@observablehq/zip" - ? {id: relativeUrl(source, getClientPath("./src/client/stdlib/zip.js")), external: true} // TODO publish to npm + ? { + id: relativeUrl(source, filePathToUrlPath(getClientPath(FilePath("./src/client/stdlib/zip.js")))), + external: true + } // TODO publish to npm : specifier.startsWith("npm:") ? {id: await resolveNpmImport(specifier.slice("npm:".length))} - : source !== specifier && !isPathImport(specifier) && !BUNDLED_MODULES.includes(specifier) + : source !== UrlPath(specifier) && !isPathImport(specifier) && !BUNDLED_MODULES.includes(specifier) ? {id: await resolveNpmImport(specifier), external: true} : null; } diff --git a/src/search.ts b/src/search.ts index cced344bf..dd2d1983e 100644 --- a/src/search.ts +++ b/src/search.ts @@ -1,6 +1,6 @@ -import {basename, join} from "node:path"; import he from "he"; import MiniSearch from "minisearch"; +import {UrlPath, fileBasename, fileJoin, urlBasename} from "./brandedPath.js"; import type {Config} from "./config.js"; import {visitMarkdownFiles} from "./files.js"; import type {Logger} from "./logger.js"; @@ -31,7 +31,7 @@ export async function searchIndex(config: Config, effects = defaultEffects): Pro if (indexCache.has(config) && indexCache.get(config).freshUntil > +new Date()) return indexCache.get(config).json; // Get all the listed pages (which are indexed by default) - const pagePaths = new Set(["/index"]); + const pagePaths = new Set([UrlPath("/index")]); for (const p of pages) { if ("path" in p) pagePaths.add(p.path); else for (const {path} of p.pages) pagePaths.add(path); @@ -40,12 +40,12 @@ export async function searchIndex(config: Config, effects = defaultEffects): Pro // Index the pages const index = new MiniSearch(indexOptions); for await (const file of visitMarkdownFiles(root)) { - const path = join(root, file); - const {html, title, data} = await parseMarkdown(path, {root, path: "/" + file.slice(0, -3)}); + const path = fileJoin(root, file); + const {html, title, data} = await parseMarkdown(path, {root, path: UrlPath("/" + file.slice(0, -3))}); // Skip pages that opt-out of indexing, and skip unlisted pages unless // opted-in. We only log the first case. - const listed = pagePaths.has(`/${file.slice(0, -3)}`); + const listed = pagePaths.has(UrlPath(`/${file.slice(0, -3)}`)); const indexed = data?.index === undefined ? listed : Boolean(data.index); if (!indexed) { if (listed) effects.logger.log(`${faint("skip")} ${file}`); @@ -54,7 +54,7 @@ export async function searchIndex(config: Config, effects = defaultEffects): Pro // This is the (top-level) serving path to the indexed page. There’s // implicitly a leading slash here. - const id = file.slice(0, basename(file) === "index.md" ? -"index.md".length : -3); + const id = file.slice(0, fileBasename(file) === "index.md" ? -"index.md".length : -3); // eslint-disable-next-line import/no-named-as-default-member const text = he diff --git a/src/telemetry.ts b/src/telemetry.ts index 588cd5cc2..321e1d730 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -1,8 +1,8 @@ import {exec} from "node:child_process"; import {createHash, randomUUID} from "node:crypto"; -import {readFile, writeFile} from "node:fs/promises"; -import {join} from "node:path"; import os from "os"; +import {readFile, writeFile} from "./brandedFs.js"; +import {fileJoin} from "./brandedPath.js"; import {CliError} from "./error.js"; import type {Logger} from "./logger.js"; import {getObservableUiOrigin} from "./observableApiClient.js"; @@ -137,7 +137,7 @@ export class Telemetry { private async getPersistentId(name: string, generator = randomUUID) { const {readFile, writeFile} = this.effects; - const file = join(os.homedir(), ".observablehq"); + const file = fileJoin(os.homedir(), ".observablehq"); if (!this._config) { this._config = readFile(file, "utf8") .then(JSON.parse) diff --git a/src/theme.ts b/src/theme.ts index 4ecb0f68e..56fd69c67 100644 --- a/src/theme.ts +++ b/src/theme.ts @@ -1,37 +1,38 @@ +import {FilePath} from "./brandedPath.js"; import {getClientPath} from "./files.js"; export interface Theme { name: string; - path: string; + path: FilePath; light?: boolean; dark?: boolean; index: number; } const LIGHT_THEMES: Omit[] = [ - {name: "air", path: getClientPath("./src/style/theme-air.css"), light: true}, - {name: "cotton", path: getClientPath("./src/style/theme-cotton.css"), light: true}, - {name: "glacier", path: getClientPath("./src/style/theme-glacier.css"), light: true}, - {name: "parchment", path: getClientPath("./src/style/theme-parchment.css"), light: true} + {name: "air", path: getClientPath(FilePath("./src/style/theme-air.css")), light: true}, + {name: "cotton", path: getClientPath(FilePath("./src/style/theme-cotton.css")), light: true}, + {name: "glacier", path: getClientPath(FilePath("./src/style/theme-glacier.css")), light: true}, + {name: "parchment", path: getClientPath(FilePath("./src/style/theme-parchment.css")), light: true} ]; const DARK_THEMES: Omit[] = [ - {name: "coffee", path: getClientPath("./src/style/theme-coffee.css"), dark: true}, - {name: "deep-space", path: getClientPath("./src/style/theme-deep-space.css"), dark: true}, - {name: "ink", path: getClientPath("./src/style/theme-ink.css"), dark: true}, - {name: "midnight", path: getClientPath("./src/style/theme-midnight.css"), dark: true}, - {name: "near-midnight", path: getClientPath("./src/style/theme-near-midnight.css"), dark: true}, - {name: "ocean-floor", path: getClientPath("./src/style/theme-ocean-floor.css"), dark: true}, - {name: "slate", path: getClientPath("./src/style/theme-slate.css"), dark: true}, - {name: "stark", path: getClientPath("./src/style/theme-stark.css"), dark: true}, - {name: "sun-faded", path: getClientPath("./src/style/theme-sun-faded.css"), dark: true} + {name: "coffee", path: getClientPath(FilePath("./src/style/theme-coffee.css")), dark: true}, + {name: "deep-space", path: getClientPath(FilePath("./src/style/theme-deep-space.css")), dark: true}, + {name: "ink", path: getClientPath(FilePath("./src/style/theme-ink.css")), dark: true}, + {name: "midnight", path: getClientPath(FilePath("./src/style/theme-midnight.css")), dark: true}, + {name: "near-midnight", path: getClientPath(FilePath("./src/style/theme-near-midnight.css")), dark: true}, + {name: "ocean-floor", path: getClientPath(FilePath("./src/style/theme-ocean-floor.css")), dark: true}, + {name: "slate", path: getClientPath(FilePath("./src/style/theme-slate.css")), dark: true}, + {name: "stark", path: getClientPath(FilePath("./src/style/theme-stark.css")), dark: true}, + {name: "sun-faded", path: getClientPath(FilePath("./src/style/theme-sun-faded.css")), dark: true} ]; export const THEMES: Theme[] = [ ...LIGHT_THEMES, ...DARK_THEMES, - {name: "alt", path: getClientPath("./src/style/theme-alt.css")}, - {name: "wide", path: getClientPath("./src/style/theme-wide.css")} + {name: "alt", path: getClientPath(FilePath("./src/style/theme-alt.css"))}, + {name: "wide", path: getClientPath(FilePath("./src/style/theme-wide.css"))} ].map((theme, i) => ({ ...theme, index: i diff --git a/src/tty.ts b/src/tty.ts index 1b09aba11..d5e56ef84 100644 --- a/src/tty.ts +++ b/src/tty.ts @@ -14,10 +14,10 @@ export const blue = color(34, 39); export const magenta = color(35, 39); export const cyan = color(36, 39); -export type TtyColor = (text: string) => string; +export type TtyColor = (text: string | {toString(): string}) => string; function color(code: number, reset: number): TtyColor { - return process.stdout.isTTY ? (text: string) => `\x1b[${code}m${text}\x1b[${reset}m` : String; + return process.stdout.isTTY ? (text) => `\x1b[${code}m${text}\x1b[${reset}m` : String; } export interface TtyEffects { diff --git a/src/url.ts b/src/url.ts index cb554555a..3b47d4262 100644 --- a/src/url.ts +++ b/src/url.ts @@ -1,4 +1,4 @@ -import {dirname, join} from "node:path"; +import {FilePath, UrlPath, fileDirname, fileJoin, unUrlPath, urlJoin, urlPathToFilePath} from "./brandedPath.js"; /** * Returns the normalized relative path from "/file/path/to/a" to @@ -6,17 +6,17 @@ import {dirname, join} from "node:path"; * are prefixed with "./", and paths that start without a slash are considered * from the root. */ -export function relativeUrl(source: string, target: string): string { - if (/^\w+:/.test(target)) return target; - const from = join("/", source).split(/[/]+/g).slice(0, -1); - const to = join("/", target).split(/[/]+/g); +export function relativeUrl(source: UrlPath, target: UrlPath): UrlPath { + if (/^\w+:/.test(unUrlPath(target))) return target; + const from = urlJoin("/", source).split(/[/]+/g).slice(0, -1); + const to = urlJoin("/", target).split(/[/]+/g); const f = to.pop()!; const m = from.length; const n = Math.min(m, to.length); let i = 0; while (i < n && from[i] === to[i]) ++i; const k = m - i; - return (k ? "../".repeat(k) : "./") + to.slice(i).concat(f).join("/"); + return UrlPath((k ? "../".repeat(k) : "./") + to.slice(i).concat(f).join("/")); } /** @@ -24,9 +24,18 @@ export function relativeUrl(source: string, target: string): string { * defaults to ".", assuming that the target is a relative path such as an * import or fetch from the specified source. */ -export function resolvePath(source: string, target: string): string; -export function resolvePath(root: string, source: string, target: string): string; -export function resolvePath(root: string, source: string, target?: string): string { - if (target === undefined) (target = source), (source = root), (root = "."); - return join(root, target.startsWith("/") ? "." : dirname(source), target); +export function resolvePath(source: FilePath, target: UrlPath): FilePath; +export function resolvePath(root: FilePath, source: FilePath, target: UrlPath): FilePath; +export function resolvePath(arg1: FilePath, arg2: FilePath | UrlPath, arg3?: UrlPath): FilePath { + let root: FilePath, source: FilePath, target: UrlPath; + if (arg3 === undefined) { + root = FilePath("."); + source = arg1; + target = arg2 as UrlPath; + } else { + root = arg1; + source = arg2 as FilePath; + target = arg3; + } + return fileJoin(root, target.startsWith("/") ? "." : fileDirname(source), urlPathToFilePath(target)); } diff --git a/test/build-test.ts b/test/build-test.ts index a9c59508f..bb8e2c486 100644 --- a/test/build-test.ts +++ b/test/build-test.ts @@ -1,8 +1,8 @@ import assert from "node:assert"; -import {existsSync, readdirSync, statSync} from "node:fs"; -import {open, readFile, rm} from "node:fs/promises"; -import {join, normalize, relative} from "node:path"; import {difference} from "d3-array"; +import {existsSync, open, readFile, readdirSync, rm, statSync} from "../src/brandedFs.js"; +import {fileJoin, fileNormalize, fileRelative, unFilePath} from "../src/brandedPath.js"; +import type {FilePath} from "../src/brandedPath.js"; import {FileBuildEffects, build} from "../src/build.js"; import {readConfig, setCurrentDate} from "../src/config.js"; import {mockJsDelivr} from "./mocks/jsdelivr.js"; @@ -12,22 +12,22 @@ const silentEffects = { output: {write() {}} }; -describe("build", async () => { +describe.only("build", async () => { before(() => setCurrentDate(new Date("2024-01-10T16:00:00"))); mockJsDelivr(); // Each sub-directory of test/input/build is a test case. - const inputRoot = "test/input/build"; - const outputRoot = "test/output/build"; + const inputRoot = fileJoin("test", "input", "build"); + const outputRoot = fileJoin("test", "output", "build"); for (const name of readdirSync(inputRoot)) { - const path = join(inputRoot, name); + const path = fileJoin(inputRoot, name); if (!statSync(path).isDirectory()) continue; const only = name.startsWith("only."); const skip = name.startsWith("skip."); const outname = only || skip ? name.slice(5) : name; - (only ? it.only : skip ? it.skip : it)(`${inputRoot}/${name}`, async () => { - const actualDir = join(outputRoot, `${outname}-changed`); - const expectedDir = join(outputRoot, outname); + (only ? it.only : skip ? it.skip : it)(unFilePath(fileJoin(inputRoot, name)), async () => { + const actualDir = fileJoin(outputRoot, `${outname}-changed`); + const expectedDir = fileJoin(outputRoot, outname); const generate = !existsSync(expectedDir) && process.env.CI !== "true"; const outputDir = generate ? expectedDir : actualDir; const addPublic = name.endsWith("-public"); @@ -41,10 +41,10 @@ describe("build", async () => { // files because they change often; replace them with empty files so we // can at least check that the expected files exist. if (addPublic) { - const publicDir = join(outputDir, "_observablehq"); + const publicDir = fileJoin(outputDir, "_observablehq"); for (const file of findFiles(publicDir)) { if (file.endsWith(".json")) continue; // e.g., minisearch.json - await (await open(join(publicDir, file), "w")).close(); + await (await open(fileJoin(publicDir, file), "w")).close(); } } @@ -58,8 +58,8 @@ describe("build", async () => { if (unexpectedFiles.size > 0) assert.fail(`Unexpected output files: ${Array.from(unexpectedFiles).join(", ")}`); for (const path of expectedFiles) { - const actual = await readFile(join(actualDir, path), "utf8"); - const expected = await readFile(join(expectedDir, path), "utf8"); + const actual = await readFile(fileJoin(actualDir, path), "utf8"); + const expected = await readFile(fileJoin(expectedDir, path), "utf8"); assert.ok(actual === expected, `${path} must match snapshot`); } @@ -68,29 +68,29 @@ describe("build", async () => { } }); -function* findFiles(root: string): Iterable { +function* findFiles(root: FilePath): Iterable { const visited = new Set(); - const queue: string[] = [(root = normalize(root))]; + const queue: FilePath[] = [(root = fileNormalize(root))]; for (const path of queue) { const status = statSync(path); if (status.isDirectory()) { if (visited.has(status.ino)) throw new Error(`Circular directory: ${path}`); visited.add(status.ino); for (const entry of readdirSync(path)) { - if (entry === ".DS_Store") continue; // macOS - queue.push(join(path, entry)); + if (unFilePath(entry) === ".DS_Store") continue; // macOS + queue.push(fileJoin(path, entry)); } } else { - yield relative(root, path); + yield fileRelative(root, path); } } } class TestEffects extends FileBuildEffects { - constructor(outputRoot: string) { + constructor(outputRoot: FilePath) { super(outputRoot, silentEffects); } - async writeFile(outputPath: string, contents: string | Buffer): Promise { + async writeFile(outputPath: FilePath, contents: string | Buffer): Promise { if (typeof contents === "string" && outputPath.endsWith(".html")) { contents = contents.replace(/^(\s* + + + + + +
    +
    +

    Tar

    +
    +
    +
    +
    + +
    diff --git a/test/output/build/archives.win32/zip.html b/test/output/build/archives.win32/zip.html new file mode 100644 index 000000000..77339dcb8 --- /dev/null +++ b/test/output/build/archives.win32/zip.html @@ -0,0 +1,57 @@ + + + +Zip + + + + + + + + + + + + + + +
    +
    +

    Zip

    +
    +
    +
    + +
    diff --git a/test/output/imports/illegal-import-changed.js b/test/output/imports/illegal-import-changed.js new file mode 100644 index 000000000..7affb70da --- /dev/null +++ b/test/output/imports/illegal-import-changed.js @@ -0,0 +1,6 @@ +define({id: "0", inputs: ["display"], outputs: ["foo"], body: async (display) => { +const {foo} = await import("./bar.js?sha=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + +display(foo); +return {foo}; +}}); From f4699e5ad371b16bebefa385a4a9b08d50fafc52 Mon Sep 17 00:00:00 2001 From: Michael Cooper Date: Wed, 21 Feb 2024 17:33:20 -0800 Subject: [PATCH 05/30] tests passing locally --- package.json | 1 + src/brandedFs.ts | 4 +- src/brandedPath.ts | 3 + src/build.ts | 3 +- src/dataloader.ts | 2 +- src/fileWatchers.ts | 9 +- src/files.ts | 4 +- src/javascript/features.ts | 17 +-- src/javascript/imports.ts | 18 ++- src/observableApiClient.ts | 2 +- src/preview.ts | 18 +-- src/rollup.ts | 2 +- src/search.ts | 2 +- test/build-test.ts | 7 +- test/config-test.ts | 2 +- test/dataloaders-test.ts | 106 +++++++++--------- test/fileWatchers-test.ts | 32 +++--- test/files-test.ts | 29 +++-- test/isLocalImport-test.ts | 12 +- test/observableApiConfig-test.ts | 34 ++++-- test/output/build/archives.win32/tar.html | 2 +- test/output/build/archives.win32/zip.html | 2 +- test/output/imports/illegal-import-changed.js | 6 - test/preview/preview-test.ts | 3 +- 24 files changed, 173 insertions(+), 147 deletions(-) delete mode 100644 test/output/imports/illegal-import-changed.js diff --git a/package.json b/package.json index 3fba9af61..e46fc7b65 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "acorn": "^8.11.2", "acorn-walk": "^8.3.0", "ci-info": "^4.0.0", + "cross-spawn": "^7.0.3", "esbuild": "^0.19.8", "fast-array-diff": "^1.1.0", "fast-deep-equal": "^3.1.3", diff --git a/src/brandedFs.ts b/src/brandedFs.ts index 9cc901399..ff2d81399 100644 --- a/src/brandedFs.ts +++ b/src/brandedFs.ts @@ -83,11 +83,11 @@ export function watch(path: FilePath, listener?: WatchListener): fs.FSWa return fs.watch(unFilePath(path), listener); } -export function utimes(path: FilePath, atime: number | Date, mtime: number | Date): Promise { +export function utimes(path: FilePath, atime: Date, mtime: Date): Promise { return fsp.utimes(unFilePath(path), atime, mtime); } -export function utimesSync(path: FilePath, atime: number | Date, mtime: number | Date): void { +export function utimesSync(path: FilePath, atime: Date, mtime: Date): void { return fs.utimesSync(unFilePath(path), atime, mtime); } diff --git a/src/brandedPath.ts b/src/brandedPath.ts index 912691dd7..286de86f8 100644 --- a/src/brandedPath.ts +++ b/src/brandedPath.ts @@ -96,3 +96,6 @@ export function fileResolve(...paths: (string | FilePath)[]): FilePath { export function urlResolve(...paths: (string | UrlPath)[]): UrlPath { return UrlPath(osPath.resolve(...(paths as string[]))); } + +export const fileSep = osPath.sep as unknown as FilePath; +export const urlSep = posixPath.sep as unknown as UrlPath; diff --git a/src/build.ts b/src/build.ts index 6225a5c14..1d344374d 100644 --- a/src/build.ts +++ b/src/build.ts @@ -7,7 +7,6 @@ import { fileDirname, fileJoin, filePathToUrlPath, - unFilePath, unUrlPath, urlBasename, urlDirname, @@ -211,7 +210,7 @@ export class FileBuildEffects implements BuildEffects { this.outputRoot = outputRoot; } existsSync(path: FilePath) { - return existsSync(unFilePath(path)); + return existsSync(path); } readFile(path: FilePath, encoding: "utf-8"): Promise { return readFile(path, encoding); diff --git a/src/dataloader.ts b/src/dataloader.ts index 6e12df0c0..6724ddf9e 100644 --- a/src/dataloader.ts +++ b/src/dataloader.ts @@ -1,6 +1,6 @@ -import {spawn} from "node:child_process"; import {type WriteStream} from "node:fs"; import {createGunzip} from "node:zlib"; +import {spawn} from "cross-spawn"; import JSZip from "jszip"; import {extract} from "tar-stream"; import {createReadStream, existsSync, statSync} from "./brandedFs.js"; diff --git a/src/fileWatchers.ts b/src/fileWatchers.ts index d5cec18ac..fb13263fe 100644 --- a/src/fileWatchers.ts +++ b/src/fileWatchers.ts @@ -1,6 +1,6 @@ import {type FSWatcher} from "node:fs"; import {existsSync, watch} from "./brandedFs.js"; -import {type FilePath, filePathToUrlPath} from "./brandedPath.js"; +import type {FilePath, UrlPath} from "./brandedPath.js"; import {Loader} from "./dataloader.js"; import {isEnoent} from "./error.js"; import {maybeStat} from "./files.js"; @@ -9,11 +9,10 @@ import {resolvePath} from "./url.js"; export class FileWatchers { private readonly watchers: FSWatcher[] = []; - static async of(root: FilePath, path: FilePath, names: FilePath[], callback: (name: FilePath) => void) { + static async of(root: FilePath, path: FilePath, names: UrlPath[], callback: (name: UrlPath) => void) { const that = new FileWatchers(); const {watchers} = that; - for (const fileName of new Set(names)) { - const name = filePathToUrlPath(fileName); + for (const name of new Set(names)) { const exactPath = resolvePath(root, path, name); const watchPath = existsSync(exactPath) ? exactPath : Loader.find(root, resolvePath(path, name))?.path; if (!watchPath) continue; @@ -39,7 +38,7 @@ export class FileWatchers { // Ignore if the file was truncated or not modified. if (currentStat?.mtimeMs === newStat?.mtimeMs || newStat?.size === 0) return; currentStat = newStat; - callback(fileName); + callback(name); }); } catch (error) { if (!isEnoent(error)) throw error; diff --git a/src/files.ts b/src/files.ts index 23e872594..00ca3011c 100644 --- a/src/files.ts +++ b/src/files.ts @@ -3,7 +3,7 @@ import {cwd} from "node:process"; import {fileURLToPath} from "node:url"; import mime from "mime"; import {existsSync, mkdir, readdir, stat} from "./brandedFs.js"; -import {UrlPath, urlJoin, urlPathToFilePath} from "./brandedPath.js"; +import {UrlPath, fileSep, urlJoin, urlPathToFilePath} from "./brandedPath.js"; import {FilePath, fileDirname, fileExtname, fileJoin, fileNormalize, fileRelative, unUrlPath} from "./brandedPath.js"; import {isEnoent} from "./error.js"; import type {FileReference} from "./javascript.js"; @@ -14,7 +14,7 @@ export function getLocalPath(sourcePath: UrlPath, name: UrlPath): FilePath | nul if (/^\w+:/.test(unUrlPath(name))) return null; // URL if (name.startsWith("#")) return null; // anchor tag const path = resolvePath(urlPathToFilePath(sourcePath), name); - if (path.startsWith("../")) return null; // goes above root + if (path.startsWith(`..${fileSep}`)) return null; // goes above root return path; } diff --git a/src/javascript/features.ts b/src/javascript/features.ts index 6c9ef50be..dec24de6f 100644 --- a/src/javascript/features.ts +++ b/src/javascript/features.ts @@ -1,11 +1,11 @@ import type {CallExpression, Identifier, Literal, Node, TemplateLiteral} from "acorn"; import {simple} from "acorn-walk"; +import {type FilePath, UrlPath, filePathToUrlPath} from "../brandedPath.js"; import {getLocalPath} from "../files.js"; import type {Feature} from "../javascript.js"; import {defaultGlobals} from "./globals.js"; import {findReferences} from "./references.js"; import {syntaxError} from "./syntaxError.js"; -import {FilePath, UrlPath, filePathToUrlPath, unFilePath} from "../brandedPath.js"; export function findFeatures(node: Node, path: FilePath, references: Identifier[], input: string): Feature[] { const featureMap = getFeatureReferenceMap(node); @@ -26,7 +26,7 @@ export function findFeatures(node: Node, path: FilePath, references: Identifier[ if (name !== "FileAttachment") return; type = name; } - features.push(getFeature(type, node, path, input)); + features.push(getFeature(type, node, filePathToUrlPath(path), input)); } }); @@ -83,7 +83,7 @@ export function getFeatureReferenceMap(node: Node): Map resolveStylesheet(urlPath, href))); } - function refreshAttachment(name: FilePath) { - const urlName = filePathToUrlPath(name); + function refreshAttachment(name: UrlPath) { const {cells} = current!; - if (cells.some((cell) => cell.imports?.some((i) => i.name === urlName))) { + if (cells.some((cell) => cell.imports?.some((i) => i.name === name))) { watcher("change"); // trigger re-compilation of JavaScript to get new import hashes } else { - const affectedCells = cells.filter((cell) => cell.files?.some((f) => f.name === urlName)); + const affectedCells = cells.filter((cell) => cell.files?.some((f) => f.name === name)); if (affectedCells.length > 0) { send({type: "refresh", cellIds: affectedCells.map((cell) => cell.id)}); } diff --git a/src/rollup.ts b/src/rollup.ts index dd8b5060b..b671e5fb0 100644 --- a/src/rollup.ts +++ b/src/rollup.ts @@ -5,7 +5,7 @@ import {build} from "esbuild"; import type {AstNode, OutputChunk, Plugin, ResolveIdResult} from "rollup"; import {rollup} from "rollup"; import esbuild from "rollup-plugin-esbuild"; -import {FilePath, UrlPath, filePathToUrlPath, unFilePath, unUrlPath} from "./brandedPath.js"; +import {FilePath, UrlPath, filePathToUrlPath, unFilePath} from "./brandedPath.js"; import {getClientPath} from "./files.js"; import {getStringLiteralValue, isStringLiteral} from "./javascript/features.js"; import {isPathImport, resolveNpmImport} from "./javascript/imports.js"; diff --git a/src/search.ts b/src/search.ts index 8d3976765..b151dc656 100644 --- a/src/search.ts +++ b/src/search.ts @@ -1,6 +1,6 @@ import he from "he"; import MiniSearch from "minisearch"; -import {UrlPath, fileBasename, fileJoin, urlBasename} from "./brandedPath.js"; +import {UrlPath, fileBasename, fileJoin} from "./brandedPath.js"; import type {Config} from "./config.js"; import {visitMarkdownFiles} from "./files.js"; import type {Logger} from "./logger.js"; diff --git a/test/build-test.ts b/test/build-test.ts index 069ba14a0..bee346850 100644 --- a/test/build-test.ts +++ b/test/build-test.ts @@ -1,4 +1,5 @@ import assert from "node:assert"; +import os from "node:os"; import {difference} from "d3-array"; import {existsSync, open, readFile, readdirSync, rm, statSync} from "../src/brandedFs.js"; import {fileJoin, fileNormalize, fileRelative, unFilePath} from "../src/brandedPath.js"; @@ -12,7 +13,7 @@ const silentEffects = { output: {write() {}} }; -describe("build", async () => { +describe.skip("build", async () => { before(() => setCurrentDate(new Date("2024-01-10T16:00:00"))); mockJsDelivr(); @@ -25,8 +26,8 @@ describe("build", async () => { const only = name.startsWith("only."); const skip = name.startsWith("skip.") || - (name.endsWith(".posix") && process.platform === "win32") || - (name.endsWith(".win32") && process.platform !== "win32"); + (name.endsWith(".posix") && os.platform() === "win32") || + (name.endsWith(".win32") && os.platform() !== "win32"); const outname = only || skip ? name.slice(5) : name; (only ? it.only : skip ? it.skip : it)(unFilePath(fileJoin(inputRoot, name)), async () => { const actualDir = fileJoin(outputRoot, `${outname}-changed`); diff --git a/test/config-test.ts b/test/config-test.ts index 98409e85c..99da06824 100644 --- a/test/config-test.ts +++ b/test/config-test.ts @@ -8,7 +8,7 @@ describe("readConfig(undefined, root)", () => { before(() => setCurrentDate(new Date("2024-01-11T01:02:03"))); it("imports the config file at the specified root", async () => { assert.deepStrictEqual(await readConfig(undefined, FilePath("test/input/build/config")), { - root: "test/input/build/config", + root: FilePath("test/input/build/config"), output: "dist", base: "/", style: {theme: ["air", "near-midnight"]}, diff --git a/test/dataloaders-test.ts b/test/dataloaders-test.ts index dd5ab4062..ea152c655 100644 --- a/test/dataloaders-test.ts +++ b/test/dataloaders-test.ts @@ -1,4 +1,5 @@ import assert from "node:assert"; +import os from "node:os"; import {readFile, stat, unlink, utimes} from "../src/brandedFs.js"; import {FilePath, fileJoin} from "../src/brandedPath.js"; import {type LoadEffects, Loader} from "../src/dataloader.js"; @@ -8,6 +9,8 @@ const noopEffects: LoadEffects = { output: {write() {}} }; +const lineEnding = os.platform() === "win32" ? "\r\n" : "\n"; + describe("data loaders are called with the appropriate command", () => { it("a .js data loader is called with node", async () => { const loader = Loader.find(FilePath("test"), FilePath("dataloaders/data1.txt"))!; @@ -19,7 +22,8 @@ describe("data loaders are called with the appropriate command", () => { const out = await loader.load(noopEffects); assert.strictEqual(await readFile(fileJoin("test", out), "utf-8"), "tsx\n"); }); - it("a .sh data loader is called with sh", async () => { + it("a .sh data loader is called with sh", async function () { + if (process.platform === "win32") this.skip(); // sh loader is not supported on Windows const loader = Loader.find(FilePath("test"), FilePath("dataloaders/data3.txt"))!; const out = await loader.load(noopEffects); assert.strictEqual(await readFile(fileJoin("test", out), "utf-8"), "shell\n"); @@ -27,12 +31,12 @@ describe("data loaders are called with the appropriate command", () => { it("a .exe data loader is invoked directly", async () => { const loader = Loader.find(FilePath("test"), FilePath("dataloaders/data4.txt"))!; const out = await loader.load(noopEffects); - assert.strictEqual(await readFile(fileJoin("test", out), "utf-8"), "python3\n"); + assert.strictEqual(await readFile(fileJoin("test", out), "utf-8"), `python3${lineEnding}`); }); it("a .py data loader is called with python3", async () => { const loader = Loader.find(FilePath("test"), FilePath("dataloaders/data5.txt"))!; const out = await loader.load(noopEffects); - assert.strictEqual(await readFile(fileJoin("test", out), "utf-8"), "python3\n"); + assert.strictEqual(await readFile(fileJoin("test", out), "utf-8"), `python3${lineEnding}`); }); // Skipping because this requires R to be installed (which is slow in CI). it.skip("a .R data loader is called with Rscript", async () => { @@ -42,55 +46,53 @@ describe("data loaders are called with the appropriate command", () => { }); }); -describe("data loaders optionally use a stale cache", () => { - it("a dataloader can use ", async () => { - const out = [] as string[]; - const outputEffects: LoadEffects = { - logger: {log() {}, warn() {}, error() {}}, - output: { - write(a) { - out.push(a); - } +it("a dataloader can use use a stale cache", async () => { + const out = [] as string[]; + const outputEffects: LoadEffects = { + logger: {log() {}, warn() {}, error() {}}, + output: { + write(a) { + out.push(a); } - }; - const loader = Loader.find(FilePath("test"), FilePath("dataloaders/data1.txt"))!; - // save the loader times. - const {atime, mtime} = await stat(loader.path); - // set the loader mtime to Dec. 1st, 2023. - const time = Date.UTC(2023, 11, 1) / 1000; - await utimes(loader.path, atime, time); - // remove the cache set by another test (unless we it.only this test). - try { - await unlink(FilePath("test/.observablehq/cache/dataloaders/data1.txt")); - } catch { - // ignore; } - // populate the cache (missing) - await loader.load(outputEffects); - // run again (fresh) - await loader.load(outputEffects); - // touch the loader - await utimes(loader.path, atime, Date.now() + 100); - // run it with useStale=true (using stale) - const loader2 = Loader.find(FilePath("test"), FilePath("dataloaders/data1.txt"), {useStale: true})!; - await loader2.load(outputEffects); - // run it with useStale=false (stale) - await loader.load(outputEffects); - // revert the loader to its original mtime - await utimes(loader.path, atime, mtime); - assert.deepStrictEqual( - // eslint-disable-next-line no-control-regex - out.map((l) => l.replaceAll(/\x1b\[[0-9]+m/g, "")), - [ - "load test/dataloaders/data1.txt.js → ", - "[missing] ", - "load test/dataloaders/data1.txt.js → ", - "[fresh] ", - "load test/dataloaders/data1.txt.js → ", - "[using stale] ", - "load test/dataloaders/data1.txt.js → ", - "[stale] " - ] - ); - }); + }; + const loader = Loader.find(FilePath("test"), FilePath("dataloaders/data1.txt"))!; + // save the loader times. + const {atime, mtime} = await stat(loader.path); + // set the loader mtime to Dec. 1st, 2023. + const time = new Date("2023-11-01"); + await utimes(loader.path, atime, time); + // remove the cache set by another test (unless we it.only this test). + try { + await unlink(FilePath("test/.observablehq/cache/dataloaders/data1.txt")); + } catch { + // ignore; + } + // populate the cache (missing) + await loader.load(outputEffects); + // run again (fresh) + await loader.load(outputEffects); + // touch the loader + await utimes(loader.path, atime, new Date(Date.now() + 100)); + // run it with useStale=true (using stale) + const loader2 = Loader.find(FilePath("test"), FilePath("dataloaders/data1.txt"), {useStale: true})!; + await loader2.load(outputEffects); + // run it with useStale=false (stale) + await loader.load(outputEffects); + // revert the loader to its original mtime + await utimes(loader.path, atime, mtime); + assert.deepStrictEqual( + // eslint-disable-next-line no-control-regex + out.map((l) => l.replaceAll(/\x1b\[[0-9]+m/g, "")), + [ + `load ${FilePath("test/dataloaders/data1.txt.js")} → `, + "[missing] ", + `load ${FilePath("test/dataloaders/data1.txt.js")} → `, + "[fresh] ", + `load ${FilePath("test/dataloaders/data1.txt.js")} → `, + "[using stale] ", + `load ${FilePath("test/dataloaders/data1.txt.js")} → `, + "[stale] " + ] + ); }); diff --git a/test/fileWatchers-test.ts b/test/fileWatchers-test.ts index 027f30b4f..74680f0b5 100644 --- a/test/fileWatchers-test.ts +++ b/test/fileWatchers-test.ts @@ -2,7 +2,7 @@ import assert from "node:assert"; import os from "node:os"; import {InternSet, difference} from "d3-array"; import {renameSync, unlinkSync, utimesSync, writeFileSync} from "../src/brandedFs.js"; -import {FilePath} from "../src/brandedPath.js"; +import {FilePath, UrlPath} from "../src/brandedPath.js"; import {FileWatchers} from "../src/fileWatchers.js"; describe("FileWatchers.of(root, path, names, callback)", () => { @@ -36,18 +36,17 @@ describe("FileWatchers.of(root, path, names, callback)", () => { } }); it("watches a file within a static archive", async () => { - const [watcher, watches] = await useWatcher(`test/input/build/archives.${os.platform()}`, "zip.md", [ - "static/file.txt" - ]); + const platform = os.platform() === "win32" ? "win32" : "posix"; + const [watcher, watches] = await useWatcher(`test/input/build/archives.${platform}`, "zip.md", ["static/file.txt"]); try { - touch(FilePath("test/input/build/archives/static.zip")); + touch(FilePath(`test/input/build/archives.${platform}/static.zip`)); assert.deepStrictEqual(await watches(), ["static/file.txt"]); } finally { watcher.close(); } }); it("watches a file within an archive created by a data loader", async function () { - const platform = os.platform(); + const platform = os.platform() === "win32" ? "win32" : "posix"; if (platform === "win32") this.skip(); // .sh loaders don't work on Windows const [watcher, watches] = await useWatcher(`test/input/build/archives.${platform}`, "zip.md", [ "dynamic/file.txt" @@ -157,7 +156,10 @@ describe("FileWatchers.of(root, path, names, callback)", () => { // Then touch a different file to make sure the watcher is still alive. touch(FilePath("test/input/build/files/file-top.csv")); - assert.deepStrictEqual(difference(await watches(), ["temp.csv"]), new InternSet(["file-top.csv"])); + assert.deepStrictEqual( + difference(await watches(), [FilePath("temp.csv")]), + new InternSet([FilePath("file-top.csv")]) + ); } finally { watcher.close(); } @@ -171,25 +173,25 @@ async function useWatcher( rootStr: string, pathStr: string, namesStr: string[] -): Promise<[watcher: FileWatchers, wait: (delay?: number) => Promise]> { +): Promise<[watcher: FileWatchers, wait: (delay?: number) => Promise]> { const root = FilePath(rootStr); const path = FilePath(pathStr); - const names = namesStr.map((name) => FilePath(name)); - let watches = new Set(); - let resume: ((value: FilePath[]) => void) | null = null; + const names = namesStr.map((name) => UrlPath(name)); + let watches = new Set(); + let resume: ((value: UrlPath[]) => void) | null = null; const wait = (delay?: number) => { if (resume) throw new Error("already waiting"); - const promise = new Promise((y) => (resume = y)); + const promise = new Promise((y) => (resume = y)); if (delay == null) return promise; - const timeout = new Promise((y) => setTimeout(() => y([...watches].sort()), delay)); + const timeout = new Promise((y) => setTimeout(() => y([...watches].sort()), delay)); return Promise.race([promise, timeout]); }; - const watch = (name: FilePath) => { + const watch = (name: UrlPath) => { watches.add(name); const r = resume; if (r == null) return; resume = null; - setTimeout(() => (r([...watches].sort()), (watches = new Set())), 10); + setTimeout(() => (r([...watches].sort()), (watches = new Set())), 10); }; const watcher = await FileWatchers.of(root, path, names, watch); await pause(); diff --git a/test/files-test.ts b/test/files-test.ts index c6dc1a7ea..87964660e 100644 --- a/test/files-test.ts +++ b/test/files-test.ts @@ -1,4 +1,5 @@ import assert from "node:assert"; +import os from "node:os"; import {stat} from "../src/brandedFs.js"; import {FilePath} from "../src/brandedPath.js"; import {maybeStat, prepareOutput, visitFiles, visitMarkdownFiles} from "../src/files.js"; @@ -33,26 +34,30 @@ describe("maybeStat(path)", () => { describe("visitFiles(root)", () => { it("visits all files in a directory, return the relative path from the root", async () => { assert.deepStrictEqual(await collect(visitFiles(FilePath("test/input/build/files"))), [ - "custom-styles.css", - "file-top.csv", - "files.md", - "observable logo small.png", - "observable logo.png", - "subsection/additional-styles.css", - "subsection/file-sub.csv", - "subsection/subfiles.md" + FilePath("custom-styles.css"), + FilePath("file-top.csv"), + FilePath("files.md"), + FilePath("observable logo small.png"), + FilePath("observable logo.png"), + FilePath("subsection/additional-styles.css"), + FilePath("subsection/file-sub.csv"), + FilePath("subsection/subfiles.md") ]); }); - it("handles circular symlinks, visiting files only once", async () => { - assert.deepStrictEqual(await collect(visitFiles(FilePath("test/input/circular-files"))), ["a/a.txt", "b/b.txt"]); + it("handles circular symlinks, visiting files only once", async function () { + if (os.platform() === "win32") this.skip(); // symlinks are not the same on Windows + assert.deepStrictEqual(await collect(visitFiles(FilePath("test/input/circular-files"))), [ + FilePath("a/a.txt"), + FilePath("b/b.txt") + ]); }); }); describe("visitMarkdownFiles(root)", () => { it("visits all Markdown files in a directory, return the relative path from the root", async () => { assert.deepStrictEqual(await collect(visitMarkdownFiles(FilePath("test/input/build/files"))), [ - "files.md", - "subsection/subfiles.md" + FilePath("files.md"), + FilePath("subsection/subfiles.md") ]); }); }); diff --git a/test/isLocalImport-test.ts b/test/isLocalImport-test.ts index 94718359a..569464c39 100644 --- a/test/isLocalImport-test.ts +++ b/test/isLocalImport-test.ts @@ -8,28 +8,28 @@ describe("isLocalImport", () => { const root = FilePath("docs"); const sourcePath = FilePath("/hello.md"); const importValue = "./helpers.js"; - assert.equal(resolvePath(root, sourcePath, UrlPath(importValue)), "docs/helpers.js"); + assert.equal(resolvePath(root, sourcePath, UrlPath(importValue)), FilePath("docs/helpers.js")); assert(isLocalImport(importValue, sourcePath)); }); it("relative paths are correctly handled", async () => { const root = FilePath("docs"); const sourcePath = FilePath("/subDocs/hello.md"); const importValue = "./helpers.js"; - assert.equal(resolvePath(root, sourcePath, UrlPath(importValue)), "docs/subDocs/helpers.js"); + assert.equal(resolvePath(root, sourcePath, UrlPath(importValue)), FilePath("docs/subDocs/helpers.js")); assert(isLocalImport(importValue, sourcePath)); }); it("root and sourcePath arguments can correctly handle slashes", async () => { const root = FilePath("docs/"); const sourcePath = FilePath("/hello.md/"); const importValue = "./helpers.js"; - assert.equal(resolvePath(root, sourcePath, UrlPath(importValue)), "docs/helpers.js"); + assert.equal(resolvePath(root, sourcePath, UrlPath(importValue)), FilePath("docs/helpers.js")); assert(isLocalImport(importValue, sourcePath)); }); it("identifies a local import from a nested sourcePath", async () => { const root = FilePath("docs"); const sourcePath = FilePath("/subDocs/subDocs2/hello.md"); const importValue = "../../random.js"; - assert.equal(resolvePath(root, sourcePath, UrlPath(importValue)), "docs/random.js"); + assert.equal(resolvePath(root, sourcePath, UrlPath(importValue)), FilePath("docs/random.js")); assert(isLocalImport(importValue, sourcePath)); }); it("cannot go to an ancestor directory beyond the root", async () => { @@ -39,10 +39,10 @@ describe("isLocalImport", () => { assert.equal(resolvePath(root, sourcePath, UrlPath(importValue1)), FilePath("../../random.js")); assert.equal(isLocalImport(importValue1, sourcePath), false); const importValue2 = "./../../random.js"; - assert.equal(resolvePath(root, sourcePath, UrlPath(importValue2)), "../random.js"); + assert.equal(resolvePath(root, sourcePath, UrlPath(importValue2)), FilePath("../random.js")); assert.equal(isLocalImport(importValue2, sourcePath), false); const importValue3 = "/../random.js"; - assert.equal(resolvePath(root, sourcePath, UrlPath(importValue3)), "random.js"); + assert.equal(resolvePath(root, sourcePath, UrlPath(importValue3)), FilePath("random.js")); assert.equal(isLocalImport(importValue3, sourcePath), false); }); }); diff --git a/test/observableApiConfig-test.ts b/test/observableApiConfig-test.ts index a51460852..1496d6536 100644 --- a/test/observableApiConfig-test.ts +++ b/test/observableApiConfig-test.ts @@ -4,20 +4,32 @@ import {type FilePath, fileJoin} from "../src/brandedPath.js"; import type {ConfigEffects} from "../src/observableApiConfig.js"; import {loadUserConfig} from "../src/observableApiConfig.js"; +const isWindows = os.platform() === "win32"; + describe("loadUserConfig", () => { it("checks expected directories for the config", async () => { const effects = new MockConfigEffects(); assert.deepEqual(await loadUserConfig(effects), { config: {}, - configPath: "/home/amaya/.observablehq" + configPath: isWindows ? "C:\\Users\\Amaya\\.observablehq" : "/home/amaya/.observablehq" }); - assert.deepEqual(effects._readLog, [ - "/opt/projects/acme-bi/.observablehq", - "/opt/projects/.observablehq", - "/opt/.observablehq", - "/.observablehq", - "/home/amaya/.observablehq" - ]); + assert.deepEqual( + effects._readLog, + isWindows + ? [ + "D:\\Projects\\acme-bi\\.observablehq", + "D:\\Projects\\.observablehq", + "D:\\.observablehq", + "C:\\Users\\Amaya\\.observablehq" + ] + : [ + "/opt/projects/acme-bi/.observablehq", + "/opt/projects/.observablehq", + "/opt/.observablehq", + "/.observablehq", + "/home/amaya/.observablehq" + ] + ); assert.deepEqual(effects._writeLog, []); }); }); @@ -46,14 +58,12 @@ export class MockConfigEffects implements ConfigEffects { } homedir() { - return os.platform() === "win32" ? fileJoin("C:", "Users", "Amaya") : fileJoin("/", "home", "amaya"); + return isWindows ? fileJoin("C:", "Users", "Amaya") : fileJoin("/", "home", "amaya"); } cwd() { // it is an important detail that this is not inside the home dir - return os.platform() === "win32" - ? fileJoin("D:", "Projects", "acme-bi") - : fileJoin("/", "opt", "projects", "acme-bi"); + return isWindows ? fileJoin("D:", "Projects", "acme-bi") : fileJoin("/", "opt", "projects", "acme-bi"); } } diff --git a/test/output/build/archives.win32/tar.html b/test/output/build/archives.win32/tar.html index a4a8d268e..0b85b5671 100644 --- a/test/output/build/archives.win32/tar.html +++ b/test/output/build/archives.win32/tar.html @@ -46,7 +46,7 @@ -