diff --git a/package.json b/package.json index c80e5032..c71cd70a 100644 --- a/package.json +++ b/package.json @@ -238,6 +238,7 @@ ], "dependencies": { "esbuild": "0.25.4", + "ignore": "^7.0.5", "jwt-decode": "^4.0.0", "prettier": "^3.0.0" }, diff --git a/src/bundler/index.test.ts b/src/bundler/index.test.ts index c074fa19..96eef30d 100644 --- a/src/bundler/index.test.ts +++ b/src/bundler/index.test.ts @@ -1,5 +1,7 @@ import { expect, test, afterEach, vi } from "vitest"; import { oneoffContext } from "./context.js"; +import { TestFilesystem } from "./test_helpers.js"; +import * as fs from "fs"; // Although these tests are run as ESM by ts-lint, this file is built as both // CJS and ESM by TypeScript so normal recipes like `__dirname` for getting the @@ -13,6 +15,7 @@ import { entryPointsByEnvironment, useNodeDirectiveRegex, mustBeIsolate, + loadConvexIgnore, } from "./index.js"; const sorted = (arr: T[], key: (el: T) => any): T[] => { @@ -180,3 +183,237 @@ test("must use isolate", () => { expect(mustBeIsolate("schema2.js")).not.toBeTruthy(); expect(mustBeIsolate("schema/http.js")).not.toBeTruthy(); }); + +test("loadConvexIgnore loads patterns from .convexignore file", async () => { + const ctx = await getDefaultCtx(); + + const fixtureContent = fs.readFileSync( + dirname + "/test_fixtures/convexignore/basic/.convexignore", + "utf8" + ); + + const testFs = new TestFilesystem({ + ".convexignore": fixtureContent + }); + + const mockCtx = { ...ctx, fs: testFs }; + const ig = loadConvexIgnore(mockCtx, "/"); + + // Test that the patterns are loaded correctly + expect(ig.ignores("file.test.ts")).toBeTruthy(); + expect(ig.ignores("component.spec.js")).toBeTruthy(); + expect(ig.ignores("tmp/file.ts")).toBeTruthy(); + expect(ig.ignores("data.tmp")).toBeTruthy(); + expect(ig.ignores("ignored-file.ts")).toBeTruthy(); + + // Test that non-ignored files are not ignored + expect(ig.ignores("regular-file.ts")).toBeFalsy(); + expect(ig.ignores("src/index.ts")).toBeFalsy(); +}); + +test("loadConvexIgnore checks multiple locations for .convexignore", async () => { + const ctx = await getDefaultCtx(); + + // Only the convex/.convexignore exists + const testFs = new TestFilesystem({ + "test": { + "project": { + "convex": { + ".convexignore": "*.ignored" + } + } + } + }); + + const mockCtx = { ...ctx, fs: testFs }; + const ig = loadConvexIgnore(mockCtx, "/test/project"); + + // Should load from convex/.convexignore when root .convexignore doesn't exist + expect(ig.ignores("file.ignored")).toBeTruthy(); + expect(ig.ignores("file.ts")).toBeFalsy(); +}); + +test("entryPoints respects .convexignore patterns", async () => { + const ctx = await getDefaultCtx(); + + const testFs = new TestFilesystem({ + "test": { + "convex": { + ".convexignore": "ignored.ts\n*.test.ts", + "normal.ts": 'export const foo = "bar";', + "ignored.ts": 'export const ignored = true;', + "file.test.ts": 'export const test = true;', + } + } + }); + + const mockCtx = { ...ctx, fs: testFs }; + const entries = await entryPoints(mockCtx, "/test/convex"); + + // Should only include normal.ts, not the ignored files + expect(entries).toHaveLength(1); + expect(entries[0]).toContain("normal.ts"); + expect(entries[0]).not.toContain("ignored.ts"); + expect(entries[0]).not.toContain("file.test.ts"); +}); + +test("loadConvexIgnore handles no .convexignore file gracefully", async () => { + const ctx = await getDefaultCtx(); + + const testFs = new TestFilesystem({ + "test": { + "project": { + "some-file.ts": "export default {};" + } + } + }); + + const mockCtx = { ...ctx, fs: testFs }; + const ig = loadConvexIgnore(mockCtx, "/test/project"); + + // Should not throw and should return an ignore instance that ignores nothing + expect(ig.ignores("any-file.ts")).toBeFalsy(); +}); + +test("bundle respects .convexignore with multiple convex files", async () => { + const ctx = await getDefaultCtx(); + + const fixtureContent = fs.readFileSync( + dirname + "/test_fixtures/convexignore/multiple_files/.convexignore", + "utf8" + ); + + const testFs = new TestFilesystem({ + "test": { + "convex": { + ".convexignore": fixtureContent, + "api.ts": 'export default "api content";', + "_private.ts": 'export default "private content";', + "_utils.ts": 'export default "utils content";', + "old-api.ts": 'export default "old api content";', + "mutations.ts": 'export default "mutations content";', + "queries.ts": 'export default "queries content";', + } + } + }); + + const mockCtx = { ...ctx, fs: testFs }; + const entries = await entryPoints(mockCtx, "/test/convex"); + + // Should include only non-ignored files + expect(entries.map(e => e.split('/').pop())).toEqual( + expect.arrayContaining(["api.ts", "mutations.ts", "queries.ts"]) + ); + expect(entries).not.toContain(expect.stringContaining("_private.ts")); + expect(entries).not.toContain(expect.stringContaining("_utils.ts")); + expect(entries).not.toContain(expect.stringContaining("old-api.ts")); +}); + +test(".convexignore handles complex patterns and edge cases", async () => { + const ctx = await getDefaultCtx(); + + const fixtureContent = fs.readFileSync( + dirname + "/test_fixtures/convexignore/complex_patterns/.convexignore", + "utf8" + ); + + const testFs = new TestFilesystem({ + ".convexignore": fixtureContent + }); + + const mockCtx = { ...ctx, fs: testFs }; + const ig = loadConvexIgnore(mockCtx, "/"); + + // Test glob patterns + expect(ig.ignores("deep/nested/file.test.ts")).toBeTruthy(); + expect(ig.ignores("test/helper.ts")).toBeTruthy(); + expect(ig.ignores("src/test/utils.ts")).toBeTruthy(); + expect(ig.ignores("node_modules/package/index.js")).toBeTruthy(); + + // Test negation + expect(ig.ignores("important.test.ts")).toBeFalsy(); + + // Test directories + expect(ig.ignores("build/output.js")).toBeTruthy(); + expect(ig.ignores("dist/bundle.js")).toBeTruthy(); + expect(ig.ignores(".next/static/chunk.js")).toBeTruthy(); + + // Test extensions + expect(ig.ignores("debug.log")).toBeTruthy(); + expect(ig.ignores("file.tmp")).toBeTruthy(); + expect(ig.ignores("backup.bak")).toBeTruthy(); + + // Test hidden files + expect(ig.ignores(".gitignore")).toBeTruthy(); + expect(ig.ignores(".env")).toBeTruthy(); + expect(ig.ignores(".env.local")).toBeFalsy(); // negated +}); + +test(".convexignore with invalid patterns doesn't break bundling", async () => { + const ctx = await getDefaultCtx(); + + const testFs = new TestFilesystem({ + ".convexignore": `# Some valid patterns +*.test.ts + +# These patterns are actually valid in gitignore format +# Square brackets and parentheses are literal characters unless properly escaped +[invalid +(unclosed + +# More valid patterns +tmp/` + }); + + const mockCtx = { ...ctx, fs: testFs }; + const ig = loadConvexIgnore(mockCtx, "/"); + + // Valid patterns should still work + expect(ig.ignores("file.test.ts")).toBeTruthy(); + expect(ig.ignores("tmp/file.js")).toBeTruthy(); + + // These patterns behave differently - parentheses match literally, brackets don't + expect(ig.ignores("[invalid")).toBeFalsy(); + expect(ig.ignores("(unclosed")).toBeTruthy(); + + // Non-matching files should not be ignored + expect(ig.ignores("valid-file.ts")).toBeFalsy(); +}); + +test(".convexignore respects gitignore semantics", async () => { + const ctx = await getDefaultCtx(); + + const fixtureContent = fs.readFileSync( + dirname + "/test_fixtures/convexignore/gitignore_semantics/.convexignore", + "utf8" + ); + + const testFs = new TestFilesystem({ + ".convexignore": fixtureContent + }); + + const mockCtx = { ...ctx, fs: testFs }; + const ig = loadConvexIgnore(mockCtx, "/"); + + // Root-only patterns + expect(ig.ignores("root-only.ts")).toBeTruthy(); + expect(ig.ignores("subdir/root-only.ts")).toBeFalsy(); + + // Directory patterns + expect(ig.ignores("logs/error.log")).toBeTruthy(); + expect(ig.ignores("logs")).toBeFalsy(); // directories need trailing slash + + // Double asterisk patterns + expect(ig.ignores("generated/code.ts")).toBeTruthy(); + expect(ig.ignores("src/generated/api.ts")).toBeTruthy(); + expect(ig.ignores("deep/nested/generated/file.ts")).toBeTruthy(); + + // Wildcard patterns + expect(ig.ignores("api.test.ts")).toBeTruthy(); + expect(ig.ignores("utils.test.js")).toBeTruthy(); + + // Single character wildcard + expect(ig.ignores("file1.ts")).toBeTruthy(); + expect(ig.ignores("fileA.ts")).toBeTruthy(); + expect(ig.ignores("file10.ts")).toBeFalsy(); // two characters +}); diff --git a/src/bundler/index.ts b/src/bundler/index.ts index a924aff2..965b2b3f 100644 --- a/src/bundler/index.ts +++ b/src/bundler/index.ts @@ -15,6 +15,7 @@ import { findExactVersionAndDependencies, } from "./external.js"; import { innerEsbuild, isEsbuildBuildError } from "./debugBundle.js"; +import ignore from "ignore"; export { nodeFs, RecordingFs } from "./fs.js"; export type { Filesystem } from "./fs.js"; @@ -147,9 +148,9 @@ async function doEsbuild( // all the relevant information. printedMessage: recommendUseNode ? `\nIt looks like you are using Node APIs from a file without the "use node" directive.\n` + - `Split out actions using Node.js APIs like this into a new file only containing actions that uses "use node" ` + - `so these actions will run in a Node.js environment.\n` + - `For more information see https://docs.convex.dev/functions/runtimes#nodejs-runtime\n` + `Split out actions using Node.js APIs like this into a new file only containing actions that uses "use node" ` + + `so these actions will run in a Node.js environment.\n` + + `For more information see https://docs.convex.dev/functions/runtimes#nodejs-runtime\n` : null, }); } @@ -335,6 +336,29 @@ export async function doesImportConvexHttpRouter(source: string) { } } +export function loadConvexIgnore(ctx: Context, projectRoot: string) { + const ig = ignore(); + const candidates = [ + path.join(projectRoot, ".convexignore"), + path.join(projectRoot, "convex", ".convexignore"), + ]; + + const existingFiles = candidates + .filter(p => ctx.fs.exists(p)) + .map(p => { + logVerbose(chalk.green(`Loading .convexignore from ${p}`)); + return ctx.fs.readUtf8File(p); + }); + + if (existingFiles.length === 0) { + logVerbose(chalk.gray("No .convexignore file found, all files will be processed")); + } else { + existingFiles.forEach(patterns => ig.add(patterns)); + } + + return ig; +} + const ENTRY_POINT_EXTENSIONS = [ // ESBuild js loader ".js", @@ -357,11 +381,22 @@ export async function entryPoints( ): Promise { const entryPoints = []; + // Load .convexignore patterns + const projectRoot = path.dirname(dir); + const ig = loadConvexIgnore(ctx, projectRoot); + for (const { isDir, path: fpath, depth } of walkDir(ctx.fs, dir)) { if (isDir) { continue; } const relPath = path.relative(dir, fpath); + + // Check if file should be ignored based on .convexignore + if (ig.ignores(relPath)) { + logVerbose(chalk.yellow(`Skipping ignored file ${fpath}`)); + continue; + } + const parsedPath = path.parse(fpath); const base = parsedPath.base; const extension = parsedPath.ext.toLowerCase(); diff --git a/src/bundler/test_fixtures/convexignore/basic/.convexignore b/src/bundler/test_fixtures/convexignore/basic/.convexignore new file mode 100644 index 00000000..ef13c2a4 --- /dev/null +++ b/src/bundler/test_fixtures/convexignore/basic/.convexignore @@ -0,0 +1,10 @@ +# Ignore test files +*.test.ts +*.spec.js + +# Ignore temporary files +tmp/ +*.tmp + +# Ignore specific file +ignored-file.ts \ No newline at end of file diff --git a/src/bundler/test_fixtures/convexignore/complex_patterns/.convexignore b/src/bundler/test_fixtures/convexignore/complex_patterns/.convexignore new file mode 100644 index 00000000..1ddceedb --- /dev/null +++ b/src/bundler/test_fixtures/convexignore/complex_patterns/.convexignore @@ -0,0 +1,23 @@ +# Empty lines and comments should be ignored + +# Glob patterns +**/*.test.ts +**/test/** +node_modules/ + +# Negation patterns +!important.test.ts + +# Directory patterns +build/ +dist/ +.next/ + +# Extension patterns +*.log +*.tmp +*.bak + +# Hidden files +.* +!.env.local \ No newline at end of file diff --git a/src/bundler/test_fixtures/convexignore/gitignore_semantics/.convexignore b/src/bundler/test_fixtures/convexignore/gitignore_semantics/.convexignore new file mode 100644 index 00000000..9d586985 --- /dev/null +++ b/src/bundler/test_fixtures/convexignore/gitignore_semantics/.convexignore @@ -0,0 +1,14 @@ +# Leading slash for root-only matches +/root-only.ts + +# Trailing slash for directories only +logs/ + +# Double asterisk for any depth +**/generated/** + +# Single asterisk for any characters +*.test.* + +# Question mark for single character +file?.ts \ No newline at end of file diff --git a/src/bundler/test_fixtures/convexignore/multiple_files/.convexignore b/src/bundler/test_fixtures/convexignore/multiple_files/.convexignore new file mode 100644 index 00000000..28a69595 --- /dev/null +++ b/src/bundler/test_fixtures/convexignore/multiple_files/.convexignore @@ -0,0 +1,8 @@ +# Ignore private files +_*.ts + +# Ignore test helpers +helpers/*.ts + +# Specific files +old-api.ts \ No newline at end of file diff --git a/src/bundler/test_helpers.ts b/src/bundler/test_helpers.ts new file mode 100644 index 00000000..899dd1d6 --- /dev/null +++ b/src/bundler/test_helpers.ts @@ -0,0 +1,198 @@ +// this is just a tst helper, we can ignore +/* eslint-disable no-restricted-syntax */ +/* eslint-disable no-restricted-imports */ + +import { Dirent, Stats, ReadStream, Mode } from "fs"; +import { Filesystem, TempPath } from "./fs.js"; +import path from "path"; + +interface TestFile { + content: string; + stats?: Partial; +} + +interface TestDirectory { + files: TestFileStructure; + stats?: Partial; +} + +export interface TestFileStructure { + [key: string]: TestFile | TestDirectory | TestFileStructure | string; +} + +function isTestDirectory(value: any): value is TestDirectory { + return value && typeof value === "object" && "files" in value; +} + +function isTestFile(value: any): value is TestFile { + return value && typeof value === "object" && "content" in value; +} + +export class TestFilesystem implements Filesystem { + private fileSystem: Map = new Map(); + private defaultStats: Stats; + + constructor(structure: TestFileStructure = {}) { + const now = new Date(); + this.defaultStats = { + dev: 1, + ino: 1, + mode: 33188, + nlink: 1, + uid: 1000, + gid: 1000, + rdev: 0, + size: 100, + blksize: 4096, + blocks: 8, + atimeMs: now.getTime(), + mtimeMs: now.getTime(), + ctimeMs: now.getTime(), + birthtimeMs: now.getTime(), + atime: now, + mtime: now, + ctime: now, + birthtime: now, + isFile: () => true, + isDirectory: () => false, + isBlockDevice: () => false, + isCharacterDevice: () => false, + isSymbolicLink: () => false, + isFIFO: () => false, + isSocket: () => false, + }; + + this.loadStructure("/", structure); + } + + private loadStructure(basePath: string, structure: TestFileStructure) { + for (const [name, value] of Object.entries(structure)) { + const fullPath = path.join(basePath, name); + + if (typeof value === "string") { + // Simple string content + this.fileSystem.set(fullPath, { content: value }); + } else if (isTestFile(value)) { + // TestFile object + this.fileSystem.set(fullPath, value); + } else if (isTestDirectory(value)) { + // TestDirectory object + this.fileSystem.set(fullPath, value); + this.loadStructure(fullPath, value.files); + } else if (typeof value === "object") { + // Plain object treated as directory + const dir: TestDirectory = { files: value as any }; + this.fileSystem.set(fullPath, dir); + this.loadStructure(fullPath, value as TestFileStructure); + } + } + } + + listDir(dirPath: string): Dirent[] { + const entries: Dirent[] = []; + const normalizedPath = path.normalize(dirPath); + + // Find all entries that are direct children of this directory + for (const [filePath, entry] of this.fileSystem.entries()) { + const dir = path.dirname(filePath); + if (dir === normalizedPath) { + const name = path.basename(filePath); + const dirent = Object.assign(Object.create(null), { + name, + path: dirPath, + parentPath: path.dirname(dirPath), + isFile: () => !isTestDirectory(entry), + isDirectory: () => isTestDirectory(entry), + isBlockDevice: () => false, + isCharacterDevice: () => false, + isSymbolicLink: () => false, + isFIFO: () => false, + isSocket: () => false, + }) as Dirent; + entries.push(dirent); + } + } + + return entries; + } + + exists(filePath: string): boolean { + return this.fileSystem.has(path.normalize(filePath)); + } + + stat(filePath: string): Stats { + const entry = this.fileSystem.get(path.normalize(filePath)); + if (!entry) { + throw new Error(`ENOENT: no such file or directory, stat '${filePath}'`); + } + + const customStats = (isTestFile(entry) ? entry.stats : isTestDirectory(entry) ? entry.stats : {}) || {}; + const isDir = isTestDirectory(entry); + + return { + ...this.defaultStats, + ...customStats, + isFile: () => !isDir, + isDirectory: () => isDir, + size: isTestFile(entry) ? entry.content.length : 0, + }; + } + + readUtf8File(filePath: string): string { + const entry = this.fileSystem.get(path.normalize(filePath)); + if (!entry) { + throw new Error(`ENOENT: no such file or directory, open '${filePath}'`); + } + if (isTestDirectory(entry)) { + throw new Error(`EISDIR: illegal operation on a directory, read '${filePath}'`); + } + return isTestFile(entry) ? entry.content : (entry as any); + } + + createReadStream(_filePath: string, _options: { highWaterMark?: number }): ReadStream { + // For test purposes, we don't need a real stream + throw new Error("createReadStream not implemented in TestFilesystem"); + } + + access(filePath: string): void { + if (!this.exists(filePath)) { + throw new Error(`ENOENT: no such file or directory, access '${filePath}'`); + } + } + + writeUtf8File(filePath: string, contents: string, _mode?: Mode): void { + this.fileSystem.set(path.normalize(filePath), { content: contents }); + } + + mkdir(dirPath: string, options?: { allowExisting?: boolean; recursive?: boolean }): void { + const normalizedPath = path.normalize(dirPath); + if (this.exists(normalizedPath) && !options?.allowExisting) { + throw new Error(`EEXIST: file already exists, mkdir '${dirPath}'`); + } + this.fileSystem.set(normalizedPath, { files: {} }); + } + + rmdir(dirPath: string): void { + this.fileSystem.delete(path.normalize(dirPath)); + } + + unlink(filePath: string): void { + this.fileSystem.delete(path.normalize(filePath)); + } + + swapTmpFile(fromPath: TempPath, toPath: string): void { + const content = this.fileSystem.get(fromPath); + if (content) { + this.fileSystem.set(path.normalize(toPath), content); + this.fileSystem.delete(fromPath); + } + } + + registerPath(_filePath: string, _st: Stats | null): void { + // No-op for test filesystem + } + + invalidate(): void { + // No-op for test filesystem + } +} \ No newline at end of file diff --git a/src/cli/lib/codegen.ts b/src/cli/lib/codegen.ts index 879a2f2c..0fb4c5c0 100644 --- a/src/cli/lib/codegen.ts +++ b/src/cli/lib/codegen.ts @@ -1,3 +1,4 @@ + import path from "path"; import prettier from "prettier"; import { withTmpDir, TempDir } from "../../bundler/fs.js";