Move all packages to cjs

cli/.gitignore

@@ -1 +1 @@
-lib
+lib/

cli/src/fs.test.ts

@@ -1,8 +1,8 @@
 import {
   loadContent,
   makeCLIContent,
-} from "@ailly/core/dist/content/content.js";
-import { makePipelineSettings } from "@ailly/core/dist/index.js";
+} from "@ailly/core/lib/content/content.js";
+import { makePipelineSettings } from "@ailly/core/lib/index.js";
 import {
   FileSystem,
   RecordFileSystemAdapter,
@@ -80,7 +80,7 @@ describe("makeCLIContent", () => {
       false
     );
 
-    expect(cliContent).toEqual({
+    expect(cliContent).toMatchObject({
       name: "stdout",
       outPath: "/dev/stdout",
       path: "/dev/stdout",
@@ -113,7 +113,7 @@ describe("makeCLIContent", () => {
       false
     );
 
-    expect(cliContent).toEqual({
+    expect(cliContent).toMatchObject({
       name: "stdout",
       outPath: "/dev/stdout",
       path: "/dev/stdout",
@@ -146,7 +146,7 @@ describe("makeCLIContent", () => {
       false
     );
 
-    expect(cliContent).toEqual({
+    expect(cliContent).toMatchObject({
       name: "stdout",
       outPath: "/dev/stdout",
       path: "/dev/stdout",
@@ -179,7 +179,7 @@ describe("makeCLIContent", () => {
       false
     );
 
-    expect(cliContent).toEqual({
+    expect(cliContent).toMatchObject({
       name: "stdout",
       outPath: "/dev/stdout",
       path: "/dev/stdout",

cli/src/fs.ts

@@ -4,12 +4,12 @@ import {
   View,
   loadContent,
   makeCLIContent,
-} from "@ailly/core/dist/content/content.js";
+} from "@ailly/core/lib/content/content.js";
 import {
   PipelineSettings,
   LOGGER as ROOT_LOGGER,
   makePipelineSettings,
-} from "@ailly/core/dist/index.js";
+} from "@ailly/core/lib/index.js";
 import { assertExists } from "@davidsouther/jiffies/lib/cjs/assert.js";
 import { FileSystem } from "@davidsouther/jiffies/lib/cjs/fs.js";
 import {
@@ -184,7 +184,7 @@ export async function loadTemplateView(
 }
 
 async function readAll(readable: typeof process.stdin): Promise<string> {
-  return new Promise((resolve, reject) => {
+  return new Promise<string>((resolve, reject) => {
     const chunks: string[] = [];
 
     readable.on("readable", () => {

cli/src/index.ts

@@ -1,12 +1,12 @@
 #! /usr/bin/env node
-import { GenerateManager } from "@ailly/core/dist/actions/generate_manager.js";
+import { GenerateManager } from "@ailly/core/lib/actions/generate_manager.js";
 import {
-  AillyEdit,
-  Content,
   isAillyEditReplace,
   writeContent,
-} from "@ailly/core/dist/content/content.js";
-import { GitignoreFs } from "@ailly/core/dist/content/gitignore_fs.js";
+  type AillyEdit,
+  type Content,
+} from "@ailly/core/lib/content/content.js";
+import { GitignoreFs } from "@ailly/core/lib/content/gitignore_fs.js";
 import { FileSystem } from "@davidsouther/jiffies/lib/cjs/fs.js";
 import { NodeFileSystemAdapter } from "@davidsouther/jiffies/lib/cjs/fs_node.js";
 import { createInterface } from "node:readline";

cli/src/version.ts

@@ -1,7 +1,5 @@
-import { version as core } from "@ailly/core/dist/index.js";
-import { getRevision, getVersion } from "@ailly/core/dist/version.js";
-import { join, normalize } from "node:path";
-import { fileURLToPath } from "node:url";
+import { version as core } from "@ailly/core/lib/index.js";
+import { getRevision, getVersion } from "@ailly/core/lib/version.js";
 
 export function version() {
   let dirname: string | undefined;

cli/tsconfig.json

@@ -1,8 +1,9 @@
 {
   "extends": "../tsconfig.base.json",
   "compilerOptions": {
-    "rootDir": "src",
-    "outDir": "lib"
+    "rootDir": "./src",
+    "outDir": "./lib"
   },
-  "include": ["src"]
+  "include": ["src/**/*"],
+  "exclude": ["src/**/*.test.ts"]
 }

core/.gitignore

@@ -0,0 +1 @@
+lib

core/.npmignore

@@ -1,4 +1,4 @@
 *
 **/tsconfig.tsbuildinfo
-!dist/**/*
-!src/**/*
+!lib/**/*
+!src/**/*

core/package.json

@@ -9,7 +9,7 @@
     "node": ">=16"
   },
   "scripts": {
-    "vscode:prepublish": "npm run build",
+    "check": "tsc --noEmit",
     "build": "tsc",
     "test": "vitest",
     "ci": "vitest --run --reporter=verbose"

core/src/actions/augment.ts

@@ -1,6 +1,6 @@
-import { Content } from "../content/content.js";
+import type { Content } from "../content/content.js";
+import { LOGGER } from "../index.js";
 import { RAG } from "../plugin/rag.js";
-import { LOGGER } from "../util.js";
 
 export async function augment(content: Content[], rag: RAG): Promise<void> {
   const _content = [...content];

core/src/actions/generate_manager.ts

@@ -1,24 +1,24 @@
+import { Content } from "../content/content";
+import { partitionPrompts } from "../content/partition.js";
+import { Engine, getEngine } from "../engine";
 import {
   DEFAULT_ENGINE,
   PipelineSettings,
   Thread,
-  getEngine,
   getPlugin,
 } from "../index.js";
-import { LOGGER } from "../util.js";
-import { partitionPrompts } from "../content/partition.js";
+import type { Plugin } from "../plugin/index.js";
+import { LOGGER, withResolvers, type PromiseWithResolvers } from "../util.js";
 import {
   PromptThread,
   PromptThreadSummary,
   PromptThreadsSummary,
 } from "./prompt_thread.js";
 
-import type { Content } from "../content/content";
-import type { Plugin } from "../plugin";
-import type { Engine } from "../engine";
-
 export class GenerateManager {
   done: boolean = false;
+  settled: PromiseWithResolvers<PromiseSettledResult<Content>[]> =
+    withResolvers();
   started: boolean = false;
   threads: Thread[];
   threadRunners: PromptThread[] = [];
@@ -51,7 +51,7 @@ export class GenerateManager {
       PromptThread.run(t, this.context, this.settings, this.engine, this.rag)
     );
 
-    this.allSettled().then(() => {
+    this.settled.promise.then(() => {
       this.done = true;
     });
   }
@@ -77,11 +77,26 @@ export class GenerateManager {
     );
   }
 
+  drainAll() {
+    this.threads.forEach((thread) =>
+      thread.forEach(async (c) => {
+        const stream = await c.responseStream.promise;
+        if (!stream.locked) {
+          for await (const _ of stream as unknown as AsyncIterable<string>) {
+            // Drain c.responseStream
+          }
+        }
+      })
+    );
+  }
+
   async allSettled(): Promise<PromiseSettledResult<Content>[]> {
     const runners = this.threadRunners.map((r) => r.allSettled());
     const runnersPromises = Promise.all(runners);
+    this.drainAll();
     const settled = await runnersPromises;
     const flattened = settled.flat();
+    this.settled.resolve(flattened);
     return flattened;
   }
 

core/src/actions/prompt_thread.test.ts

@@ -1,25 +1,25 @@
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { range } from "@davidsouther/jiffies/lib/cjs/range.js";
-import { PromptThread, generateOne, scheduler } from "./prompt_thread";
-import { LOGGER } from "../util";
-import { cleanState } from "@davidsouther/jiffies/lib/cjs/scope/state";
-import { loadContent } from "../content/content";
 import {
   FileSystem,
   ObjectFileSystemAdapter,
 } from "@davidsouther/jiffies/lib/cjs/fs";
-import { getPlugin, makePipelineSettings } from "..";
-import { getEngine } from "../engine";
 import { LEVEL } from "@davidsouther/jiffies/lib/cjs/log";
-import { TIMEOUT } from "../engine/noop";
+import { range } from "@davidsouther/jiffies/lib/cjs/range.js";
+import { cleanState } from "@davidsouther/jiffies/lib/cjs/scope/state";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { getPlugin, makePipelineSettings } from "..";
+import { loadContent } from "../content/content.js";
+import { getEngine } from "../engine/index.js";
+import { TIMEOUT } from "../engine/noop.js";
+import { LOGGER, withResolvers } from "../util.js";
+import { PromptThread, drain, generateOne, scheduler } from "./prompt_thread";
 
 describe("scheduler", () => {
   it("limits outstanding tasks", async () => {
     const tasks = range(0, 5).map((i) => ({
       i,
       started: false,
       finished: false,
-      ...Promise.withResolvers<void>(),
+      ...withResolvers<void>(),
     }));
     const runners = tasks.map((task) => async () => {
       console.log(`starting ${task.i}`);
@@ -84,7 +84,7 @@ describe("generateOne", () => {
   });
 
   it("skips some", async () => {
-    generateOne(
+    await generateOne(
       state.context["/a.txt"],
       state.context,
       await makePipelineSettings({ root: "/", overwrite: false }),
@@ -93,7 +93,7 @@ describe("generateOne", () => {
     expect(state.logger.info).toHaveBeenCalledWith("Skipping /a.txt");
     state.logger.info.mockClear();
 
-    generateOne(
+    await generateOne(
       state.context["/b.txt"],
       state.context,
       await makePipelineSettings({ root: "/" }),
@@ -112,6 +112,7 @@ describe("generateOne", () => {
       await makePipelineSettings({ root: "/" }),
       state.engine
     );
+    await drain(content);
     expect(state.logger.info).toHaveBeenCalledWith("Preparing /c.txt");
     expect(state.logger.info).toHaveBeenCalledWith("Calling noop");
     expect(content.response).toMatch(/^noop response for c.txt:/);

core/src/actions/prompt_thread.ts

@@ -1,4 +1,4 @@
-import { DEFAULT_SCHEDULER_LIMIT, PipelineSettings } from "../index.js";
+import { DEFAULT_SCHEDULER_LIMIT, LOGGER, PipelineSettings } from "../index.js";
 import { View, type Content } from "../content/content.js";
 import type { Engine } from "../engine/index.js";
 import type { Plugin } from "../plugin/index.js";
@@ -7,7 +7,6 @@ import {
   mergeContentViews,
   mergeViews,
 } from "../content/template.js";
-import { LOGGER } from "../util.js";
 
 export interface PromptThreadsSummary {
   totalPrompts: number;
@@ -180,7 +179,9 @@ export function generateOne(
     LOGGER.info(`Skipping ${c.path}`);
     const stream = new TextEncoderStream();
     stream.writable.getWriter().write(c.response ?? "");
-    c.responseStream = stream.readable;
+    c.responseStream.resolve(
+      stream.readable.pipeThrough(new TextDecoderStream())
+    );
     return Promise.resolve();
   }
 
@@ -210,7 +211,7 @@ export function generateOne(
   };
   try {
     const generator = engine.generate(c, settings);
-    c.responseStream = generator.stream;
+    c.responseStream.resolve(generator.stream);
     return generator.done.then(
       () => {
         c.response = generator.message();
@@ -225,3 +226,12 @@ export function generateOne(
     return Promise.resolve();
   }
 }
+
+export async function drain(content: Content) {
+  const stream = await content.responseStream.promise;
+  if (stream.locked) {
+    return;
+  }
+  for await (const _ of stream as unknown as AsyncIterable<string>) {
+  }
+}