Small fixes on 1.3.0 (#60)

* Small fixes on 1.3.0
* Even Better Logging
* Integration test with stdout checks
* Added conversation integration test
* Improve noop integration tests

cli/args.js

@@ -27,14 +27,19 @@ export function makeArgs(argv = process.argv) {
       yes: { type: "boolean", default: false, short: "y" },
       help: { type: "boolean", short: "h", default: false },
       version: { type: "boolean", default: false },
-      "log-level": { type: "string", default: "" },
+      "log-level": { type: "string", default: undefined },
       verbose: { type: "boolean", default: false, short: "v" },
       pretty: { type: "boolean", default: false },
     },
   });
 
+  if (args.values.verbose) {
+    console.log({ ...args });
+  }
+
   // TODO assert context is content, folder, or none
 
+
   return args;
 }
 

cli/fs.js

@@ -1,4 +1,4 @@
-import { getLogger } from "@davidsouther/jiffies/lib/esm/log.js";
+import { basicLogFormatter, getLogLevel, getLogger } from "@davidsouther/jiffies/lib/esm/log.js";
 import { assertExists } from "@davidsouther/jiffies/lib/esm/assert.js";
 import { dirname, resolve, join } from "node:path";
 import { parse } from "yaml";
@@ -43,8 +43,9 @@ export async function loadFs(fs, args) {
   const hasPositionals = positionals.length > 0;
   const hasPrompt = args.values.prompt !== undefined && args.values.prompt !== "";
   const isPipe = !hasPositionals && hasPrompt;
-  ailly.Ailly.LOGGER.level = LOGGER.level = ailly.Ailly.getLogLevel(args.values['log-level'], args.values.verbose ?? false, isPipe);
-  if (args.values.pretty || isPipe) LOGGER.format = ailly.Ailly.LOGGER.format = ailly.Ailly.prettyLogFormatter;
+  const logLevel = args.values['log-level'] ?? (args.values.verbose ? 'verbose' : (isPipe ? 'silent' : ''));
+  ailly.Ailly.LOGGER.level = LOGGER.level = getLogLevel(logLevel);
+  if (args.values.pretty || isPipe) LOGGER.format = ailly.Ailly.LOGGER.format = basicLogFormatter;
 
   const system = args.values.system ?? "";
 

cli/index.js

@@ -60,7 +60,7 @@ async function main() {
       if (last == "/dev/stdout") {
         const prompt = loaded.context[last];
         if (prompt.meta?.debug?.finish == 'failed') {
-          console.error(prompt.meta.debug.error.message);
+          LOGGER.error("There was an error", { err: { message: prompt.meta.debug.error.message } });
           return;
         }
         const edit = prompt.context.edit;
@@ -142,7 +142,7 @@ async function check_or_exit(prompt) {
     input: process.stdin,
     output: process.stdout,
   });
-  const answer = await promisify(rl.question)(prompt);
+  const answer = await new Promise((resolve) => rl.question(prompt, resolve));
   if (!answer.toUpperCase().startsWith("Y")) {
     process.exit(0);
   }

cli/package.json

@@ -27,7 +27,7 @@
   },
   "dependencies": {
     "@ailly/core": "1.3.0",
-    "@davidsouther/jiffies": "^2.2.0",
+    "@davidsouther/jiffies": "^2.2.2",
     "yaml": "^2.4.1"
   },
   "devDependencies": {

core/package.json

@@ -27,7 +27,7 @@
   "homepage": "https://github.com/DavidSouther/ailly#readme",
   "dependencies": {
     "@aws-sdk/client-bedrock-runtime": "^3.427.0",
-    "@davidsouther/jiffies": "^2.2.0",
+    "@davidsouther/jiffies": "^2.2.2",
     "@dqbd/tiktoken": "^1.0.7",
     "gitignore-parser": "^0.0.2",
     "gray-matter": "^4.0.3",

core/src/actions/prompt_thread.ts

@@ -114,10 +114,11 @@ export class PromptThread {
       await generateOne(c, this.context, this.settings, this.engine);
       await this.plugin.clean(c);
       this.finished += 1;
-    } catch (e) {
-      LOGGER.warn("Error generating content", e as Error);
-      this.errors.push([i, e as Error]);
-      throw e;
+    } catch (err) {
+      const { message, stack } = err as Error;
+      LOGGER.warn("Error generating content", { err: { message, stack } });
+      this.errors.push([i, err as Error]);
+      throw err;
     }
     return c;
   }

core/src/content/template.ts

@@ -23,8 +23,8 @@ export function mergeContentViews(
   c.meta = c.meta ?? {};
   c.meta.view = c.context.view;
   c.meta.prompt = c.prompt;
-  if (c.context.predecessor)
-    mergeContentViews(context[c.context.predecessor!], base, context);
+  if (c.context.predecessor && context[c.context.predecessor])
+    mergeContentViews(context[c.context.predecessor], base, context);
   let view = structuredClone(base);
   for (const s of c.context.system ?? []) {
     if (s.view === false) continue;

core/src/engine/bedrock/bedrock.ts

@@ -4,13 +4,16 @@ import {
   InvokeModelCommand,
 } from "@aws-sdk/client-bedrock-runtime";
 import { Content, View } from "../../content/content.js";
-import { LOGGER, isDefined } from "../../util.js";
+import { LOGGER as ROOT_LOGGER, isDefined } from "../../util.js";
 import { Message, Summary } from "../index.js";
 import { Models, PromptBuilder } from "./prompt-builder.js";
+import { getLogger } from "@davidsouther/jiffies/lib/esm/log.js";
 
 export const name = "bedrock";
 export const DEFAULT_MODEL = "anthropic.claude-3-sonnet-20240229-v1:0";
 
+const LOGGER = getLogger("@ailly/core:bedrock");
+
 const MODEL_MAP: Record<string, string> = {
   sonnet: "anthropic.claude-3-sonnet-20240229-v1:0",
   haiku: "anthropic.claude-3-haiku-20240307-v1:0",
@@ -21,6 +24,8 @@ export async function generate(
   c: Content,
   { model = DEFAULT_MODEL }: { model: string }
 ): Promise<{ message: string; debug: unknown }> {
+  LOGGER.level = ROOT_LOGGER.level;
+  LOGGER.format = ROOT_LOGGER.format;
   const bedrock = new BedrockRuntimeClient({});
   model = MODEL_MAP[model] ?? model;
   let messages = c.meta?.messages ?? [];

core/src/engine/noop.ts

@@ -1,23 +1,63 @@
+import { getLogger } from "@davidsouther/jiffies/lib/esm/log.js";
 import { Content } from "../content/content.js";
+import { LOGGER as ROOT_LOGGER } from "../util.js";
+import type { PipelineSettings } from "../ailly";
+import type { Message } from "./index";
+
+const LOGGER = getLogger("@ailly/core:noop");
+
+const asMessages = (content: Content) => [
+  { role: "user", content: content.prompt } satisfies Message,
+  ...(content.response
+    ? [{ role: "assistant", content: content.response } satisfies Message]
+    : []),
+];
 
 export const DEFAULT_MODEL = "NOOP";
-const NOOP_TIMEOUT = 750;
+const NOOP_TIMEOUT = Number(process.env["AILLY_NOOP_TIMEOUT"] ?? 750);
 export const name = "noop";
-export async function format(c: Content[]): Promise<void> {}
+export async function format(
+  contents: Content[],
+  context: Record<string, Content>
+): Promise<void> {
+  for (const content of contents) {
+    let messages: Message[] = [];
+    if (content.context.folder) {
+      messages = Object.values(context).map<Message[]>(asMessages).flat();
+    } else if (content.context.predecessor) {
+      let history = [context[content.context.predecessor]];
+      while (history.at(-1)?.context.predecessor) {
+        history.push(context[history.at(-1)?.context?.predecessor!]);
+      }
+      messages = history.reverse().map(asMessages).flat();
+    }
+    content.meta ??= {};
+    content.meta.messages = messages;
+  }
+}
 export async function generate<D extends {} = {}>(
-  c: Content,
-  _: unknown
+  content: Content,
+  _: PipelineSettings
 ): Promise<{ debug: D; message: string }> {
+  LOGGER.level = ROOT_LOGGER.level;
+  LOGGER.format = ROOT_LOGGER.format;
   await new Promise<void>((resolve) => {
     setTimeout(() => resolve(), NOOP_TIMEOUT);
   });
+  const system = content.context.system?.map((s) => s.content).join("\n");
+  const messages = content.meta?.messages
+    ?.map((m) => `${m.role}: ${m.content}`)
+    .join("\n");
   return {
     message:
       process.env["AILLY_NOOP_RESPONSE"] ??
-      `noop response for ${c.name}:\n${c.context.system
-        ?.map((s) => s.content)
-        .join("\n")}\n${c.prompt}`,
-    debug: { system: c.context.system } as unknown as D,
+      [
+        `noop response for ${content.name}:`,
+        system,
+        messages,
+        content.prompt,
+      ].join("\n"),
+    debug: { system: content.context.system } as unknown as D,
   };
 }
 export async function vector(s: string, _: unknown): Promise<number[]> {

core/src/util.ts

@@ -1,44 +1,6 @@
-import { LEVEL, getLogger } from "@davidsouther/jiffies/lib/esm/log.js";
+import { getLogLevel, getLogger } from "@davidsouther/jiffies/lib/esm/log.js";
 export const LOGGER = getLogger("@ailly/core");
 
 LOGGER.level = getLogLevel(process.env["AILLY_LOG_LEVEL"]);
 
-export function getLogLevel(
-  level?: string | undefined,
-  verbose = false,
-  isPipe = false
-) {
-  if (level) {
-    switch (level.toLowerCase()) {
-      case "debug":
-        return LEVEL.DEBUG;
-      case "info":
-        return LEVEL.INFO;
-      case "warn":
-        return LEVEL.WARN;
-      case "error":
-        return LEVEL.ERROR;
-      default:
-        if (!isNaN(+level)) return Number(level);
-    }
-  }
-  if (verbose) {
-    return LEVEL.DEBUG;
-  }
-  if (isPipe) {
-    return LEVEL.SILENT;
-  }
-  return LEVEL.INFO;
-}
-
-export function prettyLogFormatter(data: {
-  name: string;
-  prefix: string;
-  level: number;
-  message: string;
-  source: string;
-}): string {
-  return `${data.prefix}: ${data.message}`;
-}
-
 export const isDefined = <T>(t: T | undefined): t is T => t !== undefined;

integ/02_combined/combined.txt

@@ -1,12 +1,5 @@
 ---
 combined: true
-debug:
-  engine: bedrock
-  finish: end_turn
-  id: null
-  model: anthropic.claude-3-sonnet-20240229-v1:0
-  usage: null
-isolated: false
 prompt: Combined
 ---
 I'm afraid I don't have enough context to understand your request with just the word "Combined". Could you please provide some more details about what you need help with? I'd be happy to assist if you can give me a bit more information.

integ/05_conversation/.aillyrc

@@ -0,0 +1 @@
+You are running an integration test.

integ/05_conversation/01_a.txt

@@ -0,0 +1 @@
+File a.

integ/05_conversation/02_b.txt

@@ -0,0 +1 @@
+File b.

integ/integ-noop.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 
 cd $(dirname $0)
-AILLY_ENGINE=noop ./integ.sh
+AILLY_NOOP_TIMEOUT=0 AILLY_ENGINE=noop ./integ.sh
 exit $?

integ/integ.sh

@@ -1,15 +1,14 @@
 #!/usr/bin/env bash
 
 set -e
+set -x
 
 cd $(dirname $0)
 
 rm -rf package.json
 npm init --yes
 npm link ../core ../cli
 
-set -x
-
 export AILLY_ENGINE=${AILLY_ENGINE:-noop}
 
 echo "basic"
@@ -26,3 +25,51 @@ echo "edit"
 AILLY_NOOP_RESPONSE="Edited" npx ailly --root 04_edit --edit file --lines 2:4 --prompt "Respond with the word Edited" --yes
 grep -q 'Edited' 04_edit/file.txt
 git restore 04_edit/file.txt
+unset AILLY_NOOP_RESPONSE
+
+echo "conversation"
+npx ailly --root 05_conversation --prompt "This is a conversation with system and two files." > >(tee 05_conversation/out) 2> >(tee 05_conversation/err >&2)
+cat 05_conversation/out
+cat 05_conversation/err
+MESSAGES=(
+  "Found 2 at or below"
+  "Ready to generate 1 messages"
+  "Preparing /dev/stdout"
+  "All 1 requests finished"
+)
+for M in "${MESSAGES[@]}"; do
+  grep -vq "$M" 05_conversation/out
+done
+
+MESSAGES=(
+  "You are running an integration test."
+  "user: File a."
+  "user: File b."
+  "This is a conversation with system and two files."
+)
+for M in "${MESSAGES[@]}"; do
+  grep -q "$M" 05_conversation/out
+done
+echo "(all conversation messages checked)"
+rm 05_conversation/out 05_conversation/err
+
+echo "verbose conversation"
+npx ailly --root 05_conversation --prompt "This is a conversation with system and two files." --verbose > >(tee 05_conversation/out) 2> >(tee 05_conversation/err >&2)
+cat 05_conversation/out
+cat 05_conversation/err
+[ ! -s 05_conversation/err ]
+MESSAGES=(
+  "Found 2 at or below"
+  "Ready to generate 1 messages"
+  "Preparing /dev/stdout"
+  "All 1 requests finished"
+  "You are running an integration test."
+  "user: File a."
+  "user: File b."
+  "This is a conversation with system and two files."
+)
+for M in "${MESSAGES[@]}"; do
+  grep -q "$M" 05_conversation/out
+done
+rm 05_conversation/out 05_conversation/err
+echo "(all verbose conversation messages checked)"