adding fetch tool (#1474)

pelikhan · web-flow · commit 1b52b320c41d · 2025-04-20T21:32:51.000-07:00
* adding fetch tool

* basic fetch working
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -65,6 +65,7 @@
         "echomodel",
         "emojify",
         "Entra",
+        "evalprompt",
         "Evals",
         "execa",
         "extname",
@@ -92,6 +93,7 @@
         "htmlescape",
         "huggingface",
         "icontains",
+        "importprompt",
         "jaegertracing",
         "Jamba",
         "JSONLLM",
@@ -163,12 +165,14 @@
         "previ",
         "PRICINGS",
         "priompt",
+        "promptcontext",
         "promptdom",
         "promptfoo",
         "promptfooconfig",
         "promptjson",
         "promptrunner",
         "prompty",
+        "proxify",
         "pyodide",
         "quoteify",
         "qwen",
@@ -210,6 +214,7 @@
         "tvly",
         "typecheck",
         "unfence",
+        "unmarkdown",
         "unthink",
         "unwrappers",
         "urllib",
diff --git a/docs/genaisrc/tsconfig.json b/docs/genaisrc/tsconfig.json
@@ -17,9 +17,8 @@
         "erasableSyntaxOnly": true
     },
     "include": [
-        "*.mjs",
-        "*.mts",
-        "src/*.mts",
+        "**/*.mjs",
+        "**/*.mts",
         "./genaiscript.d.ts"
     ]
 }
diff --git a/docs/src/components/BuiltinTools.mdx b/docs/src/components/BuiltinTools.mdx
@@ -6,6 +6,7 @@ import { LinkCard } from '@astrojs/starlight/components';
 
 ### Builtin tools
 
+<LinkCard title="fetch" description="Fetch data from a URL from allowed domains." href="/genaiscript/reference/scripts/system#systemfetch" />
 <LinkCard title="fs_ask_file" description="Runs a LLM query over the content of a file. Use this tool to extract information from a file." href="/genaiscript/reference/scripts/system#systemfs_ask_file" />
 <LinkCard title="fs_data_query" description="Query data in a file using GROQ syntax" href="/genaiscript/reference/scripts/system#systemfs_data_query" />
 <LinkCard title="fs_diff_files" description="Computes a diff between two different files. Use git diff instead to compare versions of a file." href="/genaiscript/reference/scripts/system#systemfs_diff_files" />
diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx
@@ -60,8 +60,7 @@ system({ ...,
     parameters: {
         model: {
             type: "string",
-            description: "LLM model to use",
-            default: "gpt-35-turbo",
+            description: "LLM model to use"
         },
     },
 })
@@ -1184,6 +1183,92 @@ export default function (ctx: ChatGenerationContext) {
 `````
 
 
+### `system.fetch`
+
+A tool that can fetch data from a URL
+
+
+
+-  tool `fetch`: Fetch data from a URL from allowed domains.
+
+`````js wrap title="system.fetch"
+system({
+    title: "A tool that can fetch data from a URL",
+    parameters: {
+        domains: {
+            type: "array",
+            items: {
+                type: "string",
+                description: "A list of allowed domains to fetch data from.",
+            },
+        },
+    },
+})
+
+export default function (ctx: ChatGenerationContext) {
+    const { defTool, env } = ctx
+
+    const dbg = host.logger(`system:fetch`)
+    const domains = env.vars["system.fetch.domains"] || []
+    dbg(`allowed domains: %o`, domains)
+
+    defTool(
+        "fetch",
+        "Fetch data from a URL from allowed domains.",
+        {
+            url: {
+                type: "string",
+                description: "The URL to fetch data from.",
+                required: true,
+            },
+            convert: {
+                type: "string",
+                description: "Converts HTML to Markdown or plain text.",
+                required: false,
+                enum: ["markdown", "text"],
+            },
+            skipToContent: {
+                type: "string",
+                description: "Skip to a specific string in the content.",
+                required: false,
+            },
+        },
+        async ({ context, ...args }) => {
+            const { url, convert, skipToContent } = args as {
+                url: string
+                convert: FetchTextOptions["convert"]
+                skipToContent: string
+            }
+            const method = "GET"
+            const uri = new URL(url)
+            const domain = uri.hostname
+            if (!domains.includes(domain))
+                return `error: domain ${domain} is not allowed.`
+
+            dbg(`${method} ${url}`)
+            const res = await host.fetchText(url, { convert })
+            dbg(`response: %d`, res.status)
+            if (!res.ok) return `error: ${res.status}`
+            if (!res.text) return res.file ?? res.status
+
+            let result = res.text
+            if (skipToContent) {
+                const index = result.indexOf(skipToContent)
+                if (index === -1)
+                    return `error: skipTo '${skipToContent}' not found.`
+                result = result.slice(index + skipToContent.length)
+            }
+            return result
+        },
+        {
+            detectPromptInjection: "available",
+        }
+    )
+}
+
+`````
+
+
 ### `system.files`
 
 File generation
diff --git a/eval/extrism/genaisrc/tsconfig.json b/eval/extrism/genaisrc/tsconfig.json
@@ -17,9 +17,8 @@
         "erasableSyntaxOnly": true
     },
     "include": [
-        "*.mjs",
-        "*.mts",
-        "src/*.mts",
+        "**/*.mjs",
+        "**/*.mts",
         "./genaiscript.d.ts"
     ]
 }
diff --git a/genaisrc/tsconfig.json b/genaisrc/tsconfig.json
@@ -17,9 +17,8 @@
         "erasableSyntaxOnly": true
     },
     "include": [
-        "*.mjs",
-        "*.mts",
-        "src/*.mts",
+        "**/*.mjs",
+        "**/*.mts",
         "./genaiscript.d.ts"
     ]
 }
diff --git a/packages/auto/tsconfig.json b/packages/auto/tsconfig.json
@@ -17,9 +17,8 @@
         "erasableSyntaxOnly": true
     },
     "include": [
-        "*.mjs",
-        "*.mts",
-        "src/*.mts",
+        "**/*.mjs",
+        "**/*.mts",
         "./genaiscript.d.ts"
     ]
 }
diff --git a/packages/cli/genaisrc/system.fetch.genai.mts b/packages/cli/genaisrc/system.fetch.genai.mts
@@ -0,0 +1,73 @@
+system({
+    title: "A tool that can fetch data from a URL",
+    parameters: {
+        domains: {
+            type: "array",
+            items: {
+                type: "string",
+                description: "A list of allowed domains to fetch data from.",
+            },
+        },
+    },
+})
+
+export default function (ctx: ChatGenerationContext) {
+    const { defTool, env } = ctx
+
+    const dbg = host.logger(`system:fetch`)
+    const domains = env.vars["system.fetch.domains"] || []
+    dbg(`allowed domains: %o`, domains)
+
+    defTool(
+        "fetch",
+        "Fetch data from a URL from allowed domains.",
+        {
+            url: {
+                type: "string",
+                description: "The URL to fetch data from.",
+                required: true,
+            },
+            convert: {
+                type: "string",
+                description: "Converts HTML to Markdown or plain text.",
+                required: false,
+                enum: ["markdown", "text"],
+            },
+            skipToContent: {
+                type: "string",
+                description: "Skip to a specific string in the content.",
+                required: false,
+            },
+        },
+        async ({ context, ...args }) => {
+            const { url, convert, skipToContent } = args as {
+                url: string
+                convert: FetchTextOptions["convert"]
+                skipToContent: string
+            }
+            const method = "GET"
+            const uri = new URL(url)
+            const domain = uri.hostname
+            if (!domains.includes(domain))
+                return `error: domain ${domain} is not allowed.`
+
+            dbg(`${method} ${url}`)
+            const res = await host.fetchText(url, { convert })
+            dbg(`response: %d`, res.status)
+            if (!res.ok) return `error: ${res.status}`
+            if (!res.text) return res.file ?? res.status
+
+            let result = res.text
+            if (skipToContent) {
+                const index = result.indexOf(skipToContent)
+                if (index === -1)
+                    return `error: skipTo '${skipToContent}' not found.`
+                result = result.slice(index + skipToContent.length)
+            }
+            return result
+        },
+        {
+            detectPromptInjection: "available",
+        }
+    )
+}
diff --git a/packages/core/bundleprompts.js b/packages/core/bundleprompts.js
@@ -218,8 +218,7 @@ system({ ...,
     parameters: {
         model: {
             type: "string",
-            description: "LLM model to use",
-            default: "gpt-35-turbo",
+            description: "LLM model to use"
         },
     },
 })
diff --git a/packages/core/src/chatrenderterminal.ts b/packages/core/src/chatrenderterminal.ts
@@ -30,14 +30,17 @@ const dbg = genaiscriptDebug("chat:render")
 
 function renderTrimmed(s: string, rows: number, width: number) {
     const lines = s.split(/\n/g).filter((l) => !!l)
-    const head = Math.min(rows >> 1, lines.length - 1)
-    const tail = rows - head
-    const trimmed = lines.slice(0, head)
-    if (tail) {
-        const hidden = lines.length - head - tail
-        if (hidden === 1) trimmed.push(lines.at(-tail - 1))
-        else if (hidden > 0) trimmed.push(`... (${hidden} lines)`)
-        trimmed.push(...lines.slice(-tail))
+    let trimmed = lines.slice(0)
+    if (lines.length > rows) {
+        const head = Math.min(rows >> 1, lines.length - 1)
+        const tail = rows - head
+        trimmed = lines.slice(0, head)
+        if (tail) {
+            const hidden = lines.length - head - tail
+            if (hidden === 1) trimmed.push(lines.at(-tail - 1))
+            else if (hidden > 0) trimmed.push(`... (${hidden} lines)`)
+            trimmed.push(...lines.slice(-tail))
+        }
     }
     const res = trimmed.map((l) =>
         wrapColor(CONSOLE_COLOR_DEBUG, "│" + ellipse(l, width) + "\n")
diff --git a/packages/core/src/fetchtext.ts b/packages/core/src/fetchtext.ts
@@ -9,7 +9,7 @@ import { deleteUndefinedValues } from "./cleaners"
 import { prettyBytes } from "./pretty"
 import debug from "debug"
 import { uriRedact } from "./url"
-import { HTMLToMarkdown, HTMLToText } from "./html"
+import { HTMLTablesToJSON, HTMLToMarkdown, HTMLToText } from "./html"
 import { createFetch } from "./fetch"
 const dbg = debug("genaiscript:fetch:text")
 
@@ -110,6 +110,8 @@ export async function fetchText(
             })
         else if (convert === "text")
             content = await HTMLToText(content, { trace, cancellationToken })
+        else if (convert === "tables")
+            content = JSON.stringify(await HTMLTablesToJSON(content))
     }
     ok = true
     const file: WorkspaceFile = deleteUndefinedValues({
diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts
@@ -5695,7 +5695,7 @@ type FetchOptions = RequestInit & {
 }
 
 type FetchTextOptions = Omit<FetchOptions, "body" | "signal" | "window"> & {
-    convert?: "markdown" | "text"
+    convert?: "markdown" | "text" | "tables"
 }
 
 interface PythonRuntimeOptions {
diff --git a/packages/core/src/vars.ts b/packages/core/src/vars.ts
diff --git a/packages/sample/genaisrc/blog/tsconfig.json b/packages/sample/genaisrc/blog/tsconfig.json
diff --git a/packages/sample/genaisrc/fetch-tools.genai.mts b/packages/sample/genaisrc/fetch-tools.genai.mts
diff --git a/packages/sample/genaisrc/tsconfig.json b/packages/sample/genaisrc/tsconfig.json
diff --git a/packages/vscode/genaisrc/tsconfig.json b/packages/vscode/genaisrc/tsconfig.json

Original file line number	Diff line number	Diff line change
`@@ -17,9 +17,8 @@`
`17`	`17`	`"erasableSyntaxOnly": true`
`18`	`18`	`},`
`19`	`19`	`"include": [`
`20`		`- "*.mjs",`
`21`		`- "*.mts",`
`22`		`- "src/*.mts",`
	`20`	`+ "*/.mjs",`
	`21`	`+ "*/.mts",`
`23`	`22`	`"./genaiscript.d.ts"`
`24`	`23`	`]`
`25`	`24`	`}`