diff --git a/docs/core_docs/docs/integrations/chat/ollama.mdx b/docs/core_docs/docs/integrations/chat/ollama.mdx index c03892968179..1919517e956f 100644 --- a/docs/core_docs/docs/integrations/chat/ollama.mdx +++ b/docs/core_docs/docs/integrations/chat/ollama.mdx @@ -13,14 +13,14 @@ For a complete list of supported models and model variants, see the [Ollama mode ## Setup -Follow [these instructions](https://github.com/jmorganca/ollama) to set up and run a local Ollama instance. +Follow [these instructions](https://github.com/jmorganca/ollama) to set up and run a local Ollama instance. Then, download the `@langchain/ollama` package. import IntegrationInstallTooltip from "@mdx_components/integration_install_tooltip.mdx"; ```bash npm2yarn -npm install @langchain/community +npm install @langchain/ollama ``` ## Usage @@ -30,6 +30,28 @@ import OllamaExample from "@examples/models/chat/integration_ollama.ts"; {OllamaExample} +## Tools + +Ollama now offers support for native tool calling. The example below demonstrates how you can invoke a tool from an Ollama model. + +import OllamaToolsExample from "@examples/models/chat/integration_ollama_tools.ts"; + +{OllamaToolsExample} + +:::tip +You can see the LangSmith trace of the above example [here](https://smith.langchain.com/public/940f4279-6825-4d19-9653-4c50d3c70625/r) +::: + +Since `ChatOllama` supports the `.bindTools()` method, you can also call `.withStructuredOutput()` to get a structured output from the tool. + +import OllamaWSOExample from "@examples/models/chat/integration_ollama_wso.ts"; + +{OllamaWSOExample} + +:::tip +You can see the LangSmith trace of the above example [here](https://smith.langchain.com/public/ed113c53-1299-4814-817e-1157c9eac47e/r) +::: + ## JSON mode Ollama also supports a JSON mode that coerces model outputs to only return JSON. Here's an example of how this can be useful for extraction: @@ -38,7 +60,9 @@ import OllamaJSONModeExample from "@examples/models/chat/integration_ollama_json {OllamaJSONModeExample} -You can see a simple LangSmith trace of this here: https://smith.langchain.com/public/92aebeca-d701-4de0-a845-f55df04eff04/r +:::tip +You can see a simple LangSmith trace of this [here](https://smith.langchain.com/public/1fbd5660-b7fd-41c3-9d3a-a6ecc735277c/r) +::: ## Multimodal models diff --git a/docs/core_docs/docs/integrations/chat/ollama_functions.mdx b/docs/core_docs/docs/integrations/chat/ollama_functions.mdx index 19aa6e78ead5..e4b38a3f5406 100644 --- a/docs/core_docs/docs/integrations/chat/ollama_functions.mdx +++ b/docs/core_docs/docs/integrations/chat/ollama_functions.mdx @@ -4,6 +4,12 @@ sidebar_label: Ollama Functions # Ollama Functions +:::warning + +The LangChain Ollama integration package has official support for tool calling. [Click here to view the documentation](/docs/integrations/chat/ollama#tools). + +::: + LangChain offers an experimental wrapper around open source models run locally via [Ollama](https://github.com/jmorganca/ollama) that gives it the same API as OpenAI Functions. @@ -48,7 +54,9 @@ import OllamaFunctionsExtraction from "@examples/models/chat/ollama_functions/ex {OllamaFunctionsExtraction} -You can see a LangSmith trace of what this looks like here: https://smith.langchain.com/public/31457ea4-71ca-4e29-a1e0-aa80e6828883/r +:::tip +You can see a simple LangSmith trace of this [here](https://smith.langchain.com/public/74692bfc-0224-4221-b187-ddbf20d7ecc0/r) +::: ## Customization diff --git a/docs/core_docs/docs/tutorials/local_rag.ipynb b/docs/core_docs/docs/tutorials/local_rag.ipynb index 08d4fafa6e1c..be5cc79bbdd1 100644 --- a/docs/core_docs/docs/tutorials/local_rag.ipynb +++ b/docs/core_docs/docs/tutorials/local_rag.ipynb @@ -153,7 +153,7 @@ "metadata": {}, "outputs": [], "source": [ - "import { ChatOllama } from \"@langchain/community/chat_models/ollama\";\n", + "import { ChatOllama } from \"@langchain/ollama\";\n", "\n", "const ollamaLlm = new ChatOllama({\n", " baseUrl: \"http://localhost:11434\", // Default value\n", diff --git a/examples/package.json b/examples/package.json index 8f81162219bd..bd42ade656ab 100644 --- a/examples/package.json +++ b/examples/package.json @@ -52,6 +52,7 @@ "@langchain/mistralai": "workspace:*", "@langchain/mongodb": "workspace:*", "@langchain/nomic": "workspace:*", + "@langchain/ollama": "workspace:*", "@langchain/openai": "workspace:*", "@langchain/pinecone": "workspace:*", "@langchain/qdrant": "workspace:*", diff --git a/examples/src/models/chat/integration_ollama.ts b/examples/src/models/chat/integration_ollama.ts index 70e989779ac8..0015ec9d611e 100644 --- a/examples/src/models/chat/integration_ollama.ts +++ b/examples/src/models/chat/integration_ollama.ts @@ -1,4 +1,4 @@ -import { ChatOllama } from "@langchain/community/chat_models/ollama"; +import { ChatOllama } from "@langchain/ollama"; import { StringOutputParser } from "@langchain/core/output_parsers"; const model = new ChatOllama({ diff --git a/examples/src/models/chat/integration_ollama_json_mode.ts b/examples/src/models/chat/integration_ollama_json_mode.ts index cea45a20f718..d782f4804431 100644 --- a/examples/src/models/chat/integration_ollama_json_mode.ts +++ b/examples/src/models/chat/integration_ollama_json_mode.ts @@ -1,4 +1,4 @@ -import { ChatOllama } from "@langchain/community/chat_models/ollama"; +import { ChatOllama } from "@langchain/ollama"; import { ChatPromptTemplate } from "@langchain/core/prompts"; const prompt = ChatPromptTemplate.fromMessages([ @@ -25,8 +25,12 @@ const result = await chain.invoke({ console.log(result); /* - AIMessage { - content: '{"original": "I love programming", "translated": "Ich liebe das Programmieren"}', - additional_kwargs: {} - } +AIMessage { + content: '{\n' + + '"original": "I love programming",\n' + + '"translated": "Ich liebe Programmierung"\n' + + '}', + response_metadata: { ... }, + usage_metadata: { ... } +} */ diff --git a/examples/src/models/chat/integration_ollama_multimodal.ts b/examples/src/models/chat/integration_ollama_multimodal.ts index 8f2e84008fa9..e81c6a26c82b 100644 --- a/examples/src/models/chat/integration_ollama_multimodal.ts +++ b/examples/src/models/chat/integration_ollama_multimodal.ts @@ -1,4 +1,4 @@ -import { ChatOllama } from "@langchain/community/chat_models/ollama"; +import { ChatOllama } from "@langchain/ollama"; import { HumanMessage } from "@langchain/core/messages"; import * as fs from "node:fs/promises"; diff --git a/examples/src/models/chat/integration_ollama_tools.ts b/examples/src/models/chat/integration_ollama_tools.ts new file mode 100644 index 000000000000..58301357bc2a --- /dev/null +++ b/examples/src/models/chat/integration_ollama_tools.ts @@ -0,0 +1,44 @@ +import { tool } from "@langchain/core/tools"; +import { ChatOllama } from "@langchain/ollama"; +import { z } from "zod"; + +const weatherTool = tool((_) => "Da weather is weatherin", { + name: "get_current_weather", + description: "Get the current weather in a given location", + schema: z.object({ + location: z.string().describe("The city and state, e.g. San Francisco, CA"), + }), +}); + +// Define the model +const model = new ChatOllama({ + model: "llama3-groq-tool-use", +}); + +// Bind the tool to the model +const modelWithTools = model.bindTools([weatherTool]); + +const result = await modelWithTools.invoke( + "What's the weather like today in San Francisco? Ensure you use the 'get_current_weather' tool." +); + +console.log(result); +/* +AIMessage { + "content": "", + "tool_calls": [ + { + "name": "get_current_weather", + "args": { + "location": "San Francisco, CA" + }, + "type": "tool_call" + } + ], + "usage_metadata": { + "input_tokens": 177, + "output_tokens": 30, + "total_tokens": 207 + } +} +*/ diff --git a/examples/src/models/chat/integration_ollama_wso.ts b/examples/src/models/chat/integration_ollama_wso.ts new file mode 100644 index 000000000000..ad6d6ee164dd --- /dev/null +++ b/examples/src/models/chat/integration_ollama_wso.ts @@ -0,0 +1,25 @@ +import { ChatOllama } from "@langchain/ollama"; +import { z } from "zod"; + +// Define the model +const model = new ChatOllama({ + model: "llama3-groq-tool-use", +}); + +// Define the tool schema you'd like the model to use. +const schema = z.object({ + location: z.string().describe("The city and state, e.g. San Francisco, CA"), +}); + +// Pass the schema to the withStructuredOutput method to bind it to the model. +const modelWithTools = model.withStructuredOutput(schema, { + name: "get_current_weather", +}); + +const result = await modelWithTools.invoke( + "What's the weather like today in San Francisco? Ensure you use the 'get_current_weather' tool." +); +console.log(result); +/* +{ location: 'San Francisco, CA' } +*/ diff --git a/examples/src/models/chat/ollama_functions/extraction.ts b/examples/src/models/chat/ollama_functions/extraction.ts index 6024489a92a7..a812d67a484f 100644 --- a/examples/src/models/chat/ollama_functions/extraction.ts +++ b/examples/src/models/chat/ollama_functions/extraction.ts @@ -44,20 +44,36 @@ const model = new OllamaFunctions({ }); // Use a JsonOutputFunctionsParser to get the parsed JSON response directly. -const chain = await prompt.pipe(model).pipe(new JsonOutputFunctionsParser()); +const chain = prompt.pipe(model).pipe(new JsonOutputFunctionsParser()); const response = await chain.invoke({ input: "Alex is 5 feet tall. Claudia is 1 foot taller than Alex and jumps higher than him. Claudia has orange hair and Alex is blonde.", }); -console.log(response); +console.log(JSON.stringify(response, null, 2)); /* - { - people: [ - { name: 'Alex', height: 5, hairColor: 'blonde' }, - { name: 'Claudia', height: 6, hairColor: 'orange' } - ] - } +{ + "people": [ + { + "name": "Alex", + "height": 5, + "hairColor": "blonde" + }, + { + "name": "Claudia", + "height": { + "$num": 1, + "add": [ + { + "name": "Alex", + "prop": "height" + } + ] + }, + "hairColor": "orange" + } + ] +} */ diff --git a/libs/langchain-community/src/chat_models/ollama.ts b/libs/langchain-community/src/chat_models/ollama.ts index 7c037864498a..c43fdbb78d86 100644 --- a/libs/langchain-community/src/chat_models/ollama.ts +++ b/libs/langchain-community/src/chat_models/ollama.ts @@ -20,11 +20,19 @@ import { type OllamaMessage, } from "../utils/ollama.js"; +/** + * @deprecated Deprecated in favor of the `@langchain/ollama` package. Import from `@langchain/ollama` instead. + */ export interface ChatOllamaInput extends OllamaInput {} +/** + * @deprecated Deprecated in favor of the `@langchain/ollama` package. Import from `@langchain/ollama` instead. + */ export interface ChatOllamaCallOptions extends BaseLanguageModelCallOptions {} /** + * @deprecated Deprecated in favor of the `@langchain/ollama` package. Import from `@langchain/ollama` instead. + * * A class that enables calls to the Ollama API to access large language * models in a chat-like fashion. It extends the SimpleChatModel class and * implements the OllamaInput interface. diff --git a/libs/langchain-community/src/experimental/chat_models/ollama_functions.ts b/libs/langchain-community/src/experimental/chat_models/ollama_functions.ts index 2fb02ac82885..13fcc246e0c0 100644 --- a/libs/langchain-community/src/experimental/chat_models/ollama_functions.ts +++ b/libs/langchain-community/src/experimental/chat_models/ollama_functions.ts @@ -17,15 +17,24 @@ You must always select one of the above tools and respond with only a JSON objec "tool_input": }}`; +/** + * @deprecated Deprecated in favor of the `@langchain/ollama` package. Import `ChatOllama` from `@langchain/ollama` instead. + */ export interface ChatOllamaFunctionsCallOptions extends BaseFunctionCallOptions {} +/** + * @deprecated Deprecated in favor of the `@langchain/ollama` package. Import `ChatOllama` from `@langchain/ollama` instead. + */ export type OllamaFunctionsInput = Partial & BaseChatModelParams & { llm?: ChatOllama; toolSystemPromptTemplate?: string; }; +/** + * @deprecated Deprecated in favor of the `@langchain/ollama` package. Import `ChatOllama` from `@langchain/ollama` instead. + */ export class OllamaFunctions extends BaseChatModel { llm: ChatOllama; diff --git a/libs/langchain-ollama/.eslintrc.cjs b/libs/langchain-ollama/.eslintrc.cjs new file mode 100644 index 000000000000..e3033ac0160c --- /dev/null +++ b/libs/langchain-ollama/.eslintrc.cjs @@ -0,0 +1,74 @@ +module.exports = { + extends: [ + "airbnb-base", + "eslint:recommended", + "prettier", + "plugin:@typescript-eslint/recommended", + ], + parserOptions: { + ecmaVersion: 12, + parser: "@typescript-eslint/parser", + project: "./tsconfig.json", + sourceType: "module", + }, + plugins: ["@typescript-eslint", "no-instanceof"], + ignorePatterns: [ + ".eslintrc.cjs", + "scripts", + "node_modules", + "dist", + "dist-cjs", + "*.js", + "*.cjs", + "*.d.ts", + ], + rules: { + "no-process-env": 2, + "no-instanceof/no-instanceof": 2, + "@typescript-eslint/explicit-module-boundary-types": 0, + "@typescript-eslint/no-empty-function": 0, + "@typescript-eslint/no-shadow": 0, + "@typescript-eslint/no-empty-interface": 0, + "@typescript-eslint/no-use-before-define": ["error", "nofunc"], + "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }], + "@typescript-eslint/no-floating-promises": "error", + "@typescript-eslint/no-misused-promises": "error", + camelcase: 0, + "class-methods-use-this": 0, + "import/extensions": [2, "ignorePackages"], + "import/no-extraneous-dependencies": [ + "error", + { devDependencies: ["**/*.test.ts"] }, + ], + "import/no-unresolved": 0, + "import/prefer-default-export": 0, + "keyword-spacing": "error", + "max-classes-per-file": 0, + "max-len": 0, + "no-await-in-loop": 0, + "no-bitwise": 0, + "no-console": 0, + "no-restricted-syntax": 0, + "no-shadow": 0, + "no-continue": 0, + "no-void": 0, + "no-underscore-dangle": 0, + "no-use-before-define": 0, + "no-useless-constructor": 0, + "no-return-await": 0, + "consistent-return": 0, + "no-else-return": 0, + "func-names": 0, + "no-lonely-if": 0, + "prefer-rest-params": 0, + "new-cap": ["error", { properties: false, capIsNew: false }], + }, + overrides: [ + { + files: ["**/*.test.ts"], + rules: { + "@typescript-eslint/no-unused-vars": "off", + }, + }, + ], +}; diff --git a/libs/langchain-ollama/.gitignore b/libs/langchain-ollama/.gitignore new file mode 100644 index 000000000000..c10034e2f1be --- /dev/null +++ b/libs/langchain-ollama/.gitignore @@ -0,0 +1,7 @@ +index.cjs +index.js +index.d.ts +index.d.cts +node_modules +dist +.yarn diff --git a/libs/langchain-ollama/.prettierrc b/libs/langchain-ollama/.prettierrc new file mode 100644 index 000000000000..ba08ff04f677 --- /dev/null +++ b/libs/langchain-ollama/.prettierrc @@ -0,0 +1,19 @@ +{ + "$schema": "https://json.schemastore.org/prettierrc", + "printWidth": 80, + "tabWidth": 2, + "useTabs": false, + "semi": true, + "singleQuote": false, + "quoteProps": "as-needed", + "jsxSingleQuote": false, + "trailingComma": "es5", + "bracketSpacing": true, + "arrowParens": "always", + "requirePragma": false, + "insertPragma": false, + "proseWrap": "preserve", + "htmlWhitespaceSensitivity": "css", + "vueIndentScriptAndStyle": false, + "endOfLine": "lf" +} diff --git a/libs/langchain-ollama/.release-it.json b/libs/langchain-ollama/.release-it.json new file mode 100644 index 000000000000..522ee6abf705 --- /dev/null +++ b/libs/langchain-ollama/.release-it.json @@ -0,0 +1,10 @@ +{ + "github": { + "release": true, + "autoGenerate": true, + "tokenRef": "GITHUB_TOKEN_RELEASE" + }, + "npm": { + "versionArgs": ["--workspaces-update=false"] + } +} diff --git a/libs/langchain-ollama/LICENSE b/libs/langchain-ollama/LICENSE new file mode 100644 index 000000000000..8cd8f501eb49 --- /dev/null +++ b/libs/langchain-ollama/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2023 LangChain + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/libs/langchain-ollama/README.md b/libs/langchain-ollama/README.md new file mode 100644 index 000000000000..d643f8d5929f --- /dev/null +++ b/libs/langchain-ollama/README.md @@ -0,0 +1,67 @@ +# @langchain/ollama + +This package contains the LangChain.js integrations for Ollama via the `ollama` TypeScript SDK. + +## Installation + +```bash npm2yarn +npm install @langchain/ollama +``` + +TODO: add setup instructions for Ollama locally + +## Chat Models + +```typescript +import { ChatOllama } from "@langchain/ollama"; + +const model = new ChatOllama({ + model: "llama3", // Default value. +}); + +const result = await model.invoke(["human", "Hello, how are you?"]); +``` + +## Development + +To develop the `@langchain/ollama` package, you'll need to follow these instructions: + +### Install dependencies + +```bash +yarn install +``` + +### Build the package + +```bash +yarn build +``` + +Or from the repo root: + +```bash +yarn build --filter=@langchain/ollama +``` + +### Run tests + +Test files should live within a `tests/` file in the `src/` folder. Unit tests should end in `.test.ts` and integration tests should +end in `.int.test.ts`: + +```bash +$ yarn test +$ yarn test:int +``` + +### Lint & Format + +Run the linter & formatter to ensure your code is up to standard: + +```bash +yarn lint && yarn format +``` + +### Adding new entrypoints + +If you add a new file to be exported, either import & re-export from `src/index.ts`, or add it to the `entrypoints` field in the `config` variable located inside `langchain.config.js` and run `yarn build` to generate the new entrypoint. diff --git a/libs/langchain-ollama/jest.config.cjs b/libs/langchain-ollama/jest.config.cjs new file mode 100644 index 000000000000..994826496bc5 --- /dev/null +++ b/libs/langchain-ollama/jest.config.cjs @@ -0,0 +1,21 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest/presets/default-esm", + testEnvironment: "./jest.env.cjs", + modulePathIgnorePatterns: ["dist/", "docs/"], + moduleNameMapper: { + "^(\\.{1,2}/.*)\\.js$": "$1", + }, + transform: { + "^.+\\.tsx?$": ["@swc/jest"], + }, + transformIgnorePatterns: [ + "/node_modules/", + "\\.pnp\\.[^\\/]+$", + "./scripts/jest-setup-after-env.js", + ], + setupFiles: ["dotenv/config"], + testTimeout: 20_000, + passWithNoTests: true, + collectCoverageFrom: ["src/**/*.ts"], +}; diff --git a/libs/langchain-ollama/jest.env.cjs b/libs/langchain-ollama/jest.env.cjs new file mode 100644 index 000000000000..2ccedccb8672 --- /dev/null +++ b/libs/langchain-ollama/jest.env.cjs @@ -0,0 +1,12 @@ +const { TestEnvironment } = require("jest-environment-node"); + +class AdjustedTestEnvironmentToSupportFloat32Array extends TestEnvironment { + constructor(config, context) { + // Make `instanceof Float32Array` return true in tests + // to avoid https://github.com/xenova/transformers.js/issues/57 and https://github.com/jestjs/jest/issues/2549 + super(config, context); + this.global.Float32Array = Float32Array; + } +} + +module.exports = AdjustedTestEnvironmentToSupportFloat32Array; diff --git a/libs/langchain-ollama/langchain.config.js b/libs/langchain-ollama/langchain.config.js new file mode 100644 index 000000000000..59055b64eadd --- /dev/null +++ b/libs/langchain-ollama/langchain.config.js @@ -0,0 +1,22 @@ +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * @param {string} relativePath + * @returns {string} + */ +function abs(relativePath) { + return resolve(dirname(fileURLToPath(import.meta.url)), relativePath); +} + +export const config = { + internals: [/node\:/, /@langchain\/core\//, 'ollama/browser'], + entrypoints: { + index: "index", + }, + requiresOptionalDependency: [], + tsConfigPath: resolve("./tsconfig.json"), + cjsSource: "./dist-cjs", + cjsDestination: "./dist", + abs, +}; diff --git a/libs/langchain-ollama/package.json b/libs/langchain-ollama/package.json new file mode 100644 index 000000000000..aa1e8c1a25da --- /dev/null +++ b/libs/langchain-ollama/package.json @@ -0,0 +1,91 @@ +{ + "name": "@langchain/ollama", + "version": "0.0.1-rc.0", + "description": "Ollama integration for LangChain.js", + "type": "module", + "engines": { + "node": ">=18" + }, + "main": "./index.js", + "types": "./index.d.ts", + "repository": { + "type": "git", + "url": "git@github.com:langchain-ai/langchainjs.git" + }, + "homepage": "https://github.com/langchain-ai/langchainjs/tree/main/libs/langchain-ollama/", + "scripts": { + "build": "yarn turbo:command build:internal --filter=@langchain/ollama", + "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking", + "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/", + "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts", + "lint": "yarn lint:eslint && yarn lint:dpdm", + "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm", + "clean": "rm -rf .turbo dist/", + "prepack": "yarn build", + "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts", + "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000", + "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", + "format": "prettier --config .prettierrc --write \"src\"", + "format:check": "prettier --config .prettierrc --check \"src\"", + "move-cjs-to-dist": "yarn lc-build --config ./langchain.config.js --move-cjs-dist", + "create-entrypoints": "yarn lc-build --config ./langchain.config.js --create-entrypoints", + "check-tree-shaking": "yarn lc-build --config ./langchain.config.js --tree-shaking" + }, + "author": "LangChain", + "license": "MIT", + "dependencies": { + "@langchain/core": ">0.2.17 <0.3.0", + "ollama": "^0.5.6", + "uuid": "^10.0.0" + }, + "devDependencies": { + "@jest/globals": "^29.5.0", + "@langchain/scripts": "~0.0.14", + "@langchain/standard-tests": "0.0.0", + "@swc/core": "^1.3.90", + "@swc/jest": "^0.2.29", + "@tsconfig/recommended": "^1.0.3", + "@typescript-eslint/eslint-plugin": "^6.12.0", + "@typescript-eslint/parser": "^6.12.0", + "dotenv": "^16.3.1", + "dpdm": "^3.12.0", + "eslint": "^8.33.0", + "eslint-config-airbnb-base": "^15.0.0", + "eslint-config-prettier": "^8.6.0", + "eslint-plugin-import": "^2.27.5", + "eslint-plugin-no-instanceof": "^1.0.1", + "eslint-plugin-prettier": "^4.2.1", + "jest": "^29.5.0", + "jest-environment-node": "^29.6.4", + "prettier": "^2.8.3", + "release-it": "^15.10.1", + "rollup": "^4.5.2", + "ts-jest": "^29.1.0", + "typescript": "<5.2.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.23.0" + }, + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": { + "import": "./index.d.ts", + "require": "./index.d.cts", + "default": "./index.d.ts" + }, + "import": "./index.js", + "require": "./index.cjs" + }, + "./package.json": "./package.json" + }, + "files": [ + "dist/", + "index.cjs", + "index.js", + "index.d.ts", + "index.d.cts" + ] +} diff --git a/libs/langchain-ollama/scripts/jest-setup-after-env.js b/libs/langchain-ollama/scripts/jest-setup-after-env.js new file mode 100644 index 000000000000..778cf7437a20 --- /dev/null +++ b/libs/langchain-ollama/scripts/jest-setup-after-env.js @@ -0,0 +1,3 @@ +import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; + +afterAll(awaitAllCallbacks); diff --git a/libs/langchain-ollama/src/chat_models.ts b/libs/langchain-ollama/src/chat_models.ts new file mode 100644 index 000000000000..9f70a9e0e0b0 --- /dev/null +++ b/libs/langchain-ollama/src/chat_models.ts @@ -0,0 +1,513 @@ +import { + AIMessage, + UsageMetadata, + type BaseMessage, +} from "@langchain/core/messages"; +import { + BaseLanguageModelInput, + ToolDefinition, +} from "@langchain/core/language_models/base"; +import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager"; +import { + type BaseChatModelParams, + BaseChatModel, + LangSmithParams, + BaseChatModelCallOptions, +} from "@langchain/core/language_models/chat_models"; +import { Ollama } from "ollama/browser"; +import { ChatGenerationChunk, ChatResult } from "@langchain/core/outputs"; +import { AIMessageChunk } from "@langchain/core/messages"; +import type { + ChatRequest as OllamaChatRequest, + ChatResponse as OllamaChatResponse, + Message as OllamaMessage, + Tool as OllamaTool, +} from "ollama"; +import { StructuredToolInterface } from "@langchain/core/tools"; +import { Runnable, RunnableToolLike } from "@langchain/core/runnables"; +import { convertToOpenAITool } from "@langchain/core/utils/function_calling"; +import { concat } from "@langchain/core/utils/stream"; +import { + convertOllamaMessagesToLangChain, + convertToOllamaMessages, +} from "./utils.js"; + +export interface ChatOllamaCallOptions extends BaseChatModelCallOptions { + /** + * An array of strings to stop on. + */ + stop?: string[]; + tools?: (StructuredToolInterface | RunnableToolLike | ToolDefinition)[]; +} + +export interface PullModelOptions { + /** + * Whether or not to stream the download. + * @default true + */ + stream?: boolean; + insecure?: boolean; + /** + * Whether or not to log the status of the download + * to the console. + * @default false + */ + logProgress?: boolean; +} + +/** + * Input to chat model class. + */ +export interface ChatOllamaInput extends BaseChatModelParams { + /** + * The model to invoke. If the model does not exist, it + * will be pulled. + * @default "llama3" + */ + model?: string; + /** + * The host URL of the Ollama server. + * @default "http://127.0.0.1:11434" + */ + baseUrl?: string; + /** + * Whether or not to check the model exists on the local machine before + * invoking it. If set to `true`, the model will be pulled if it does not + * exist. + * @default false + */ + checkOrPullModel?: boolean; + streaming?: boolean; + numa?: boolean; + numCtx?: number; + numBatch?: number; + numGpu?: number; + mainGpu?: number; + lowVram?: boolean; + f16Kv?: boolean; + logitsAll?: boolean; + vocabOnly?: boolean; + useMmap?: boolean; + useMlock?: boolean; + embeddingOnly?: boolean; + numThread?: number; + numKeep?: number; + seed?: number; + numPredict?: number; + topK?: number; + topP?: number; + tfsZ?: number; + typicalP?: number; + repeatLastN?: number; + temperature?: number; + repeatPenalty?: number; + presencePenalty?: number; + frequencyPenalty?: number; + mirostat?: number; + mirostatTau?: number; + mirostatEta?: number; + penalizeNewline?: boolean; + format?: string; + /** + * @default "5m" + */ + keepAlive?: string | number; +} + +/** + * Integration with the Ollama SDK. + * + * @example + * ```typescript + * import { ChatOllama } from "@langchain/ollama"; + * + * const model = new ChatOllama({ + * model: "llama3", // Default model. + * }); + * + * const result = await model.invoke([ + * "human", + * "What is a good name for a company that makes colorful socks?", + * ]); + * console.log(result); + * ``` + */ +export class ChatOllama + extends BaseChatModel + implements ChatOllamaInput +{ + // Used for tracing, replace with the same name as your class + static lc_name() { + return "ChatOllama"; + } + + model = "llama3"; + + numa?: boolean; + + numCtx?: number; + + numBatch?: number; + + numGpu?: number; + + mainGpu?: number; + + lowVram?: boolean; + + f16Kv?: boolean; + + logitsAll?: boolean; + + vocabOnly?: boolean; + + useMmap?: boolean; + + useMlock?: boolean; + + embeddingOnly?: boolean; + + numThread?: number; + + numKeep?: number; + + seed?: number; + + numPredict?: number; + + topK?: number; + + topP?: number; + + tfsZ?: number; + + typicalP?: number; + + repeatLastN?: number; + + temperature?: number; + + repeatPenalty?: number; + + presencePenalty?: number; + + frequencyPenalty?: number; + + mirostat?: number; + + mirostatTau?: number; + + mirostatEta?: number; + + penalizeNewline?: boolean; + + streaming?: boolean; + + format?: string; + + keepAlive?: string | number = "5m"; + + client: Ollama; + + checkOrPullModel = false; + + baseUrl = "http://127.0.0.1:11434"; + + constructor(fields?: ChatOllamaInput) { + super(fields ?? {}); + + this.client = new Ollama({ + host: fields?.baseUrl, + }); + this.baseUrl = fields?.baseUrl ?? this.baseUrl; + + this.model = fields?.model ?? this.model; + this.numa = fields?.numa; + this.numCtx = fields?.numCtx; + this.numBatch = fields?.numBatch; + this.numGpu = fields?.numGpu; + this.mainGpu = fields?.mainGpu; + this.lowVram = fields?.lowVram; + this.f16Kv = fields?.f16Kv; + this.logitsAll = fields?.logitsAll; + this.vocabOnly = fields?.vocabOnly; + this.useMmap = fields?.useMmap; + this.useMlock = fields?.useMlock; + this.embeddingOnly = fields?.embeddingOnly; + this.numThread = fields?.numThread; + this.numKeep = fields?.numKeep; + this.seed = fields?.seed; + this.numPredict = fields?.numPredict; + this.topK = fields?.topK; + this.topP = fields?.topP; + this.tfsZ = fields?.tfsZ; + this.typicalP = fields?.typicalP; + this.repeatLastN = fields?.repeatLastN; + this.temperature = fields?.temperature; + this.repeatPenalty = fields?.repeatPenalty; + this.presencePenalty = fields?.presencePenalty; + this.frequencyPenalty = fields?.frequencyPenalty; + this.mirostat = fields?.mirostat; + this.mirostatTau = fields?.mirostatTau; + this.mirostatEta = fields?.mirostatEta; + this.penalizeNewline = fields?.penalizeNewline; + this.streaming = fields?.streaming; + this.format = fields?.format; + this.keepAlive = fields?.keepAlive ?? this.keepAlive; + this.checkOrPullModel = fields?.checkOrPullModel ?? this.checkOrPullModel; + } + + // Replace + _llmType() { + return "ollama"; + } + + /** + * Download a model onto the local machine. + * + * @param {string} model The name of the model to download. + * @param {PullModelOptions | undefined} options Options for pulling the model. + * @returns {Promise} + */ + async pull(model: string, options?: PullModelOptions): Promise { + const { stream, insecure, logProgress } = { + stream: true, + ...options, + }; + + if (stream) { + for await (const chunk of await this.client.pull({ + model, + insecure, + stream, + })) { + if (logProgress) { + console.log(chunk); + } + } + } else { + const response = await this.client.pull({ model, insecure }); + if (logProgress) { + console.log(response); + } + } + } + + override bindTools( + tools: (StructuredToolInterface | ToolDefinition | RunnableToolLike)[], + kwargs?: Partial + ): Runnable { + return this.bind({ + tools: tools.map(convertToOpenAITool), + ...kwargs, + }); + } + + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { + const params = this.invocationParams(options); + return { + ls_provider: "ollama", + ls_model_name: this.model, + ls_model_type: "chat", + ls_temperature: params.options?.temperature ?? undefined, + ls_max_tokens: params.options?.num_predict ?? undefined, + ls_stop: options.stop, + }; + } + + invocationParams( + options?: this["ParsedCallOptions"] + ): Omit { + if (options?.tool_choice) { + throw new Error("Tool choice is not supported for ChatOllama."); + } + + return { + model: this.model, + format: this.format, + keep_alive: this.keepAlive, + options: { + numa: this.numa, + num_ctx: this.numCtx, + num_batch: this.numBatch, + num_gpu: this.numGpu, + main_gpu: this.mainGpu, + low_vram: this.lowVram, + f16_kv: this.f16Kv, + logits_all: this.logitsAll, + vocab_only: this.vocabOnly, + use_mmap: this.useMmap, + use_mlock: this.useMlock, + embedding_only: this.embeddingOnly, + num_thread: this.numThread, + num_keep: this.numKeep, + seed: this.seed, + num_predict: this.numPredict, + top_k: this.topK, + top_p: this.topP, + tfs_z: this.tfsZ, + typical_p: this.typicalP, + repeat_last_n: this.repeatLastN, + temperature: this.temperature, + repeat_penalty: this.repeatPenalty, + presence_penalty: this.presencePenalty, + frequency_penalty: this.frequencyPenalty, + mirostat: this.mirostat, + mirostat_tau: this.mirostatTau, + mirostat_eta: this.mirostatEta, + penalize_newline: this.penalizeNewline, + stop: options?.stop, + }, + tools: options?.tools?.length + ? (options.tools.map(convertToOpenAITool) as OllamaTool[]) + : undefined, + }; + } + + /** + * Check if a model exists on the local machine. + * + * @param {string} model The name of the model to check. + * @returns {Promise} Whether or not the model exists. + */ + private async checkModelExistsOnMachine(model: string): Promise { + const { models } = await this.client.list(); + return !!models.find( + (m) => m.name === model || m.name === `${model}:latest` + ); + } + + async _generate( + messages: BaseMessage[], + options: this["ParsedCallOptions"], + runManager?: CallbackManagerForLLMRun + ): Promise { + if (this.checkOrPullModel) { + if (!(await this.checkModelExistsOnMachine(this.model))) { + await this.pull(this.model, { + logProgress: true, + }); + } + } + + let finalChunk: AIMessageChunk | undefined; + for await (const chunk of this._streamResponseChunks( + messages, + options, + runManager + )) { + if (!finalChunk) { + finalChunk = chunk.message; + } else { + finalChunk = concat(finalChunk, chunk.message); + } + } + + // Convert from AIMessageChunk to AIMessage since `generate` expects AIMessage. + const nonChunkMessage = new AIMessage({ + id: finalChunk?.id, + content: finalChunk?.content ?? "", + tool_calls: finalChunk?.tool_calls, + response_metadata: finalChunk?.response_metadata, + usage_metadata: finalChunk?.usage_metadata, + }); + return { + generations: [ + { + text: + typeof nonChunkMessage.content === "string" + ? nonChunkMessage.content + : "", + message: nonChunkMessage, + }, + ], + }; + } + + /** + * Implement to support streaming. + * Should yield chunks iteratively. + */ + async *_streamResponseChunks( + messages: BaseMessage[], + options: this["ParsedCallOptions"], + runManager?: CallbackManagerForLLMRun + ): AsyncGenerator { + if (this.checkOrPullModel) { + if (!(await this.checkModelExistsOnMachine(this.model))) { + await this.pull(this.model, { + logProgress: true, + }); + } + } + + const params = this.invocationParams(options); + // TODO: remove cast after SDK adds support for tool calls + const ollamaMessages = convertToOllamaMessages(messages) as OllamaMessage[]; + + const usageMetadata: UsageMetadata = { + input_tokens: 0, + output_tokens: 0, + total_tokens: 0, + }; + + if (params.tools && params.tools.length > 0) { + const toolResult = await this.client.chat({ + ...params, + messages: ollamaMessages, + stream: false, // Ollama currently does not support streaming with tools + }); + + const { message: responseMessage, ...rest } = toolResult; + usageMetadata.input_tokens += rest.prompt_eval_count ?? 0; + usageMetadata.output_tokens += rest.eval_count ?? 0; + usageMetadata.total_tokens = + usageMetadata.input_tokens + usageMetadata.output_tokens; + + yield new ChatGenerationChunk({ + text: responseMessage.content, + message: convertOllamaMessagesToLangChain(responseMessage, { + responseMetadata: rest, + usageMetadata, + }), + }); + return runManager?.handleLLMNewToken(responseMessage.content); + } + + const stream = await this.client.chat({ + ...params, + messages: ollamaMessages, + stream: true, + }); + + let lastMetadata: Omit | undefined; + + for await (const chunk of stream) { + if (options.signal?.aborted) { + this.client.abort(); + } + const { message: responseMessage, ...rest } = chunk; + usageMetadata.input_tokens += rest.prompt_eval_count ?? 0; + usageMetadata.output_tokens += rest.eval_count ?? 0; + usageMetadata.total_tokens = + usageMetadata.input_tokens + usageMetadata.output_tokens; + lastMetadata = rest; + + yield new ChatGenerationChunk({ + text: responseMessage.content ?? "", + message: convertOllamaMessagesToLangChain(responseMessage), + }); + await runManager?.handleLLMNewToken(responseMessage.content ?? ""); + } + + // Yield the `response_metadata` as the final chunk. + yield new ChatGenerationChunk({ + text: "", + message: new AIMessageChunk({ + content: "", + response_metadata: lastMetadata, + usage_metadata: usageMetadata, + }), + }); + } +} diff --git a/libs/langchain-ollama/src/index.ts b/libs/langchain-ollama/src/index.ts new file mode 100644 index 000000000000..38c7cea7f478 --- /dev/null +++ b/libs/langchain-ollama/src/index.ts @@ -0,0 +1 @@ +export * from "./chat_models.js"; diff --git a/libs/langchain-ollama/src/tests/chat_models-tools.int.test.ts b/libs/langchain-ollama/src/tests/chat_models-tools.int.test.ts new file mode 100644 index 000000000000..0d699060eca0 --- /dev/null +++ b/libs/langchain-ollama/src/tests/chat_models-tools.int.test.ts @@ -0,0 +1,108 @@ +import { + HumanMessage, + AIMessage, + ToolMessage, + AIMessageChunk, +} from "@langchain/core/messages"; +import { tool } from "@langchain/core/tools"; +import { z } from "zod"; +import { concat } from "@langchain/core/utils/stream"; +import { ChatOllama } from "../chat_models.js"; + +const messageHistory = [ + new HumanMessage("What's the weather like today in Paris?"), + new AIMessage({ + content: "", + tool_calls: [ + { + id: "89a1e453-0bce-4de3-a456-c54bed09c520", + name: "get_current_weather", + args: { + location: "Paris, France", + }, + }, + ], + }), + new ToolMessage({ + tool_call_id: "89a1e453-0bce-4de3-a456-c54bed09c520", + content: "22", + }), + new AIMessage("The weather in Paris is 22 degrees."), + new HumanMessage( + "What's the weather like today in San Francisco? Ensure you use the 'get_current_weather' tool." + ), +]; + +const weatherTool = tool((_) => "Da weather is weatherin", { + name: "get_current_weather", + description: "Get the current weather in a given location", + schema: z.object({ + location: z.string().describe("The city and state, e.g. San Francisco, CA"), + }), +}); + +test("Ollama can call tools", async () => { + const model = new ChatOllama({ + model: "llama3-groq-tool-use", + maxRetries: 1, + }).bindTools([weatherTool]); + + const result = await model.invoke(messageHistory); + expect(result).toBeDefined(); + expect(result.tool_calls?.[0]).toBeDefined(); + if (!result.tool_calls?.[0]) return; + expect(result.tool_calls[0].name).toBe("get_current_weather"); + expect(result.tool_calls[0].id).toBeDefined(); + expect(result.tool_calls[0].id).not.toBe(""); +}); + +test("Ollama can stream tools", async () => { + const model = new ChatOllama({ + model: "llama3-groq-tool-use", + maxRetries: 1, + }).bindTools([weatherTool]); + + let finalChunk: AIMessageChunk | undefined; + for await (const chunk of await model.stream(messageHistory)) { + finalChunk = !finalChunk ? chunk : concat(finalChunk, chunk); + } + expect(finalChunk).toBeDefined(); + if (!finalChunk) return; + expect(finalChunk.tool_calls?.[0]).toBeDefined(); + if (!finalChunk.tool_calls?.[0]) return; + expect(finalChunk.tool_calls[0].name).toBe("get_current_weather"); + expect(finalChunk.tool_calls[0].id).toBeDefined(); + expect(finalChunk.tool_calls[0].id).not.toBe(""); +}); + +test("Ollama can call withStructuredOutput", async () => { + const model = new ChatOllama({ + model: "llama3-groq-tool-use", + maxRetries: 1, + }).withStructuredOutput(weatherTool.schema, { + name: weatherTool.name, + }); + + const result = await model.invoke(messageHistory); + expect(result).toBeDefined(); + expect(result.location).toBeDefined(); + expect(result.location).not.toBe(""); +}); + +test("Ollama can call withStructuredOutput includeRaw", async () => { + const model = new ChatOllama({ + model: "llama3-groq-tool-use", + maxRetries: 1, + }).withStructuredOutput(weatherTool.schema, { + name: weatherTool.name, + includeRaw: true, + }); + + const result = await model.invoke(messageHistory); + expect(result).toBeDefined(); + expect(result.parsed.location).toBeDefined(); + expect(result.parsed.location).not.toBe(""); + expect((result.raw as AIMessage).tool_calls?.[0]).toBeDefined(); + expect((result.raw as AIMessage).tool_calls?.[0].id).toBeDefined(); + expect((result.raw as AIMessage).tool_calls?.[0].id).not.toBe(""); +}); diff --git a/libs/langchain-ollama/src/tests/chat_models.int.test.ts b/libs/langchain-ollama/src/tests/chat_models.int.test.ts new file mode 100644 index 000000000000..1e9aec8673ca --- /dev/null +++ b/libs/langchain-ollama/src/tests/chat_models.int.test.ts @@ -0,0 +1,187 @@ +import { test, expect } from "@jest/globals"; +import * as fs from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import * as path from "node:path"; +import { AIMessage, HumanMessage } from "@langchain/core/messages"; +import { PromptTemplate } from "@langchain/core/prompts"; +import { + BytesOutputParser, + StringOutputParser, +} from "@langchain/core/output_parsers"; +import { ChatOllama } from "../chat_models.js"; + +test("test invoke", async () => { + const ollama = new ChatOllama({ + maxRetries: 1, + }); + const result = await ollama.invoke([ + "human", + "What is a good name for a company that makes colorful socks?", + ]); + expect(result).toBeDefined(); + expect(typeof result.content).toBe("string"); + expect(result.content.length).toBeGreaterThan(1); +}); + +test("test call with callback", async () => { + const ollama = new ChatOllama({ + maxRetries: 1, + }); + const tokens: string[] = []; + const result = await ollama.invoke( + "What is a good name for a company that makes colorful socks?", + { + callbacks: [ + { + handleLLMNewToken(token: string) { + tokens.push(token); + }, + }, + ], + } + ); + expect(tokens.length).toBeGreaterThan(1); + expect(result.content).toEqual(tokens.join("")); +}); + +test("test streaming call", async () => { + const ollama = new ChatOllama({ + maxRetries: 1, + }); + const stream = await ollama.stream( + `Translate "I love programming" into German.` + ); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + expect(chunks.length).toBeGreaterThan(1); +}); + +test("should abort the request", async () => { + const ollama = new ChatOllama({ + maxRetries: 1, + }); + const controller = new AbortController(); + + await expect(() => { + const ret = ollama.invoke("Respond with an extremely verbose response", { + signal: controller.signal, + }); + controller.abort(); + return ret; + }).rejects.toThrow("This operation was aborted"); +}); + +test("Test multiple messages", async () => { + const model = new ChatOllama({ + maxRetries: 1, + }); + const res = await model.invoke([ + new HumanMessage({ content: "My name is Jonas" }), + ]); + expect(res).toBeDefined(); + expect(res.content).toBeDefined(); + const res2 = await model.invoke([ + new HumanMessage("My name is Jonas"), + new AIMessage( + "Hello Jonas! It's nice to meet you. Is there anything I can help you with?" + ), + new HumanMessage("What did I say my name was?"), + ]); + + expect(res2).toBeDefined(); + expect(res2.content).toBeDefined(); +}); + +test("should stream through with a bytes output parser", async () => { + const TEMPLATE = `You are a pirate named Patchy. All responses must be extremely verbose and in pirate dialect. + +User: {input} +AI:`; + + // Infer the input variables from the template + const prompt = PromptTemplate.fromTemplate(TEMPLATE); + + const ollama = new ChatOllama({ + maxRetries: 1, + }); + const outputParser = new BytesOutputParser(); + const chain = prompt.pipe(ollama).pipe(outputParser); + const stream = await chain.stream({ + input: `Translate "I love programming" into German.`, + }); + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + expect(chunks.length).toBeGreaterThan(1); +}); + +test("JSON mode", async () => { + const TEMPLATE = `You are a pirate named Patchy. All responses must be in pirate dialect and in JSON format, with a property named "response" followed by the value. + +User: {input} +AI:`; + + // Infer the input variables from the template + const prompt = PromptTemplate.fromTemplate(TEMPLATE); + + const ollama = new ChatOllama({ + model: "llama2", + format: "json", + maxRetries: 1, + }); + const outputParser = new StringOutputParser(); + const chain = prompt.pipe(ollama).pipe(outputParser); + const res = await chain.invoke({ + input: `Translate "I love programming" into German.`, + }); + expect(JSON.parse(res).response).toBeDefined(); +}); + +test.skip("Test ChatOllama with an image", async () => { + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + const imageData = await fs.readFile(path.join(__dirname, "/data/hotdog.jpg")); + const chat = new ChatOllama({ + model: "llava", + maxRetries: 1, + }); + const res = await chat.invoke([ + new HumanMessage({ + content: [ + { + type: "text", + text: "What is in this image?", + }, + { + type: "image_url", + image_url: `data:image/jpeg;base64,${imageData.toString("base64")}`, + }, + ], + }), + ]); + expect(res).toBeDefined(); + expect(res.content).toBeDefined(); +}); + +test("test max tokens (numPredict)", async () => { + const ollama = new ChatOllama({ + numPredict: 10, + maxRetries: 1, + }).pipe(new StringOutputParser()); + const stream = await ollama.stream( + "explain quantum physics to me in as many words as possible" + ); + let numTokens = 0; + let response = ""; + for await (const s of stream) { + numTokens += 1; + response += s; + } + + // Ollama doesn't always stream back the exact number of tokens, so we + // check for a number which is slightly above the `numPredict`. + expect(numTokens).toBeLessThanOrEqual(12); +}); diff --git a/libs/langchain-ollama/src/tests/chat_models.standard.int.test.ts b/libs/langchain-ollama/src/tests/chat_models.standard.int.test.ts new file mode 100644 index 000000000000..01b7572edeac --- /dev/null +++ b/libs/langchain-ollama/src/tests/chat_models.standard.int.test.ts @@ -0,0 +1,148 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelIntegrationTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { RunnableLambda } from "@langchain/core/runnables"; +import { z } from "zod"; +import { zodToJsonSchema } from "zod-to-json-schema"; +import { ChatOllama, ChatOllamaCallOptions } from "../chat_models.js"; + +const currentWeatherName = "get_current_weather"; +const currentWeatherDescription = + "Get the current weather for a given location."; +const currentWeatherSchema = z + .object({ + location: z + .string() + .describe("The city to get the weather for, e.g. San Francisco"), + }) + .describe(currentWeatherDescription); + +// The function calling tests can be flaky due to the model not invoking a tool. +// If the tool calling tests fail because a tool was not called, retry them. +// If they fail for another reason, there is an actual issue. +class ChatOllamaStandardIntegrationTests extends ChatModelIntegrationTests< + ChatOllamaCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatOllama, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + model: "llama3-groq-tool-use", + }, + }); + } + + /** + * Overriding base method because Ollama requires a different + * prompting method to reliably invoke tools. + */ + async testWithStructuredOutput() { + if (!this.chatModelHasStructuredOutput) { + console.log("Test requires withStructuredOutput. Skipping..."); + return; + } + + const model = new this.Cls(this.constructorArgs); + if (!model.withStructuredOutput) { + throw new Error( + "withStructuredOutput undefined. Cannot test tool message histories." + ); + } + const modelWithTools = model.withStructuredOutput(currentWeatherSchema, { + name: currentWeatherName, + }); + + const result = await modelWithTools.invoke( + "What's the weather like today in San Francisco? Use the 'get_current_weather' tool to respond." + ); + expect(result.location).toBeDefined(); + expect(typeof result.location).toBe("string"); + } + + /** + * Overriding base method because Ollama requires a different + * prompting method to reliably invoke tools. + */ + async testBindToolsWithRunnableToolLike() { + const model = new ChatOllama(this.constructorArgs); + const runnableLike = RunnableLambda.from((_) => { + // no-op + }).asTool({ + name: currentWeatherName, + description: currentWeatherDescription, + schema: currentWeatherSchema, + }); + + const modelWithTools = model.bindTools([runnableLike]); + + const result = await modelWithTools.invoke( + "What's the weather like today in San Francisco? Use the 'get_current_weather' tool to respond." + ); + expect(result.tool_calls).toHaveLength(1); + if (!result.tool_calls) { + throw new Error("result.tool_calls is undefined"); + } + const { tool_calls } = result; + expect(tool_calls[0].name).toBe(currentWeatherName); + } + + /** + * Overriding base method because Ollama requires a different + * prompting method to reliably invoke tools. + */ + async testBindToolsWithOpenAIFormattedTools() { + const model = new ChatOllama(this.constructorArgs); + + const modelWithTools = model.bindTools([ + { + type: "function", + function: { + name: currentWeatherName, + description: currentWeatherDescription, + parameters: zodToJsonSchema(currentWeatherSchema), + }, + }, + ]); + + const result = await modelWithTools.invoke( + "What's the weather like today in San Francisco? Use the 'get_current_weather' tool to respond." + ); + expect(result.tool_calls).toHaveLength(1); + if (!result.tool_calls) { + throw new Error("result.tool_calls is undefined"); + } + const { tool_calls } = result; + expect(tool_calls[0].name).toBe(currentWeatherName); + } + + /** + * Overriding base method because Ollama requires a different + * prompting method to reliably invoke tools. + */ + async testWithStructuredOutputIncludeRaw() { + const model = new ChatOllama(this.constructorArgs); + + const modelWithTools = model.withStructuredOutput(currentWeatherSchema, { + includeRaw: true, + name: currentWeatherName, + }); + + const result = await modelWithTools.invoke( + "What's the weather like today in San Francisco? Use the 'get_current_weather' tool to respond." + ); + expect(result.raw).toBeInstanceOf(this.invokeResponseType); + expect(result.parsed.location).toBeDefined(); + expect(typeof result.parsed.location).toBe("string"); + } +} + +const testClass = new ChatOllamaStandardIntegrationTests(); + +test("ChatOllamaStandardIntegrationTests", async () => { + const testResults = await testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-ollama/src/tests/chat_models.standard.test.ts b/libs/langchain-ollama/src/tests/chat_models.standard.test.ts new file mode 100644 index 000000000000..73c33ad87609 --- /dev/null +++ b/libs/langchain-ollama/src/tests/chat_models.standard.test.ts @@ -0,0 +1,36 @@ +/* eslint-disable no-process-env */ +import { test, expect } from "@jest/globals"; +import { ChatModelUnitTests } from "@langchain/standard-tests"; +import { AIMessageChunk } from "@langchain/core/messages"; +import { ChatOllama, ChatOllamaCallOptions } from "../chat_models.js"; + +class ChatOllamaStandardUnitTests extends ChatModelUnitTests< + ChatOllamaCallOptions, + AIMessageChunk +> { + constructor() { + super({ + Cls: ChatOllama, + chatModelHasToolCalling: true, + chatModelHasStructuredOutput: true, + constructorArgs: { + model: "llama3-groq-tool-use", + }, + }); + } + + testChatModelInitApiKey() { + this.skipTestMessage( + "testChatModelInitApiKey", + "ChatOllama", + "API key is not required for ChatOllama" + ); + } +} + +const testClass = new ChatOllamaStandardUnitTests(); + +test("ChatOllamaStandardUnitTests", () => { + const testResults = testClass.runTests(); + expect(testResults).toBe(true); +}); diff --git a/libs/langchain-ollama/src/tests/chat_models.test.ts b/libs/langchain-ollama/src/tests/chat_models.test.ts new file mode 100644 index 000000000000..5d609f496501 --- /dev/null +++ b/libs/langchain-ollama/src/tests/chat_models.test.ts @@ -0,0 +1,5 @@ +import { test } from "@jest/globals"; + +test("Test chat model", async () => { + // Your test here +}); diff --git a/libs/langchain-ollama/src/tests/data/hotdog.jpg b/libs/langchain-ollama/src/tests/data/hotdog.jpg new file mode 100644 index 000000000000..dfab265903be Binary files /dev/null and b/libs/langchain-ollama/src/tests/data/hotdog.jpg differ diff --git a/libs/langchain-ollama/src/utils.ts b/libs/langchain-ollama/src/utils.ts new file mode 100644 index 000000000000..9e0b9e287059 --- /dev/null +++ b/libs/langchain-ollama/src/utils.ts @@ -0,0 +1,188 @@ +import { + AIMessage, + AIMessageChunk, + BaseMessage, + HumanMessage, + MessageContentText, + SystemMessage, + ToolMessage, + UsageMetadata, +} from "@langchain/core/messages"; +import type { + Message as OllamaMessage, + ToolCall as OllamaToolCall, +} from "ollama"; +import { v4 as uuidv4 } from "uuid"; + +export function convertOllamaMessagesToLangChain( + messages: OllamaMessage, + extra?: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + responseMetadata?: Record; + usageMetadata?: UsageMetadata; + } +): AIMessageChunk { + return new AIMessageChunk({ + content: messages.content ?? "", + tool_call_chunks: messages.tool_calls?.map((tc) => ({ + name: tc.function.name, + args: JSON.stringify(tc.function.arguments), + type: "tool_call_chunk", + index: 0, + id: uuidv4(), + })), + response_metadata: extra?.responseMetadata, + usage_metadata: extra?.usageMetadata, + }); +} + +function extractBase64FromDataUrl(dataUrl: string): string { + const match = dataUrl.match(/^data:.*?;base64,(.*)$/); + return match ? match[1] : ""; +} + +function convertAMessagesToOllama(messages: AIMessage): OllamaMessage[] { + if (typeof messages.content === "string") { + return [ + { + role: "assistant", + content: messages.content, + }, + ]; + } + + const textFields = messages.content.filter( + (c) => c.type === "text" && typeof c.text === "string" + ); + const textMessages = (textFields as MessageContentText[]).map((c) => ({ + role: "assistant", + content: c.text, + })); + let toolCallMsgs: OllamaMessage | undefined; + + if ( + messages.content.find((c) => c.type === "tool_use") && + messages.tool_calls?.length + ) { + // `tool_use` content types are accepted if the message has tool calls + const toolCalls: OllamaToolCall[] | undefined = messages.tool_calls?.map( + (tc) => ({ + id: tc.id, + type: "function", + function: { + name: tc.name, + arguments: tc.args, + }, + }) + ); + + if (toolCalls) { + toolCallMsgs = { + role: "assistant", + tool_calls: toolCalls, + content: "", + }; + } + } else if ( + messages.content.find((c) => c.type === "tool_use") && + !messages.tool_calls?.length + ) { + throw new Error( + "'tool_use' content type is not supported without tool calls." + ); + } + + return [...textMessages, ...(toolCallMsgs ? [toolCallMsgs] : [])]; +} + +function convertHumanGenericMessagesToOllama( + message: HumanMessage +): OllamaMessage[] { + if (typeof message.content === "string") { + return [ + { + role: "user", + content: message.content, + }, + ]; + } + return message.content.map((c) => { + if (c.type === "text") { + return { + role: "user", + content: c.text, + }; + } else if (c.type === "image_url") { + if (typeof c.image_url === "string") { + return { + role: "user", + content: "", + images: [extractBase64FromDataUrl(c.image_url)], + }; + } else if (c.image_url.url && typeof c.image_url.url === "string") { + return { + role: "user", + content: "", + images: [extractBase64FromDataUrl(c.image_url.url)], + }; + } + } + throw new Error(`Unsupported content type: ${c.type}`); + }); +} + +function convertSystemMessageToOllama(message: SystemMessage): OllamaMessage[] { + if (typeof message.content === "string") { + return [ + { + role: "system", + content: message.content, + }, + ]; + } else if ( + message.content.every( + (c) => c.type === "text" && typeof c.text === "string" + ) + ) { + return (message.content as MessageContentText[]).map((c) => ({ + role: "system", + content: c.text, + })); + } else { + throw new Error( + `Unsupported content type(s): ${message.content + .map((c) => c.type) + .join(", ")}` + ); + } +} + +function convertToolMessageToOllama(message: ToolMessage): OllamaMessage[] { + if (typeof message.content !== "string") { + throw new Error("Non string tool message content is not supported"); + } + return [ + { + role: "tool", + content: message.content, + }, + ]; +} + +export function convertToOllamaMessages( + messages: BaseMessage[] +): OllamaMessage[] { + return messages.flatMap((msg) => { + if (["human", "generic"].includes(msg._getType())) { + return convertHumanGenericMessagesToOllama(msg); + } else if (msg._getType() === "ai") { + return convertAMessagesToOllama(msg); + } else if (msg._getType() === "system") { + return convertSystemMessageToOllama(msg); + } else if (msg._getType() === "tool") { + return convertToolMessageToOllama(msg as ToolMessage); + } else { + throw new Error(`Unsupported message type: ${msg._getType()}`); + } + }); +} diff --git a/libs/langchain-ollama/tsconfig.cjs.json b/libs/langchain-ollama/tsconfig.cjs.json new file mode 100644 index 000000000000..3b7026ea406c --- /dev/null +++ b/libs/langchain-ollama/tsconfig.cjs.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "commonjs", + "declaration": false + }, + "exclude": ["node_modules", "dist", "docs", "**/tests"] +} diff --git a/libs/langchain-ollama/tsconfig.json b/libs/langchain-ollama/tsconfig.json new file mode 100644 index 000000000000..bc85d83b6229 --- /dev/null +++ b/libs/langchain-ollama/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "@tsconfig/recommended", + "compilerOptions": { + "outDir": "../dist", + "rootDir": "./src", + "target": "ES2021", + "lib": ["ES2021", "ES2022.Object", "DOM"], + "module": "ES2020", + "moduleResolution": "nodenext", + "esModuleInterop": true, + "declaration": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "useDefineForClassFields": true, + "strictPropertyInitialization": false, + "allowJs": true, + "strict": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "docs"] +} diff --git a/libs/langchain-ollama/turbo.json b/libs/langchain-ollama/turbo.json new file mode 100644 index 000000000000..d024cee15c81 --- /dev/null +++ b/libs/langchain-ollama/turbo.json @@ -0,0 +1,11 @@ +{ + "extends": ["//"], + "pipeline": { + "build": { + "outputs": ["**/dist/**"] + }, + "build:internal": { + "dependsOn": ["^build:internal"] + } + } +} diff --git a/yarn.lock b/yarn.lock index 88c6c6fefaf6..a13ae668681f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11496,6 +11496,41 @@ __metadata: languageName: unknown linkType: soft +"@langchain/ollama@workspace:*, @langchain/ollama@workspace:libs/langchain-ollama": + version: 0.0.0-use.local + resolution: "@langchain/ollama@workspace:libs/langchain-ollama" + dependencies: + "@jest/globals": ^29.5.0 + "@langchain/core": ">0.2.17 <0.3.0" + "@langchain/scripts": ~0.0.14 + "@langchain/standard-tests": 0.0.0 + "@swc/core": ^1.3.90 + "@swc/jest": ^0.2.29 + "@tsconfig/recommended": ^1.0.3 + "@typescript-eslint/eslint-plugin": ^6.12.0 + "@typescript-eslint/parser": ^6.12.0 + dotenv: ^16.3.1 + dpdm: ^3.12.0 + eslint: ^8.33.0 + eslint-config-airbnb-base: ^15.0.0 + eslint-config-prettier: ^8.6.0 + eslint-plugin-import: ^2.27.5 + eslint-plugin-no-instanceof: ^1.0.1 + eslint-plugin-prettier: ^4.2.1 + jest: ^29.5.0 + jest-environment-node: ^29.6.4 + ollama: ^0.5.6 + prettier: ^2.8.3 + release-it: ^15.10.1 + rollup: ^4.5.2 + ts-jest: ^29.1.0 + typescript: <5.2.0 + uuid: ^10.0.0 + zod: ^3.22.4 + zod-to-json-schema: ^3.23.0 + languageName: unknown + linkType: soft + "@langchain/openai@>=0.1.0 <0.3.0, @langchain/openai@workspace:*, @langchain/openai@workspace:^, @langchain/openai@workspace:libs/langchain-openai": version: 0.0.0-use.local resolution: "@langchain/openai@workspace:libs/langchain-openai" @@ -24952,6 +24987,7 @@ __metadata: "@langchain/mistralai": "workspace:*" "@langchain/mongodb": "workspace:*" "@langchain/nomic": "workspace:*" + "@langchain/ollama": "workspace:*" "@langchain/openai": "workspace:*" "@langchain/pinecone": "workspace:*" "@langchain/qdrant": "workspace:*" @@ -33029,6 +33065,15 @@ __metadata: languageName: node linkType: hard +"ollama@npm:^0.5.6": + version: 0.5.6 + resolution: "ollama@npm:0.5.6" + dependencies: + whatwg-fetch: ^3.6.20 + checksum: f7aafe4f0cf5e3fee9f5be7501733d3ab4ea0b02e0aafacdae90cb5a8babfa4bb4543d47fab152b5424084d3331185a09e584a5d3c74e2cefcf017dc5964f520 + languageName: node + linkType: hard + "on-finished@npm:2.4.1": version: 2.4.1 resolution: "on-finished@npm:2.4.1" @@ -40402,6 +40447,13 @@ __metadata: languageName: node linkType: hard +"whatwg-fetch@npm:^3.6.20": + version: 3.6.20 + resolution: "whatwg-fetch@npm:3.6.20" + checksum: c58851ea2c4efe5c2235f13450f426824cf0253c1d45da28f45900290ae602a20aff2ab43346f16ec58917d5562e159cd691efa368354b2e82918c2146a519c5 + languageName: node + linkType: hard + "whatwg-mimetype@npm:^3.0.0": version: 3.0.0 resolution: "whatwg-mimetype@npm:3.0.0"