From 1011db6cf0860dcd936d6d29620db8476a5f2c76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mussa?= Date: Fri, 21 Jun 2024 20:35:50 -0400 Subject: [PATCH 01/10] chore: implement DatadogLLMObsTracer --- .../docs/integrations/tracers/datadog.mdx | 35 ++ examples/src/tracers/datadog.ts | 30 ++ .../src/tracers/datadog.ts | 374 +++++++++++++++++ .../src/tracers/tests/datadog.test.ts | 383 ++++++++++++++++++ 4 files changed, 822 insertions(+) create mode 100644 docs/core_docs/docs/integrations/tracers/datadog.mdx create mode 100644 examples/src/tracers/datadog.ts create mode 100644 libs/langchain-community/src/tracers/datadog.ts create mode 100644 libs/langchain-community/src/tracers/tests/datadog.test.ts diff --git a/docs/core_docs/docs/integrations/tracers/datadog.mdx b/docs/core_docs/docs/integrations/tracers/datadog.mdx new file mode 100644 index 000000000000..40e84f368e2c --- /dev/null +++ b/docs/core_docs/docs/integrations/tracers/datadog.mdx @@ -0,0 +1,35 @@ +--- +sidebar_class_name: node-only +--- + +import CodeBlock from "@theme/CodeBlock"; + +# Datadog LLM Observability + +:::warning +LLM Observability is in public beta, and its API is subject to change. +::: + +With [Datadog LLM Observability](https://docs.datadoghq.com/llm_observability/), you can monitor, troubleshoot, and evaluate your LLM-powered applications, such as chatbots. You can investigate the root cause of issues, monitor operational performance, and evaluate the quality, privacy, and safety of your LLM applications. + +This is a community implementation, and it is not officially supported by Datadog. It is based on the [Datadog LLM Observability API](https://docs.datadoghq.com/llm_observability/api). + +:::tip Compatibility +Only available on Node.js. +::: + +## Setup + +import IntegrationInstallTooltip from "@mdx_components/integration_install_tooltip.mdx"; + + + +```bash npm2yarn +npm install @langchain/community +``` + +## Usage + +import UsageExample from "@examples/traces/datadog.ts"; + +{UsageExample} diff --git a/examples/src/tracers/datadog.ts b/examples/src/tracers/datadog.ts new file mode 100644 index 000000000000..d304c819cf2d --- /dev/null +++ b/examples/src/tracers/datadog.ts @@ -0,0 +1,30 @@ +import { OpenAI } from "@langchain/openai"; +import { DatadogLLMObsTracer } from "@/libs/llm-monitoring"; + +/** + * This example demonstrates how to use the DatadogLLMObsTracer with the OpenAI model. + * It will produce a "llm" span with the input and output of the model inside the meta field. + * + * To run this example, you need to have a valid Datadog API key and OpenAI API key. + */ +export const run = async () => { + const model = new OpenAI({ + model: "gpt-4", + temperature: 0.7, + maxTokens: 1000, + maxRetries: 5, + }); + + const res = await model.invoke( + "Question: What would be a good company name a company that makes colorful socks?\nAnswer:", + { + callbacks: [ + new DatadogLLMObsTracer({ + mlApp: "my-ml-app", + }), + ], + } + ); + + console.log({ res }); +}; diff --git a/libs/langchain-community/src/tracers/datadog.ts b/libs/langchain-community/src/tracers/datadog.ts new file mode 100644 index 000000000000..e393ce56a9cf --- /dev/null +++ b/libs/langchain-community/src/tracers/datadog.ts @@ -0,0 +1,374 @@ +import { BaseCallbackHandlerInput } from "@langchain/core/callbacks/base"; +import { BaseTracer, Run } from "@langchain/core/tracers/base"; +import { getEnvironmentVariable } from "@langchain/core/utils/env"; +import { Document } from "langchain/document"; + +import { BaseMessage } from "@langchain/core/messages"; +import { ChatGeneration } from "@langchain/core/outputs"; +import { KVMap } from "langsmith/schemas"; + +export type DatadogLLMObsSpanKind = + | "llm" + | "workflow" + | "agent" + | "tool" + | "task" + | "embedding" + | "retrieval"; + +export type DatadogLLMObsIO = + | { value: string } + | { + documents: { + text?: string; + id?: string; + name?: string; + score: string | number; + }[]; + } + | { messages: { content: string; role?: string }[] }; + +export interface DatadogLLMObsSpan { + span_id: string; + trace_id: string; + parent_id: string; + session_id?: string; + name: string; + start_ns: number; + duration: number; + error: number; + status: string; + meta: { + kind: DatadogLLMObsSpanKind; + model_name?: string; + model_provider?: string; + temperature?: string; + input: DatadogLLMObsIO; + output: DatadogLLMObsIO | undefined; + }; + metrics: { [key: string]: number }; +} + +export interface DatadogLLMObsRequestBody { + data: { + type: "span"; + session_id?: string; + attributes: { + ml_app: string; + tags: string[]; + spans: DatadogLLMObsSpan[]; + }; + }; +} + +export type FormatDocument< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + Metadata extends Record = Record +> = (document: Document) => { + text: string; + id: string; + name: string; + score: number; +}; + +export interface DatadogLLMObsTracerFields extends BaseCallbackHandlerInput { + mlApp: string; + userId?: string; + userHandle?: string; + sessionId?: string; + env?: string; + service?: string; + tags?: Record; + ddApiKey?: string; + ddLLMObsEndpoint?: string; + formatDocument?: FormatDocument; +} + +export class DatadogLLMObsTracer + extends BaseTracer + implements DatadogLLMObsTracerFields +{ + name = "datadog_tracer"; + + ddLLMObsEndpoint?: string; + + protected endpoint = + getEnvironmentVariable("DD_LLMOBS_ENDPOINT") || + "https://api.datadoghq.com/api/unstable/llm-obs/v1/trace/spans"; + + protected headers: Record = { + "Content-Type": "application/json", + }; + + mlApp: string; + + sessionId?: string; + + tags: Record = {}; + + formatDocument?: FormatDocument; + + constructor(fields: DatadogLLMObsTracerFields) { + super(fields); + const { + mlApp, + userHandle, + userId, + sessionId, + service, + env, + tags, + ddLLMObsEndpoint, + ddApiKey, + formatDocument, + } = fields; + + const apiKey = ddApiKey || getEnvironmentVariable("DD_API_KEY"); + + if (apiKey) { + this.headers["DD-API-KEY"] = apiKey; + } + + this.mlApp = mlApp; + this.sessionId = sessionId; + this.ddLLMObsEndpoint = ddLLMObsEndpoint; + this.formatDocument = formatDocument; + + this.tags = { + ...tags, + env: env || "not-set", + service: service || "not-set", + user_handle: userHandle, + user_id: userId, + }; + } + + protected async persistRun(_run: Run): Promise { + try { + const spans = this.convertRunToDDSpans(_run); + + const response = await fetch(this.ddLLMObsEndpoint || this.endpoint, { + method: "POST", + headers: this.headers, + body: JSON.stringify(this.formatRequestBody(spans)), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(error); + } + } catch (error) { + console.error(`Error writing spans to Datadog: ${error}`); + } + } + + protected convertRunToDDSpans(run: Run): DatadogLLMObsSpan[] { + const spans = [this.langchainRunToDatadogLLMObsSpan(run)]; + + if (run.child_runs) { + run.child_runs.forEach((childRun) => { + spans.push(...this.convertRunToDDSpans(childRun)); + }); + } + + return spans.filter((span) => span !== null); + } + + protected formatRequestBody( + spans: DatadogLLMObsSpan[] + ): DatadogLLMObsRequestBody { + return { + data: { + type: "span", + session_id: this.sessionId, + attributes: { + ml_app: this.mlApp, + tags: Object.entries(this.tags) + .filter(([, value]) => value) + .map(([key, value]) => `${key}:${value}`), + spans, + }, + }, + }; + } + + protected uuidToBigInt(uuid: string): string { + const hexString = uuid.replace(/-/g, ""); + const first64Bits = hexString.slice(0, 16); + const bigIntValue = BigInt("0x" + first64Bits).toString(); + + return bigIntValue; + } + + protected milisecondsToNanoseconds(ms: number): number { + return ms * 1e6; + } + + protected toDatadogSpanKind(kind: string): DatadogLLMObsSpanKind | null { + switch (kind) { + case "llm": + return "llm"; + case "tool": + return "tool"; + case "chain": + return "workflow"; + case "retriever": + return "retrieval"; + default: + return null; + } + } + + protected transformInput( + inputs: KVMap, + spanKind: DatadogLLMObsSpanKind + ): DatadogLLMObsIO { + if (spanKind === "llm") { + if (inputs?.messages) { + return { + messages: inputs?.messages?.flatMap((messages: BaseMessage[]) => + messages.map((message) => ({ + content: message.content, + role: message?._getType?.() ?? undefined, + })) + ), + }; + } + + if (inputs?.prompts) { + return { value: inputs.prompts.join("\n") }; + } + } + + return { value: JSON.stringify(inputs) }; + } + + protected transformOutput( + outputs: KVMap | undefined, + spanKind: DatadogLLMObsSpanKind + ): { + output: DatadogLLMObsIO | undefined; + tokensMetadata: Record; + } { + const tokensMetadata: Record = {}; + + if (!outputs) { + return { output: undefined, tokensMetadata }; + } + + if (spanKind === "llm") { + return { + output: { + messages: outputs?.generations?.flatMap( + (generations: ChatGeneration[]) => + generations.map(({ message, text }) => { + const tokenUsage = message?.response_metadata?.tokenUsage; + if (tokenUsage) { + tokensMetadata.completion_tokens = + tokenUsage.completionTokens; + tokensMetadata.prompt_tokens = tokenUsage.promptTokens; + tokensMetadata.total_tokens = tokenUsage.totalTokens; + } + + return { + content: message?.content ?? text, + role: message?._getType?.(), + }; + }) + ), + }, + tokensMetadata, + }; + } + + if (spanKind === "retrieval") { + return { + output: { + documents: outputs?.documents.map((document: Document) => { + if (typeof this.formatDocument === "function") { + return this.formatDocument(document); + } + + return { + text: document.pageContent, + id: document.metadata?.id, + name: document.metadata?.name, + score: document.metadata?.score, + }; + }), + }, + tokensMetadata, + }; + } + + if (outputs?.output) { + return { + output: { value: JSON.stringify(outputs.output) }, + tokensMetadata, + }; + } + + return { output: { value: JSON.stringify(outputs) }, tokensMetadata }; + } + + protected langchainRunToDatadogLLMObsSpan( + run: Run + ): DatadogLLMObsSpan | null { + if (!run.end_time || !run.trace_id) { + return null; + } + + const spanId = this.uuidToBigInt(run.id); + const traceId = this.uuidToBigInt(run.trace_id); + const parentId = run.parent_run_id + ? this.uuidToBigInt(run.parent_run_id) + : "undefined"; + const spanKind = this.toDatadogSpanKind(run.run_type); + + if (spanKind === null) { + return null; + } + + const input = this.transformInput(run.inputs, spanKind); + const { output, tokensMetadata } = this.transformOutput( + run.outputs, + spanKind + ); + + const startTimeNs = Number(this.milisecondsToNanoseconds(run.start_time)); + const endTimeNs = Number(this.milisecondsToNanoseconds(run.end_time)); + const durationNs = endTimeNs - startTimeNs; + + if (durationNs <= 0) { + return null; + } + + const spanName = + (run.serialized as { kwargs: { name?: string } })?.kwargs?.name ?? + run.name; + const spanError = run.error ? 1 : 0; + const spanStatus = run.error ? "error" : "ok"; + + const meta = { + kind: spanKind, + input, + output, + model_name: run.extra?.metadata?.ls_model_name, + model_provider: run.extra?.metadata?.ls_provider, + temperature: run.extra?.metadata?.ls_temperature, + }; + + return { + parent_id: parentId, + trace_id: traceId, + span_id: spanId, + name: spanName, + error: spanError, + status: spanStatus, + meta, + start_ns: startTimeNs, + duration: durationNs, + metrics: tokensMetadata, + }; + } +} diff --git a/libs/langchain-community/src/tracers/tests/datadog.test.ts b/libs/langchain-community/src/tracers/tests/datadog.test.ts new file mode 100644 index 000000000000..3a9190006fdc --- /dev/null +++ b/libs/langchain-community/src/tracers/tests/datadog.test.ts @@ -0,0 +1,383 @@ +import { test, jest, expect } from "@jest/globals"; +import * as uuid from "uuid"; +import { Run } from "@langchain/core/tracers/base"; + +import { HumanMessage, AIMessage } from "@langchain/core/messages"; +import { + DatadogLLMObsRequestBody, + DatadogLLMObsSpan, + DatadogLLMObsTracer, +} from "../datadog.js"; + +const _DATE = 1620000000000; +const _END_DATE = _DATE + 1000; + +Date.now = jest.fn(() => _DATE); + +const BASE_URL = "http://datadog-endpoint"; + +class FakeDatadogLLMObsTracer extends DatadogLLMObsTracer { + public persistRun(_run: Run) { + return super.persistRun(_run); + } + + public uuidToBigInt(uuid: string) { + return super.uuidToBigInt(uuid); + } + + public milisecondsToNanoseconds(ms: number) { + return super.milisecondsToNanoseconds(ms); + } +} + +beforeEach(() => { + const oldFetch = global.fetch; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + global.fetch = jest.fn().mockImplementation(async (url: any, init?: any) => { + if (!url.startsWith(BASE_URL)) return await oldFetch(url, init); + const resp: Response = new Response(); + return resp; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }) as any; +}); + +afterEach(() => { + jest.restoreAllMocks(); +}); + +const runId = uuid.v4(); +const traceId = uuid.v4(); + +const baseRun = { + id: runId, + trace_id: traceId, + parent_run_id: undefined, + name: "test", + start_time: _DATE, + end_time: _END_DATE, + execution_order: 1, + child_execution_order: 0, + + child_runs: [], + extra: {}, + tags: [], + events: [], +}; + +const createBaseSpan = (tracer: FakeDatadogLLMObsTracer) => ({ + span_id: tracer.uuidToBigInt(runId), + trace_id: tracer.uuidToBigInt(traceId), + parent_id: "undefined", + name: "test", + start_ns: tracer.milisecondsToNanoseconds(_DATE), + duration: tracer.milisecondsToNanoseconds(_END_DATE - _DATE), + error: 0, + status: "ok", + metrics: {}, +}); + +const tracerConfig = { + mlApp: "test", + userHandle: "test", + userId: "test", + sessionId: "test", + service: "test", + env: "test", + tags: {}, + ddLLMObsEndpoint: BASE_URL, +}; + +test("Test llm span with message input", async () => { + const tracer = new FakeDatadogLLMObsTracer(tracerConfig); + + const run: Run = { + ...baseRun, + run_type: "llm", + inputs: { + messages: [[new HumanMessage("test")]], + }, + outputs: { + generations: [ + [ + { + message: new AIMessage("test"), + }, + ], + ], + }, + }; + + const compareSpan: DatadogLLMObsSpan = { + ...createBaseSpan(tracer), + meta: { + kind: "llm", + input: { + messages: [{ content: "test", role: "human" }], + }, + output: { + messages: [{ content: "test", role: "ai" }], + }, + }, + }; + + const requestBody: DatadogLLMObsRequestBody = { + data: { + type: "span", + session_id: "test", + attributes: { + ml_app: "test", + tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], + spans: [compareSpan], + }, + }, + }; + + await tracer.persistRun(run); + + expect(fetch).toBeCalledWith(expect.any(String), { + body: expect.any(String), + headers: expect.any(Object), + method: "POST", + }); + + const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; + expect(parsedBody).toMatchObject( + requestBody as unknown as Record + ); +}); + +test("Test llm span with prompt input", async () => { + const tracer = new FakeDatadogLLMObsTracer(tracerConfig); + + const run: Run = { + ...baseRun, + run_type: "llm", + inputs: { + prompts: ["Hello", "World"], + }, + outputs: { + generations: [ + [ + { + text: "Hi", + }, + ], + ], + }, + }; + + const compareSpan: DatadogLLMObsSpan = { + ...createBaseSpan(tracer), + meta: { + kind: "llm", + input: { + value: "Hello\nWorld", + }, + output: { + messages: [{ content: "Hi" }], + }, + }, + }; + + const requestBody: DatadogLLMObsRequestBody = { + data: { + type: "span", + session_id: "test", + attributes: { + ml_app: "test", + tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], + spans: [compareSpan], + }, + }, + }; + + await tracer.persistRun(run); + + expect(fetch).toBeCalledWith(expect.any(String), { + body: expect.any(String), + headers: expect.any(Object), + method: "POST", + }); + + const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; + expect(parsedBody).toMatchObject( + requestBody as unknown as Record + ); +}); + +test("Test workflow span", async () => { + const tracer = new FakeDatadogLLMObsTracer(tracerConfig); + + const run: Run = { + ...baseRun, + run_type: "chain", + inputs: { + question: "test", + }, + outputs: { + output: "test", + }, + }; + + const compareSpan: DatadogLLMObsSpan = { + ...createBaseSpan(tracer), + meta: { + kind: "workflow", + input: { + value: JSON.stringify({ question: "test" }), + }, + output: { + value: JSON.stringify("test"), + }, + }, + }; + + const requestBody: DatadogLLMObsRequestBody = { + data: { + type: "span", + session_id: "test", + attributes: { + ml_app: "test", + tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], + spans: [compareSpan], + }, + }, + }; + + await tracer.persistRun(run); + + expect(fetch).toBeCalledWith(expect.any(String), { + body: expect.any(String), + headers: expect.any(Object), + method: "POST", + }); + + const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; + expect(parsedBody).toMatchObject( + requestBody as unknown as Record + ); +}); + +test("Test tool span", async () => { + const tracer = new FakeDatadogLLMObsTracer(tracerConfig); + + const run: Run = { + ...baseRun, + run_type: "tool", + inputs: { + input: { query: "test" }, + }, + outputs: { + output: "test", + }, + }; + + const compareSpan: DatadogLLMObsSpan = { + ...createBaseSpan(tracer), + meta: { + kind: "tool", + input: { + value: JSON.stringify({ input: { query: "test" } }), + }, + output: { + value: JSON.stringify("test"), + }, + }, + }; + + const requestBody: DatadogLLMObsRequestBody = { + data: { + type: "span", + session_id: "test", + attributes: { + ml_app: "test", + tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], + spans: [compareSpan], + }, + }, + }; + + await tracer.persistRun(run); + + expect(fetch).toBeCalledWith(expect.any(String), { + body: expect.any(String), + headers: expect.any(Object), + method: "POST", + }); + + const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; + expect(parsedBody).toMatchObject( + requestBody as unknown as Record + ); +}); + +test("Test retrieval span", async () => { + const tracer = new FakeDatadogLLMObsTracer(tracerConfig); + + const run: Run = { + ...baseRun, + run_type: "retriever", + inputs: { + input: { query: "test" }, + }, + outputs: { + documents: [ + { + pageContent: "test", + metadata: { id: "1", name: "test", score: 0.1 }, + }, + ], + }, + }; + + const compareSpan: DatadogLLMObsSpan = { + ...createBaseSpan(tracer), + meta: { + kind: "retrieval", + input: { + value: JSON.stringify({ input: { query: "test" } }), + }, + output: { + documents: [ + { + text: "test", + id: "1", + name: "test", + score: 0.1, + }, + ], + }, + }, + }; + + const requestBody: DatadogLLMObsRequestBody = { + data: { + type: "span", + session_id: "test", + attributes: { + ml_app: "test", + tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], + spans: [compareSpan], + }, + }, + }; + + await tracer.persistRun(run); + + expect(fetch).toBeCalledWith(expect.any(String), { + body: expect.any(String), + headers: expect.any(Object), + method: "POST", + }); + + const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; + expect(parsedBody).toMatchObject( + requestBody as unknown as Record + ); +}); From 31ff829db0cc61401d19acecba7f74269c34c779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mussa?= Date: Fri, 21 Jun 2024 20:43:22 -0400 Subject: [PATCH 02/10] chore: declare module into langchain-community/langchain.config.js --- libs/langchain-community/langchain.config.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libs/langchain-community/langchain.config.js b/libs/langchain-community/langchain.config.js index 53b32cfb4dd3..99cd8215769a 100644 --- a/libs/langchain-community/langchain.config.js +++ b/libs/langchain-community/langchain.config.js @@ -27,7 +27,7 @@ export const config = { "pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js", "web-auth-library/google", "notion-to-md/build/utils/notion.js", - "@getzep/zep-cloud/api" + "@getzep/zep-cloud/api", ], entrypoints: { load: "load/index", @@ -188,7 +188,8 @@ export const config = { // callbacks "callbacks/handlers/llmonitor": "callbacks/handlers/llmonitor", "callbacks/handlers/lunary": "callbacks/handlers/lunary", - "callbacks/handlers/upstash_ratelimit": "callbacks/handlers/upstash_ratelimit", + "callbacks/handlers/upstash_ratelimit": + "callbacks/handlers/upstash_ratelimit", // retrievers "retrievers/amazon_kendra": "retrievers/amazon_kendra", "retrievers/amazon_knowledge_base": "retrievers/amazon_knowledge_base", @@ -313,9 +314,12 @@ export const config = { "experimental/multimodal_embeddings/googlevertexai", "experimental/hubs/makersuite/googlemakersuitehub": "experimental/hubs/makersuite/googlemakersuitehub", - "experimental/chat_models/ollama_functions": "experimental/chat_models/ollama_functions", + "experimental/chat_models/ollama_functions": + "experimental/chat_models/ollama_functions", // chains - "chains/graph_qa/cypher": "chains/graph_qa/cypher" + "chains/graph_qa/cypher": "chains/graph_qa/cypher", + // tracers + "tracers/datadog": "tracers/datadog", }, requiresOptionalDependency: [ "tools/aws_sfn", @@ -511,7 +515,7 @@ export const config = { "experimental/multimodal_embeddings/googlevertexai", "experimental/hubs/makersuite/googlemakersuitehub", // chains - "chains/graph_qa/cypher" + "chains/graph_qa/cypher", ], packageSuffix: "community", tsConfigPath: resolve("./tsconfig.json"), From 7fb25bd1d04a09b1e899f0b5eaf1c95430863040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mussa?= Date: Fri, 21 Jun 2024 21:22:44 -0400 Subject: [PATCH 03/10] fix: remove dangling comma from langchain.config.js --- libs/langchain-community/langchain.config.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/libs/langchain-community/langchain.config.js b/libs/langchain-community/langchain.config.js index 99cd8215769a..59f1f30a76e0 100644 --- a/libs/langchain-community/langchain.config.js +++ b/libs/langchain-community/langchain.config.js @@ -27,7 +27,7 @@ export const config = { "pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js", "web-auth-library/google", "notion-to-md/build/utils/notion.js", - "@getzep/zep-cloud/api", + "@getzep/zep-cloud/api" ], entrypoints: { load: "load/index", @@ -188,8 +188,7 @@ export const config = { // callbacks "callbacks/handlers/llmonitor": "callbacks/handlers/llmonitor", "callbacks/handlers/lunary": "callbacks/handlers/lunary", - "callbacks/handlers/upstash_ratelimit": - "callbacks/handlers/upstash_ratelimit", + "callbacks/handlers/upstash_ratelimit": "callbacks/handlers/upstash_ratelimit", // retrievers "retrievers/amazon_kendra": "retrievers/amazon_kendra", "retrievers/amazon_knowledge_base": "retrievers/amazon_knowledge_base", @@ -314,12 +313,11 @@ export const config = { "experimental/multimodal_embeddings/googlevertexai", "experimental/hubs/makersuite/googlemakersuitehub": "experimental/hubs/makersuite/googlemakersuitehub", - "experimental/chat_models/ollama_functions": - "experimental/chat_models/ollama_functions", + "experimental/chat_models/ollama_functions": "experimental/chat_models/ollama_functions", // chains "chains/graph_qa/cypher": "chains/graph_qa/cypher", // tracers - "tracers/datadog": "tracers/datadog", + "tracers/datadog": "tracers/datadog" }, requiresOptionalDependency: [ "tools/aws_sfn", @@ -515,7 +513,7 @@ export const config = { "experimental/multimodal_embeddings/googlevertexai", "experimental/hubs/makersuite/googlemakersuitehub", // chains - "chains/graph_qa/cypher", + "chains/graph_qa/cypher" ], packageSuffix: "community", tsConfigPath: resolve("./tsconfig.json"), From b1fcbb84d7e7c2d8825b2f296ea04be655a6d496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mussa?= Date: Fri, 21 Jun 2024 21:25:54 -0400 Subject: [PATCH 04/10] fix: wrong import on datadog tracer example --- examples/src/tracers/datadog.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/src/tracers/datadog.ts b/examples/src/tracers/datadog.ts index d304c819cf2d..02cc10bd1008 100644 --- a/examples/src/tracers/datadog.ts +++ b/examples/src/tracers/datadog.ts @@ -1,5 +1,5 @@ import { OpenAI } from "@langchain/openai"; -import { DatadogLLMObsTracer } from "@/libs/llm-monitoring"; +import { DatadogLLMObsTracer } from "@langchain/community/tracers/datadog"; /** * This example demonstrates how to use the DatadogLLMObsTracer with the OpenAI model. From c5df6a22dc76f3dca13e76ff63e87b278f4fc910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mussa?= Date: Fri, 21 Jun 2024 21:41:29 -0400 Subject: [PATCH 05/10] chore: add autogenerated artifacts --- libs/langchain-community/.gitignore | 4 ++++ libs/langchain-community/package.json | 15 ++++++++++++++- libs/langchain-community/src/load/import_map.ts | 1 + libs/langchain-community/src/tracers/datadog.ts | 2 +- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/libs/langchain-community/.gitignore b/libs/langchain-community/.gitignore index 585ec0c0f9df..c40057f82853 100644 --- a/libs/langchain-community/.gitignore +++ b/libs/langchain-community/.gitignore @@ -1026,6 +1026,10 @@ chains/graph_qa/cypher.cjs chains/graph_qa/cypher.js chains/graph_qa/cypher.d.ts chains/graph_qa/cypher.d.cts +tracers/datadog.cjs +tracers/datadog.js +tracers/datadog.d.ts +tracers/datadog.d.cts node_modules dist .yarn diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index 0d1f22090644..0018b751d9b7 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -3013,6 +3013,15 @@ "import": "./chains/graph_qa/cypher.js", "require": "./chains/graph_qa/cypher.cjs" }, + "./tracers/datadog": { + "types": { + "import": "./tracers/datadog.d.ts", + "require": "./tracers/datadog.d.cts", + "default": "./tracers/datadog.d.ts" + }, + "import": "./tracers/datadog.js", + "require": "./tracers/datadog.cjs" + }, "./package.json": "./package.json" }, "files": [ @@ -4044,6 +4053,10 @@ "chains/graph_qa/cypher.cjs", "chains/graph_qa/cypher.js", "chains/graph_qa/cypher.d.ts", - "chains/graph_qa/cypher.d.cts" + "chains/graph_qa/cypher.d.cts", + "tracers/datadog.cjs", + "tracers/datadog.js", + "tracers/datadog.d.ts", + "tracers/datadog.d.cts" ] } diff --git a/libs/langchain-community/src/load/import_map.ts b/libs/langchain-community/src/load/import_map.ts index b8b0a18e0564..f04e87f535ce 100644 --- a/libs/langchain-community/src/load/import_map.ts +++ b/libs/langchain-community/src/load/import_map.ts @@ -76,3 +76,4 @@ export * as document_loaders__web__sort_xyz_blockchain from "../document_loaders export * as utils__event_source_parse from "../utils/event_source_parse.js"; export * as experimental__graph_transformers__llm from "../experimental/graph_transformers/llm.js"; export * as experimental__chat_models__ollama_functions from "../experimental/chat_models/ollama_functions.js"; +export * as tracers__datadog from "../tracers/datadog.js"; diff --git a/libs/langchain-community/src/tracers/datadog.ts b/libs/langchain-community/src/tracers/datadog.ts index e393ce56a9cf..c8e07b5ab4be 100644 --- a/libs/langchain-community/src/tracers/datadog.ts +++ b/libs/langchain-community/src/tracers/datadog.ts @@ -171,7 +171,7 @@ export class DatadogLLMObsTracer }); } - return spans.filter((span) => span !== null); + return spans.flatMap(span => (span ? [span] : [])); } protected formatRequestBody( From 5300306c23ee76d553722bccffba330d013f629a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mussa?= Date: Sat, 22 Jun 2024 09:01:01 -0400 Subject: [PATCH 06/10] fix: typo on UsageExmaple import --- docs/core_docs/docs/integrations/tracers/datadog.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/core_docs/docs/integrations/tracers/datadog.mdx b/docs/core_docs/docs/integrations/tracers/datadog.mdx index 40e84f368e2c..7dcfffa6c044 100644 --- a/docs/core_docs/docs/integrations/tracers/datadog.mdx +++ b/docs/core_docs/docs/integrations/tracers/datadog.mdx @@ -30,6 +30,6 @@ npm install @langchain/community ## Usage -import UsageExample from "@examples/traces/datadog.ts"; +import UsageExample from "@examples/tracers/datadog.ts"; {UsageExample} From d5c8aa4da6f3e72772186f3d3566fe187e2bad29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mussa?= Date: Sat, 22 Jun 2024 09:40:00 -0400 Subject: [PATCH 07/10] chore: move datadog llmobs from tracers to callbacks for consistency --- .../datadog_tracer.mdx} | 4 +-- .../src/{tracers => callbacks}/datadog.ts | 2 +- libs/langchain-community/.gitignore | 8 +++--- libs/langchain-community/langchain.config.js | 5 ++-- libs/langchain-community/package.json | 28 +++++++++---------- .../tests/datadog.test.ts | 2 +- .../src/{ => callbacks}/tracers/datadog.ts | 0 .../src/load/import_map.ts | 2 +- 8 files changed, 25 insertions(+), 26 deletions(-) rename docs/core_docs/docs/integrations/{tracers/datadog.mdx => callbacks/datadog_tracer.mdx} (90%) rename examples/src/{tracers => callbacks}/datadog.ts (89%) rename libs/langchain-community/src/{tracers => callbacks}/tests/datadog.test.ts (99%) rename libs/langchain-community/src/{ => callbacks}/tracers/datadog.ts (100%) diff --git a/docs/core_docs/docs/integrations/tracers/datadog.mdx b/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx similarity index 90% rename from docs/core_docs/docs/integrations/tracers/datadog.mdx rename to docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx index 7dcfffa6c044..321cbc7af886 100644 --- a/docs/core_docs/docs/integrations/tracers/datadog.mdx +++ b/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx @@ -4,7 +4,7 @@ sidebar_class_name: node-only import CodeBlock from "@theme/CodeBlock"; -# Datadog LLM Observability +# Datadog LLM Observability Tracer Callback :::warning LLM Observability is in public beta, and its API is subject to change. @@ -30,6 +30,6 @@ npm install @langchain/community ## Usage -import UsageExample from "@examples/tracers/datadog.ts"; +import UsageExample from "@examples/callbacks/datadog.ts"; {UsageExample} diff --git a/examples/src/tracers/datadog.ts b/examples/src/callbacks/datadog.ts similarity index 89% rename from examples/src/tracers/datadog.ts rename to examples/src/callbacks/datadog.ts index 02cc10bd1008..7c2a65a2377c 100644 --- a/examples/src/tracers/datadog.ts +++ b/examples/src/callbacks/datadog.ts @@ -1,5 +1,5 @@ import { OpenAI } from "@langchain/openai"; -import { DatadogLLMObsTracer } from "@langchain/community/tracers/datadog"; +import { DatadogLLMObsTracer } from "@langchain/community/callbacks/tracers/datadog"; /** * This example demonstrates how to use the DatadogLLMObsTracer with the OpenAI model. diff --git a/libs/langchain-community/.gitignore b/libs/langchain-community/.gitignore index c40057f82853..27711bc1d7f7 100644 --- a/libs/langchain-community/.gitignore +++ b/libs/langchain-community/.gitignore @@ -606,6 +606,10 @@ callbacks/handlers/upstash_ratelimit.cjs callbacks/handlers/upstash_ratelimit.js callbacks/handlers/upstash_ratelimit.d.ts callbacks/handlers/upstash_ratelimit.d.cts +callbacks/tracers/datadog.cjs +callbacks/tracers/datadog.js +callbacks/tracers/datadog.d.ts +callbacks/tracers/datadog.d.cts retrievers/amazon_kendra.cjs retrievers/amazon_kendra.js retrievers/amazon_kendra.d.ts @@ -1026,10 +1030,6 @@ chains/graph_qa/cypher.cjs chains/graph_qa/cypher.js chains/graph_qa/cypher.d.ts chains/graph_qa/cypher.d.cts -tracers/datadog.cjs -tracers/datadog.js -tracers/datadog.d.ts -tracers/datadog.d.cts node_modules dist .yarn diff --git a/libs/langchain-community/langchain.config.js b/libs/langchain-community/langchain.config.js index 59f1f30a76e0..90b3d85d20f5 100644 --- a/libs/langchain-community/langchain.config.js +++ b/libs/langchain-community/langchain.config.js @@ -189,6 +189,7 @@ export const config = { "callbacks/handlers/llmonitor": "callbacks/handlers/llmonitor", "callbacks/handlers/lunary": "callbacks/handlers/lunary", "callbacks/handlers/upstash_ratelimit": "callbacks/handlers/upstash_ratelimit", + "callbacks/tracers/datadog": "callbacks/tracers/datadog", // retrievers "retrievers/amazon_kendra": "retrievers/amazon_kendra", "retrievers/amazon_knowledge_base": "retrievers/amazon_knowledge_base", @@ -315,9 +316,7 @@ export const config = { "experimental/hubs/makersuite/googlemakersuitehub", "experimental/chat_models/ollama_functions": "experimental/chat_models/ollama_functions", // chains - "chains/graph_qa/cypher": "chains/graph_qa/cypher", - // tracers - "tracers/datadog": "tracers/datadog" + "chains/graph_qa/cypher": "chains/graph_qa/cypher" }, requiresOptionalDependency: [ "tools/aws_sfn", diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index 0018b751d9b7..027fad37831d 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -2068,6 +2068,15 @@ "import": "./callbacks/handlers/upstash_ratelimit.js", "require": "./callbacks/handlers/upstash_ratelimit.cjs" }, + "./callbacks/tracers/datadog": { + "types": { + "import": "./callbacks/tracers/datadog.d.ts", + "require": "./callbacks/tracers/datadog.d.cts", + "default": "./callbacks/tracers/datadog.d.ts" + }, + "import": "./callbacks/tracers/datadog.js", + "require": "./callbacks/tracers/datadog.cjs" + }, "./retrievers/amazon_kendra": { "types": { "import": "./retrievers/amazon_kendra.d.ts", @@ -3013,15 +3022,6 @@ "import": "./chains/graph_qa/cypher.js", "require": "./chains/graph_qa/cypher.cjs" }, - "./tracers/datadog": { - "types": { - "import": "./tracers/datadog.d.ts", - "require": "./tracers/datadog.d.cts", - "default": "./tracers/datadog.d.ts" - }, - "import": "./tracers/datadog.js", - "require": "./tracers/datadog.cjs" - }, "./package.json": "./package.json" }, "files": [ @@ -3634,6 +3634,10 @@ "callbacks/handlers/upstash_ratelimit.js", "callbacks/handlers/upstash_ratelimit.d.ts", "callbacks/handlers/upstash_ratelimit.d.cts", + "callbacks/tracers/datadog.cjs", + "callbacks/tracers/datadog.js", + "callbacks/tracers/datadog.d.ts", + "callbacks/tracers/datadog.d.cts", "retrievers/amazon_kendra.cjs", "retrievers/amazon_kendra.js", "retrievers/amazon_kendra.d.ts", @@ -4053,10 +4057,6 @@ "chains/graph_qa/cypher.cjs", "chains/graph_qa/cypher.js", "chains/graph_qa/cypher.d.ts", - "chains/graph_qa/cypher.d.cts", - "tracers/datadog.cjs", - "tracers/datadog.js", - "tracers/datadog.d.ts", - "tracers/datadog.d.cts" + "chains/graph_qa/cypher.d.cts" ] } diff --git a/libs/langchain-community/src/tracers/tests/datadog.test.ts b/libs/langchain-community/src/callbacks/tests/datadog.test.ts similarity index 99% rename from libs/langchain-community/src/tracers/tests/datadog.test.ts rename to libs/langchain-community/src/callbacks/tests/datadog.test.ts index 3a9190006fdc..8760fada622b 100644 --- a/libs/langchain-community/src/tracers/tests/datadog.test.ts +++ b/libs/langchain-community/src/callbacks/tests/datadog.test.ts @@ -7,7 +7,7 @@ import { DatadogLLMObsRequestBody, DatadogLLMObsSpan, DatadogLLMObsTracer, -} from "../datadog.js"; +} from "../tracers/datadog.js"; const _DATE = 1620000000000; const _END_DATE = _DATE + 1000; diff --git a/libs/langchain-community/src/tracers/datadog.ts b/libs/langchain-community/src/callbacks/tracers/datadog.ts similarity index 100% rename from libs/langchain-community/src/tracers/datadog.ts rename to libs/langchain-community/src/callbacks/tracers/datadog.ts diff --git a/libs/langchain-community/src/load/import_map.ts b/libs/langchain-community/src/load/import_map.ts index f04e87f535ce..ec4068f90db8 100644 --- a/libs/langchain-community/src/load/import_map.ts +++ b/libs/langchain-community/src/load/import_map.ts @@ -54,6 +54,7 @@ export * as chat_models__moonshot from "../chat_models/moonshot.js"; export * as chat_models__ollama from "../chat_models/ollama.js"; export * as chat_models__togetherai from "../chat_models/togetherai.js"; export * as chat_models__yandex from "../chat_models/yandex.js"; +export * as callbacks__tracers__datadog from "../callbacks/tracers/datadog.js"; export * as retrievers__chaindesk from "../retrievers/chaindesk.js"; export * as retrievers__databerry from "../retrievers/databerry.js"; export * as retrievers__remote from "../retrievers/remote/index.js"; @@ -76,4 +77,3 @@ export * as document_loaders__web__sort_xyz_blockchain from "../document_loaders export * as utils__event_source_parse from "../utils/event_source_parse.js"; export * as experimental__graph_transformers__llm from "../experimental/graph_transformers/llm.js"; export * as experimental__chat_models__ollama_functions from "../experimental/chat_models/ollama_functions.js"; -export * as tracers__datadog from "../tracers/datadog.js"; From ebb6239e8c794fe6bcc80fd8c71f9abaa0d82a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mussa?= Date: Sun, 23 Jun 2024 14:37:13 -0400 Subject: [PATCH 08/10] chore: handle propagating tags to spans --- .../src/callbacks/tests/datadog.test.ts | 13 ++++++++----- .../src/callbacks/tracers/datadog.ts | 2 ++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/libs/langchain-community/src/callbacks/tests/datadog.test.ts b/libs/langchain-community/src/callbacks/tests/datadog.test.ts index 8760fada622b..845691f0c3ee 100644 --- a/libs/langchain-community/src/callbacks/tests/datadog.test.ts +++ b/libs/langchain-community/src/callbacks/tests/datadog.test.ts @@ -220,6 +220,7 @@ test("Test workflow span", async () => { outputs: { output: "test", }, + tags: ["seq:test"] }; const compareSpan: DatadogLLMObsSpan = { @@ -227,12 +228,13 @@ test("Test workflow span", async () => { meta: { kind: "workflow", input: { - value: JSON.stringify({ question: "test" }), + value: JSON.stringify(run.inputs), }, output: { - value: JSON.stringify("test"), + value: JSON.stringify(run.outputs?.output), }, }, + tags: run.tags }; const requestBody: DatadogLLMObsRequestBody = { @@ -256,6 +258,7 @@ test("Test workflow span", async () => { }); const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; expect(parsedBody).toMatchObject( requestBody as unknown as Record @@ -281,10 +284,10 @@ test("Test tool span", async () => { meta: { kind: "tool", input: { - value: JSON.stringify({ input: { query: "test" } }), + value: JSON.stringify(run.inputs), }, output: { - value: JSON.stringify("test"), + value: JSON.stringify(run.outputs?.output), }, }, }; @@ -340,7 +343,7 @@ test("Test retrieval span", async () => { meta: { kind: "retrieval", input: { - value: JSON.stringify({ input: { query: "test" } }), + value: JSON.stringify(run.inputs), }, output: { documents: [ diff --git a/libs/langchain-community/src/callbacks/tracers/datadog.ts b/libs/langchain-community/src/callbacks/tracers/datadog.ts index c8e07b5ab4be..09597c30a2d9 100644 --- a/libs/langchain-community/src/callbacks/tracers/datadog.ts +++ b/libs/langchain-community/src/callbacks/tracers/datadog.ts @@ -38,6 +38,7 @@ export interface DatadogLLMObsSpan { duration: number; error: number; status: string; + tags?: string[]; meta: { kind: DatadogLLMObsSpanKind; model_name?: string; @@ -365,6 +366,7 @@ export class DatadogLLMObsTracer name: spanName, error: spanError, status: spanStatus, + tags: [...(run.tags?.length ? run.tags : [])], meta, start_ns: startTimeNs, duration: durationNs, From b04a3278b9fecd35808db67994fa4a617621af4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mussa?= Date: Sun, 23 Jun 2024 15:30:17 -0400 Subject: [PATCH 09/10] chore: update session_id based on recent DD API Schema update --- .../src/callbacks/tests/datadog.test.ts | 10 +++++----- .../src/callbacks/tracers/datadog.ts | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libs/langchain-community/src/callbacks/tests/datadog.test.ts b/libs/langchain-community/src/callbacks/tests/datadog.test.ts index 845691f0c3ee..4f559f680319 100644 --- a/libs/langchain-community/src/callbacks/tests/datadog.test.ts +++ b/libs/langchain-community/src/callbacks/tests/datadog.test.ts @@ -124,11 +124,11 @@ test("Test llm span with message input", async () => { const requestBody: DatadogLLMObsRequestBody = { data: { type: "span", - session_id: "test", attributes: { ml_app: "test", tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], spans: [compareSpan], + session_id: "test", }, }, }; @@ -184,11 +184,11 @@ test("Test llm span with prompt input", async () => { const requestBody: DatadogLLMObsRequestBody = { data: { type: "span", - session_id: "test", attributes: { ml_app: "test", tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], spans: [compareSpan], + session_id: "test", }, }, }; @@ -240,11 +240,11 @@ test("Test workflow span", async () => { const requestBody: DatadogLLMObsRequestBody = { data: { type: "span", - session_id: "test", attributes: { ml_app: "test", tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], spans: [compareSpan], + session_id: "test", }, }, }; @@ -295,11 +295,11 @@ test("Test tool span", async () => { const requestBody: DatadogLLMObsRequestBody = { data: { type: "span", - session_id: "test", attributes: { ml_app: "test", tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], spans: [compareSpan], + session_id: "test", }, }, }; @@ -361,11 +361,11 @@ test("Test retrieval span", async () => { const requestBody: DatadogLLMObsRequestBody = { data: { type: "span", - session_id: "test", attributes: { ml_app: "test", tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], spans: [compareSpan], + session_id: "test", }, }, }; diff --git a/libs/langchain-community/src/callbacks/tracers/datadog.ts b/libs/langchain-community/src/callbacks/tracers/datadog.ts index 09597c30a2d9..51c938c1341e 100644 --- a/libs/langchain-community/src/callbacks/tracers/datadog.ts +++ b/libs/langchain-community/src/callbacks/tracers/datadog.ts @@ -53,11 +53,11 @@ export interface DatadogLLMObsSpan { export interface DatadogLLMObsRequestBody { data: { type: "span"; - session_id?: string; attributes: { ml_app: string; tags: string[]; spans: DatadogLLMObsSpan[]; + session_id?: string; }; }; } @@ -181,13 +181,13 @@ export class DatadogLLMObsTracer return { data: { type: "span", - session_id: this.sessionId, attributes: { ml_app: this.mlApp, tags: Object.entries(this.tags) .filter(([, value]) => value) .map(([key, value]) => `${key}:${value}`), spans, + session_id: this.sessionId, }, }, }; From abb06f6599f6fc16abe62611a6fbcf6e983358b4 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Tue, 25 Jun 2024 09:33:22 -0700 Subject: [PATCH 10/10] Update imports, move to experimental --- docs/core_docs/.gitignore | 6 +++++ .../integrations/callbacks/datadog_tracer.mdx | 10 +++---- docs/core_docs/src/css/custom.css | 6 +++-- examples/src/callbacks/datadog.ts | 2 +- libs/langchain-community/.gitignore | 8 +++--- libs/langchain-community/langchain.config.js | 2 +- libs/langchain-community/package.json | 26 +++++++++---------- .../callbacks/handlers}/datadog.ts | 5 ++-- .../callbacks/handlers}/tests/datadog.test.ts | 6 ++--- .../src/load/import_map.ts | 2 +- 10 files changed, 38 insertions(+), 35 deletions(-) rename libs/langchain-community/src/{callbacks/tracers => experimental/callbacks/handlers}/datadog.ts (98%) rename libs/langchain-community/src/{callbacks => experimental/callbacks/handlers}/tests/datadog.test.ts (99%) diff --git a/docs/core_docs/.gitignore b/docs/core_docs/.gitignore index 6f2cc4c38972..ed244bba57b2 100644 --- a/docs/core_docs/.gitignore +++ b/docs/core_docs/.gitignore @@ -53,6 +53,8 @@ docs/tutorials/classification.md docs/tutorials/classification.mdx docs/tutorials/chatbot.md docs/tutorials/chatbot.mdx +docs/how_to/trim_messages.md +docs/how_to/trim_messages.mdx docs/how_to/tools_prompting.md docs/how_to/tools_prompting.mdx docs/how_to/tools_builtin.md @@ -117,6 +119,8 @@ docs/how_to/multimodal_inputs.md docs/how_to/multimodal_inputs.mdx docs/how_to/migrate_agent.md docs/how_to/migrate_agent.mdx +docs/how_to/merge_message_runs.md +docs/how_to/merge_message_runs.mdx docs/how_to/logprobs.md docs/how_to/logprobs.mdx docs/how_to/lcel_cheatsheet.md @@ -131,6 +135,8 @@ docs/how_to/graph_constructing.md docs/how_to/graph_constructing.mdx docs/how_to/functions.md docs/how_to/functions.mdx +docs/how_to/filter_messages.md +docs/how_to/filter_messages.mdx docs/how_to/few_shot_examples_chat.md docs/how_to/few_shot_examples_chat.mdx docs/how_to/few_shot_examples.md diff --git a/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx b/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx index 321cbc7af886..85732fd57428 100644 --- a/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx +++ b/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx @@ -1,10 +1,10 @@ --- -sidebar_class_name: node-only +sidebar_class_name: beta --- import CodeBlock from "@theme/CodeBlock"; -# Datadog LLM Observability Tracer Callback +# Datadog LLM Observability :::warning LLM Observability is in public beta, and its API is subject to change. @@ -12,11 +12,7 @@ LLM Observability is in public beta, and its API is subject to change. With [Datadog LLM Observability](https://docs.datadoghq.com/llm_observability/), you can monitor, troubleshoot, and evaluate your LLM-powered applications, such as chatbots. You can investigate the root cause of issues, monitor operational performance, and evaluate the quality, privacy, and safety of your LLM applications. -This is a community implementation, and it is not officially supported by Datadog. It is based on the [Datadog LLM Observability API](https://docs.datadoghq.com/llm_observability/api). - -:::tip Compatibility -Only available on Node.js. -::: +This is an experimental community implementation, and it is not officially supported by Datadog. It is based on the [Datadog LLM Observability API](https://docs.datadoghq.com/llm_observability/api). ## Setup diff --git a/docs/core_docs/src/css/custom.css b/docs/core_docs/src/css/custom.css index 62e710879169..e97a71b9d4a2 100644 --- a/docs/core_docs/src/css/custom.css +++ b/docs/core_docs/src/css/custom.css @@ -101,7 +101,8 @@ nav, h1, h2, h3, h4 { /* Override `beta` color */ .beta::after { content: "Beta"; - color: #58006e; + color: #fff; + background: #58006e; border: 1px solid #58006e; } @@ -114,7 +115,8 @@ nav, h1, h2, h3, h4 { /* Override `web-only` color */ .web-only::after { content: "Web-only"; - color: #0a0072; + color: #fff; + background: #0a0072; border: 1px solid #0a0072; } diff --git a/examples/src/callbacks/datadog.ts b/examples/src/callbacks/datadog.ts index 7c2a65a2377c..cb2e00f7dd47 100644 --- a/examples/src/callbacks/datadog.ts +++ b/examples/src/callbacks/datadog.ts @@ -1,5 +1,5 @@ import { OpenAI } from "@langchain/openai"; -import { DatadogLLMObsTracer } from "@langchain/community/callbacks/tracers/datadog"; +import { DatadogLLMObsTracer } from "@langchain/community/experimental/callbacks/handlers/datadog"; /** * This example demonstrates how to use the DatadogLLMObsTracer with the OpenAI model. diff --git a/libs/langchain-community/.gitignore b/libs/langchain-community/.gitignore index 14b1add630a1..4246a650f6df 100644 --- a/libs/langchain-community/.gitignore +++ b/libs/langchain-community/.gitignore @@ -610,10 +610,6 @@ callbacks/handlers/upstash_ratelimit.cjs callbacks/handlers/upstash_ratelimit.js callbacks/handlers/upstash_ratelimit.d.ts callbacks/handlers/upstash_ratelimit.d.cts -callbacks/tracers/datadog.cjs -callbacks/tracers/datadog.js -callbacks/tracers/datadog.d.ts -callbacks/tracers/datadog.d.cts retrievers/amazon_kendra.cjs retrievers/amazon_kendra.js retrievers/amazon_kendra.d.ts @@ -1014,6 +1010,10 @@ utils/cassandra.cjs utils/cassandra.js utils/cassandra.d.ts utils/cassandra.d.cts +experimental/callbacks/handlers/datadog.cjs +experimental/callbacks/handlers/datadog.js +experimental/callbacks/handlers/datadog.d.ts +experimental/callbacks/handlers/datadog.d.cts experimental/graph_transformers/llm.cjs experimental/graph_transformers/llm.js experimental/graph_transformers/llm.d.ts diff --git a/libs/langchain-community/langchain.config.js b/libs/langchain-community/langchain.config.js index 8f850784c7d8..7012d8e06c5a 100644 --- a/libs/langchain-community/langchain.config.js +++ b/libs/langchain-community/langchain.config.js @@ -190,7 +190,6 @@ export const config = { "callbacks/handlers/llmonitor": "callbacks/handlers/llmonitor", "callbacks/handlers/lunary": "callbacks/handlers/lunary", "callbacks/handlers/upstash_ratelimit": "callbacks/handlers/upstash_ratelimit", - "callbacks/tracers/datadog": "callbacks/tracers/datadog", // retrievers "retrievers/amazon_kendra": "retrievers/amazon_kendra", "retrievers/amazon_knowledge_base": "retrievers/amazon_knowledge_base", @@ -309,6 +308,7 @@ export const config = { "utils/event_source_parse": "utils/event_source_parse", "utils/cassandra": "utils/cassandra", // experimental + "experimental/callbacks/handlers/datadog": "experimental/callbacks/handlers/datadog", "experimental/graph_transformers/llm": "experimental/graph_transformers/llm", "experimental/multimodal_embeddings/googlevertexai": diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index b9938f0d774d..8774a64deae4 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -2077,15 +2077,6 @@ "import": "./callbacks/handlers/upstash_ratelimit.js", "require": "./callbacks/handlers/upstash_ratelimit.cjs" }, - "./callbacks/tracers/datadog": { - "types": { - "import": "./callbacks/tracers/datadog.d.ts", - "require": "./callbacks/tracers/datadog.d.cts", - "default": "./callbacks/tracers/datadog.d.ts" - }, - "import": "./callbacks/tracers/datadog.js", - "require": "./callbacks/tracers/datadog.cjs" - }, "./retrievers/amazon_kendra": { "types": { "import": "./retrievers/amazon_kendra.d.ts", @@ -2986,6 +2977,15 @@ "import": "./utils/cassandra.js", "require": "./utils/cassandra.cjs" }, + "./experimental/callbacks/handlers/datadog": { + "types": { + "import": "./experimental/callbacks/handlers/datadog.d.ts", + "require": "./experimental/callbacks/handlers/datadog.d.cts", + "default": "./experimental/callbacks/handlers/datadog.d.ts" + }, + "import": "./experimental/callbacks/handlers/datadog.js", + "require": "./experimental/callbacks/handlers/datadog.cjs" + }, "./experimental/graph_transformers/llm": { "types": { "import": "./experimental/graph_transformers/llm.d.ts", @@ -3647,10 +3647,6 @@ "callbacks/handlers/upstash_ratelimit.js", "callbacks/handlers/upstash_ratelimit.d.ts", "callbacks/handlers/upstash_ratelimit.d.cts", - "callbacks/tracers/datadog.cjs", - "callbacks/tracers/datadog.js", - "callbacks/tracers/datadog.d.ts", - "callbacks/tracers/datadog.d.cts", "retrievers/amazon_kendra.cjs", "retrievers/amazon_kendra.js", "retrievers/amazon_kendra.d.ts", @@ -4051,6 +4047,10 @@ "utils/cassandra.js", "utils/cassandra.d.ts", "utils/cassandra.d.cts", + "experimental/callbacks/handlers/datadog.cjs", + "experimental/callbacks/handlers/datadog.js", + "experimental/callbacks/handlers/datadog.d.ts", + "experimental/callbacks/handlers/datadog.d.cts", "experimental/graph_transformers/llm.cjs", "experimental/graph_transformers/llm.js", "experimental/graph_transformers/llm.d.ts", diff --git a/libs/langchain-community/src/callbacks/tracers/datadog.ts b/libs/langchain-community/src/experimental/callbacks/handlers/datadog.ts similarity index 98% rename from libs/langchain-community/src/callbacks/tracers/datadog.ts rename to libs/langchain-community/src/experimental/callbacks/handlers/datadog.ts index 51c938c1341e..57a2e16daf3b 100644 --- a/libs/langchain-community/src/callbacks/tracers/datadog.ts +++ b/libs/langchain-community/src/experimental/callbacks/handlers/datadog.ts @@ -1,8 +1,7 @@ import { BaseCallbackHandlerInput } from "@langchain/core/callbacks/base"; import { BaseTracer, Run } from "@langchain/core/tracers/base"; import { getEnvironmentVariable } from "@langchain/core/utils/env"; -import { Document } from "langchain/document"; - +import { Document } from "@langchain/core/documents"; import { BaseMessage } from "@langchain/core/messages"; import { ChatGeneration } from "@langchain/core/outputs"; import { KVMap } from "langsmith/schemas"; @@ -172,7 +171,7 @@ export class DatadogLLMObsTracer }); } - return spans.flatMap(span => (span ? [span] : [])); + return spans.flatMap((span) => (span ? [span] : [])); } protected formatRequestBody( diff --git a/libs/langchain-community/src/callbacks/tests/datadog.test.ts b/libs/langchain-community/src/experimental/callbacks/handlers/tests/datadog.test.ts similarity index 99% rename from libs/langchain-community/src/callbacks/tests/datadog.test.ts rename to libs/langchain-community/src/experimental/callbacks/handlers/tests/datadog.test.ts index 4f559f680319..0f5a015a0e81 100644 --- a/libs/langchain-community/src/callbacks/tests/datadog.test.ts +++ b/libs/langchain-community/src/experimental/callbacks/handlers/tests/datadog.test.ts @@ -7,7 +7,7 @@ import { DatadogLLMObsRequestBody, DatadogLLMObsSpan, DatadogLLMObsTracer, -} from "../tracers/datadog.js"; +} from "../datadog.js"; const _DATE = 1620000000000; const _END_DATE = _DATE + 1000; @@ -220,7 +220,7 @@ test("Test workflow span", async () => { outputs: { output: "test", }, - tags: ["seq:test"] + tags: ["seq:test"], }; const compareSpan: DatadogLLMObsSpan = { @@ -234,7 +234,7 @@ test("Test workflow span", async () => { value: JSON.stringify(run.outputs?.output), }, }, - tags: run.tags + tags: run.tags, }; const requestBody: DatadogLLMObsRequestBody = { diff --git a/libs/langchain-community/src/load/import_map.ts b/libs/langchain-community/src/load/import_map.ts index ec4068f90db8..d06053cbde26 100644 --- a/libs/langchain-community/src/load/import_map.ts +++ b/libs/langchain-community/src/load/import_map.ts @@ -54,7 +54,6 @@ export * as chat_models__moonshot from "../chat_models/moonshot.js"; export * as chat_models__ollama from "../chat_models/ollama.js"; export * as chat_models__togetherai from "../chat_models/togetherai.js"; export * as chat_models__yandex from "../chat_models/yandex.js"; -export * as callbacks__tracers__datadog from "../callbacks/tracers/datadog.js"; export * as retrievers__chaindesk from "../retrievers/chaindesk.js"; export * as retrievers__databerry from "../retrievers/databerry.js"; export * as retrievers__remote from "../retrievers/remote/index.js"; @@ -75,5 +74,6 @@ export * as document_loaders__web__searchapi from "../document_loaders/web/searc export * as document_loaders__web__serpapi from "../document_loaders/web/serpapi.js"; export * as document_loaders__web__sort_xyz_blockchain from "../document_loaders/web/sort_xyz_blockchain.js"; export * as utils__event_source_parse from "../utils/event_source_parse.js"; +export * as experimental__callbacks__handlers__datadog from "../experimental/callbacks/handlers/datadog.js"; export * as experimental__graph_transformers__llm from "../experimental/graph_transformers/llm.js"; export * as experimental__chat_models__ollama_functions from "../experimental/chat_models/ollama_functions.js";