diff --git a/docs/core_docs/.gitignore b/docs/core_docs/.gitignore index 6f2cc4c38972..ed244bba57b2 100644 --- a/docs/core_docs/.gitignore +++ b/docs/core_docs/.gitignore @@ -53,6 +53,8 @@ docs/tutorials/classification.md docs/tutorials/classification.mdx docs/tutorials/chatbot.md docs/tutorials/chatbot.mdx +docs/how_to/trim_messages.md +docs/how_to/trim_messages.mdx docs/how_to/tools_prompting.md docs/how_to/tools_prompting.mdx docs/how_to/tools_builtin.md @@ -117,6 +119,8 @@ docs/how_to/multimodal_inputs.md docs/how_to/multimodal_inputs.mdx docs/how_to/migrate_agent.md docs/how_to/migrate_agent.mdx +docs/how_to/merge_message_runs.md +docs/how_to/merge_message_runs.mdx docs/how_to/logprobs.md docs/how_to/logprobs.mdx docs/how_to/lcel_cheatsheet.md @@ -131,6 +135,8 @@ docs/how_to/graph_constructing.md docs/how_to/graph_constructing.mdx docs/how_to/functions.md docs/how_to/functions.mdx +docs/how_to/filter_messages.md +docs/how_to/filter_messages.mdx docs/how_to/few_shot_examples_chat.md docs/how_to/few_shot_examples_chat.mdx docs/how_to/few_shot_examples.md diff --git a/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx b/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx new file mode 100644 index 000000000000..85732fd57428 --- /dev/null +++ b/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx @@ -0,0 +1,31 @@ +--- +sidebar_class_name: beta +--- + +import CodeBlock from "@theme/CodeBlock"; + +# Datadog LLM Observability + +:::warning +LLM Observability is in public beta, and its API is subject to change. +::: + +With [Datadog LLM Observability](https://docs.datadoghq.com/llm_observability/), you can monitor, troubleshoot, and evaluate your LLM-powered applications, such as chatbots. You can investigate the root cause of issues, monitor operational performance, and evaluate the quality, privacy, and safety of your LLM applications. + +This is an experimental community implementation, and it is not officially supported by Datadog. It is based on the [Datadog LLM Observability API](https://docs.datadoghq.com/llm_observability/api). + +## Setup + +import IntegrationInstallTooltip from "@mdx_components/integration_install_tooltip.mdx"; + + + +```bash npm2yarn +npm install @langchain/community +``` + +## Usage + +import UsageExample from "@examples/callbacks/datadog.ts"; + +{UsageExample} diff --git a/docs/core_docs/src/css/custom.css b/docs/core_docs/src/css/custom.css index 62e710879169..e97a71b9d4a2 100644 --- a/docs/core_docs/src/css/custom.css +++ b/docs/core_docs/src/css/custom.css @@ -101,7 +101,8 @@ nav, h1, h2, h3, h4 { /* Override `beta` color */ .beta::after { content: "Beta"; - color: #58006e; + color: #fff; + background: #58006e; border: 1px solid #58006e; } @@ -114,7 +115,8 @@ nav, h1, h2, h3, h4 { /* Override `web-only` color */ .web-only::after { content: "Web-only"; - color: #0a0072; + color: #fff; + background: #0a0072; border: 1px solid #0a0072; } diff --git a/examples/src/callbacks/datadog.ts b/examples/src/callbacks/datadog.ts new file mode 100644 index 000000000000..cb2e00f7dd47 --- /dev/null +++ b/examples/src/callbacks/datadog.ts @@ -0,0 +1,30 @@ +import { OpenAI } from "@langchain/openai"; +import { DatadogLLMObsTracer } from "@langchain/community/experimental/callbacks/handlers/datadog"; + +/** + * This example demonstrates how to use the DatadogLLMObsTracer with the OpenAI model. + * It will produce a "llm" span with the input and output of the model inside the meta field. + * + * To run this example, you need to have a valid Datadog API key and OpenAI API key. + */ +export const run = async () => { + const model = new OpenAI({ + model: "gpt-4", + temperature: 0.7, + maxTokens: 1000, + maxRetries: 5, + }); + + const res = await model.invoke( + "Question: What would be a good company name a company that makes colorful socks?\nAnswer:", + { + callbacks: [ + new DatadogLLMObsTracer({ + mlApp: "my-ml-app", + }), + ], + } + ); + + console.log({ res }); +}; diff --git a/libs/langchain-community/.gitignore b/libs/langchain-community/.gitignore index efe903416f39..4246a650f6df 100644 --- a/libs/langchain-community/.gitignore +++ b/libs/langchain-community/.gitignore @@ -1010,6 +1010,10 @@ utils/cassandra.cjs utils/cassandra.js utils/cassandra.d.ts utils/cassandra.d.cts +experimental/callbacks/handlers/datadog.cjs +experimental/callbacks/handlers/datadog.js +experimental/callbacks/handlers/datadog.d.ts +experimental/callbacks/handlers/datadog.d.cts experimental/graph_transformers/llm.cjs experimental/graph_transformers/llm.js experimental/graph_transformers/llm.d.ts diff --git a/libs/langchain-community/langchain.config.js b/libs/langchain-community/langchain.config.js index d71bde86e3ce..7012d8e06c5a 100644 --- a/libs/langchain-community/langchain.config.js +++ b/libs/langchain-community/langchain.config.js @@ -308,6 +308,7 @@ export const config = { "utils/event_source_parse": "utils/event_source_parse", "utils/cassandra": "utils/cassandra", // experimental + "experimental/callbacks/handlers/datadog": "experimental/callbacks/handlers/datadog", "experimental/graph_transformers/llm": "experimental/graph_transformers/llm", "experimental/multimodal_embeddings/googlevertexai": diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index 4b6fc36db469..8774a64deae4 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -2977,6 +2977,15 @@ "import": "./utils/cassandra.js", "require": "./utils/cassandra.cjs" }, + "./experimental/callbacks/handlers/datadog": { + "types": { + "import": "./experimental/callbacks/handlers/datadog.d.ts", + "require": "./experimental/callbacks/handlers/datadog.d.cts", + "default": "./experimental/callbacks/handlers/datadog.d.ts" + }, + "import": "./experimental/callbacks/handlers/datadog.js", + "require": "./experimental/callbacks/handlers/datadog.cjs" + }, "./experimental/graph_transformers/llm": { "types": { "import": "./experimental/graph_transformers/llm.d.ts", @@ -4038,6 +4047,10 @@ "utils/cassandra.js", "utils/cassandra.d.ts", "utils/cassandra.d.cts", + "experimental/callbacks/handlers/datadog.cjs", + "experimental/callbacks/handlers/datadog.js", + "experimental/callbacks/handlers/datadog.d.ts", + "experimental/callbacks/handlers/datadog.d.cts", "experimental/graph_transformers/llm.cjs", "experimental/graph_transformers/llm.js", "experimental/graph_transformers/llm.d.ts", diff --git a/libs/langchain-community/src/experimental/callbacks/handlers/datadog.ts b/libs/langchain-community/src/experimental/callbacks/handlers/datadog.ts new file mode 100644 index 000000000000..57a2e16daf3b --- /dev/null +++ b/libs/langchain-community/src/experimental/callbacks/handlers/datadog.ts @@ -0,0 +1,375 @@ +import { BaseCallbackHandlerInput } from "@langchain/core/callbacks/base"; +import { BaseTracer, Run } from "@langchain/core/tracers/base"; +import { getEnvironmentVariable } from "@langchain/core/utils/env"; +import { Document } from "@langchain/core/documents"; +import { BaseMessage } from "@langchain/core/messages"; +import { ChatGeneration } from "@langchain/core/outputs"; +import { KVMap } from "langsmith/schemas"; + +export type DatadogLLMObsSpanKind = + | "llm" + | "workflow" + | "agent" + | "tool" + | "task" + | "embedding" + | "retrieval"; + +export type DatadogLLMObsIO = + | { value: string } + | { + documents: { + text?: string; + id?: string; + name?: string; + score: string | number; + }[]; + } + | { messages: { content: string; role?: string }[] }; + +export interface DatadogLLMObsSpan { + span_id: string; + trace_id: string; + parent_id: string; + session_id?: string; + name: string; + start_ns: number; + duration: number; + error: number; + status: string; + tags?: string[]; + meta: { + kind: DatadogLLMObsSpanKind; + model_name?: string; + model_provider?: string; + temperature?: string; + input: DatadogLLMObsIO; + output: DatadogLLMObsIO | undefined; + }; + metrics: { [key: string]: number }; +} + +export interface DatadogLLMObsRequestBody { + data: { + type: "span"; + attributes: { + ml_app: string; + tags: string[]; + spans: DatadogLLMObsSpan[]; + session_id?: string; + }; + }; +} + +export type FormatDocument< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + Metadata extends Record = Record +> = (document: Document) => { + text: string; + id: string; + name: string; + score: number; +}; + +export interface DatadogLLMObsTracerFields extends BaseCallbackHandlerInput { + mlApp: string; + userId?: string; + userHandle?: string; + sessionId?: string; + env?: string; + service?: string; + tags?: Record; + ddApiKey?: string; + ddLLMObsEndpoint?: string; + formatDocument?: FormatDocument; +} + +export class DatadogLLMObsTracer + extends BaseTracer + implements DatadogLLMObsTracerFields +{ + name = "datadog_tracer"; + + ddLLMObsEndpoint?: string; + + protected endpoint = + getEnvironmentVariable("DD_LLMOBS_ENDPOINT") || + "https://api.datadoghq.com/api/unstable/llm-obs/v1/trace/spans"; + + protected headers: Record = { + "Content-Type": "application/json", + }; + + mlApp: string; + + sessionId?: string; + + tags: Record = {}; + + formatDocument?: FormatDocument; + + constructor(fields: DatadogLLMObsTracerFields) { + super(fields); + const { + mlApp, + userHandle, + userId, + sessionId, + service, + env, + tags, + ddLLMObsEndpoint, + ddApiKey, + formatDocument, + } = fields; + + const apiKey = ddApiKey || getEnvironmentVariable("DD_API_KEY"); + + if (apiKey) { + this.headers["DD-API-KEY"] = apiKey; + } + + this.mlApp = mlApp; + this.sessionId = sessionId; + this.ddLLMObsEndpoint = ddLLMObsEndpoint; + this.formatDocument = formatDocument; + + this.tags = { + ...tags, + env: env || "not-set", + service: service || "not-set", + user_handle: userHandle, + user_id: userId, + }; + } + + protected async persistRun(_run: Run): Promise { + try { + const spans = this.convertRunToDDSpans(_run); + + const response = await fetch(this.ddLLMObsEndpoint || this.endpoint, { + method: "POST", + headers: this.headers, + body: JSON.stringify(this.formatRequestBody(spans)), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(error); + } + } catch (error) { + console.error(`Error writing spans to Datadog: ${error}`); + } + } + + protected convertRunToDDSpans(run: Run): DatadogLLMObsSpan[] { + const spans = [this.langchainRunToDatadogLLMObsSpan(run)]; + + if (run.child_runs) { + run.child_runs.forEach((childRun) => { + spans.push(...this.convertRunToDDSpans(childRun)); + }); + } + + return spans.flatMap((span) => (span ? [span] : [])); + } + + protected formatRequestBody( + spans: DatadogLLMObsSpan[] + ): DatadogLLMObsRequestBody { + return { + data: { + type: "span", + attributes: { + ml_app: this.mlApp, + tags: Object.entries(this.tags) + .filter(([, value]) => value) + .map(([key, value]) => `${key}:${value}`), + spans, + session_id: this.sessionId, + }, + }, + }; + } + + protected uuidToBigInt(uuid: string): string { + const hexString = uuid.replace(/-/g, ""); + const first64Bits = hexString.slice(0, 16); + const bigIntValue = BigInt("0x" + first64Bits).toString(); + + return bigIntValue; + } + + protected milisecondsToNanoseconds(ms: number): number { + return ms * 1e6; + } + + protected toDatadogSpanKind(kind: string): DatadogLLMObsSpanKind | null { + switch (kind) { + case "llm": + return "llm"; + case "tool": + return "tool"; + case "chain": + return "workflow"; + case "retriever": + return "retrieval"; + default: + return null; + } + } + + protected transformInput( + inputs: KVMap, + spanKind: DatadogLLMObsSpanKind + ): DatadogLLMObsIO { + if (spanKind === "llm") { + if (inputs?.messages) { + return { + messages: inputs?.messages?.flatMap((messages: BaseMessage[]) => + messages.map((message) => ({ + content: message.content, + role: message?._getType?.() ?? undefined, + })) + ), + }; + } + + if (inputs?.prompts) { + return { value: inputs.prompts.join("\n") }; + } + } + + return { value: JSON.stringify(inputs) }; + } + + protected transformOutput( + outputs: KVMap | undefined, + spanKind: DatadogLLMObsSpanKind + ): { + output: DatadogLLMObsIO | undefined; + tokensMetadata: Record; + } { + const tokensMetadata: Record = {}; + + if (!outputs) { + return { output: undefined, tokensMetadata }; + } + + if (spanKind === "llm") { + return { + output: { + messages: outputs?.generations?.flatMap( + (generations: ChatGeneration[]) => + generations.map(({ message, text }) => { + const tokenUsage = message?.response_metadata?.tokenUsage; + if (tokenUsage) { + tokensMetadata.completion_tokens = + tokenUsage.completionTokens; + tokensMetadata.prompt_tokens = tokenUsage.promptTokens; + tokensMetadata.total_tokens = tokenUsage.totalTokens; + } + + return { + content: message?.content ?? text, + role: message?._getType?.(), + }; + }) + ), + }, + tokensMetadata, + }; + } + + if (spanKind === "retrieval") { + return { + output: { + documents: outputs?.documents.map((document: Document) => { + if (typeof this.formatDocument === "function") { + return this.formatDocument(document); + } + + return { + text: document.pageContent, + id: document.metadata?.id, + name: document.metadata?.name, + score: document.metadata?.score, + }; + }), + }, + tokensMetadata, + }; + } + + if (outputs?.output) { + return { + output: { value: JSON.stringify(outputs.output) }, + tokensMetadata, + }; + } + + return { output: { value: JSON.stringify(outputs) }, tokensMetadata }; + } + + protected langchainRunToDatadogLLMObsSpan( + run: Run + ): DatadogLLMObsSpan | null { + if (!run.end_time || !run.trace_id) { + return null; + } + + const spanId = this.uuidToBigInt(run.id); + const traceId = this.uuidToBigInt(run.trace_id); + const parentId = run.parent_run_id + ? this.uuidToBigInt(run.parent_run_id) + : "undefined"; + const spanKind = this.toDatadogSpanKind(run.run_type); + + if (spanKind === null) { + return null; + } + + const input = this.transformInput(run.inputs, spanKind); + const { output, tokensMetadata } = this.transformOutput( + run.outputs, + spanKind + ); + + const startTimeNs = Number(this.milisecondsToNanoseconds(run.start_time)); + const endTimeNs = Number(this.milisecondsToNanoseconds(run.end_time)); + const durationNs = endTimeNs - startTimeNs; + + if (durationNs <= 0) { + return null; + } + + const spanName = + (run.serialized as { kwargs: { name?: string } })?.kwargs?.name ?? + run.name; + const spanError = run.error ? 1 : 0; + const spanStatus = run.error ? "error" : "ok"; + + const meta = { + kind: spanKind, + input, + output, + model_name: run.extra?.metadata?.ls_model_name, + model_provider: run.extra?.metadata?.ls_provider, + temperature: run.extra?.metadata?.ls_temperature, + }; + + return { + parent_id: parentId, + trace_id: traceId, + span_id: spanId, + name: spanName, + error: spanError, + status: spanStatus, + tags: [...(run.tags?.length ? run.tags : [])], + meta, + start_ns: startTimeNs, + duration: durationNs, + metrics: tokensMetadata, + }; + } +} diff --git a/libs/langchain-community/src/experimental/callbacks/handlers/tests/datadog.test.ts b/libs/langchain-community/src/experimental/callbacks/handlers/tests/datadog.test.ts new file mode 100644 index 000000000000..0f5a015a0e81 --- /dev/null +++ b/libs/langchain-community/src/experimental/callbacks/handlers/tests/datadog.test.ts @@ -0,0 +1,386 @@ +import { test, jest, expect } from "@jest/globals"; +import * as uuid from "uuid"; +import { Run } from "@langchain/core/tracers/base"; + +import { HumanMessage, AIMessage } from "@langchain/core/messages"; +import { + DatadogLLMObsRequestBody, + DatadogLLMObsSpan, + DatadogLLMObsTracer, +} from "../datadog.js"; + +const _DATE = 1620000000000; +const _END_DATE = _DATE + 1000; + +Date.now = jest.fn(() => _DATE); + +const BASE_URL = "http://datadog-endpoint"; + +class FakeDatadogLLMObsTracer extends DatadogLLMObsTracer { + public persistRun(_run: Run) { + return super.persistRun(_run); + } + + public uuidToBigInt(uuid: string) { + return super.uuidToBigInt(uuid); + } + + public milisecondsToNanoseconds(ms: number) { + return super.milisecondsToNanoseconds(ms); + } +} + +beforeEach(() => { + const oldFetch = global.fetch; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + global.fetch = jest.fn().mockImplementation(async (url: any, init?: any) => { + if (!url.startsWith(BASE_URL)) return await oldFetch(url, init); + const resp: Response = new Response(); + return resp; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }) as any; +}); + +afterEach(() => { + jest.restoreAllMocks(); +}); + +const runId = uuid.v4(); +const traceId = uuid.v4(); + +const baseRun = { + id: runId, + trace_id: traceId, + parent_run_id: undefined, + name: "test", + start_time: _DATE, + end_time: _END_DATE, + execution_order: 1, + child_execution_order: 0, + + child_runs: [], + extra: {}, + tags: [], + events: [], +}; + +const createBaseSpan = (tracer: FakeDatadogLLMObsTracer) => ({ + span_id: tracer.uuidToBigInt(runId), + trace_id: tracer.uuidToBigInt(traceId), + parent_id: "undefined", + name: "test", + start_ns: tracer.milisecondsToNanoseconds(_DATE), + duration: tracer.milisecondsToNanoseconds(_END_DATE - _DATE), + error: 0, + status: "ok", + metrics: {}, +}); + +const tracerConfig = { + mlApp: "test", + userHandle: "test", + userId: "test", + sessionId: "test", + service: "test", + env: "test", + tags: {}, + ddLLMObsEndpoint: BASE_URL, +}; + +test("Test llm span with message input", async () => { + const tracer = new FakeDatadogLLMObsTracer(tracerConfig); + + const run: Run = { + ...baseRun, + run_type: "llm", + inputs: { + messages: [[new HumanMessage("test")]], + }, + outputs: { + generations: [ + [ + { + message: new AIMessage("test"), + }, + ], + ], + }, + }; + + const compareSpan: DatadogLLMObsSpan = { + ...createBaseSpan(tracer), + meta: { + kind: "llm", + input: { + messages: [{ content: "test", role: "human" }], + }, + output: { + messages: [{ content: "test", role: "ai" }], + }, + }, + }; + + const requestBody: DatadogLLMObsRequestBody = { + data: { + type: "span", + attributes: { + ml_app: "test", + tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], + spans: [compareSpan], + session_id: "test", + }, + }, + }; + + await tracer.persistRun(run); + + expect(fetch).toBeCalledWith(expect.any(String), { + body: expect.any(String), + headers: expect.any(Object), + method: "POST", + }); + + const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; + expect(parsedBody).toMatchObject( + requestBody as unknown as Record + ); +}); + +test("Test llm span with prompt input", async () => { + const tracer = new FakeDatadogLLMObsTracer(tracerConfig); + + const run: Run = { + ...baseRun, + run_type: "llm", + inputs: { + prompts: ["Hello", "World"], + }, + outputs: { + generations: [ + [ + { + text: "Hi", + }, + ], + ], + }, + }; + + const compareSpan: DatadogLLMObsSpan = { + ...createBaseSpan(tracer), + meta: { + kind: "llm", + input: { + value: "Hello\nWorld", + }, + output: { + messages: [{ content: "Hi" }], + }, + }, + }; + + const requestBody: DatadogLLMObsRequestBody = { + data: { + type: "span", + attributes: { + ml_app: "test", + tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], + spans: [compareSpan], + session_id: "test", + }, + }, + }; + + await tracer.persistRun(run); + + expect(fetch).toBeCalledWith(expect.any(String), { + body: expect.any(String), + headers: expect.any(Object), + method: "POST", + }); + + const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; + expect(parsedBody).toMatchObject( + requestBody as unknown as Record + ); +}); + +test("Test workflow span", async () => { + const tracer = new FakeDatadogLLMObsTracer(tracerConfig); + + const run: Run = { + ...baseRun, + run_type: "chain", + inputs: { + question: "test", + }, + outputs: { + output: "test", + }, + tags: ["seq:test"], + }; + + const compareSpan: DatadogLLMObsSpan = { + ...createBaseSpan(tracer), + meta: { + kind: "workflow", + input: { + value: JSON.stringify(run.inputs), + }, + output: { + value: JSON.stringify(run.outputs?.output), + }, + }, + tags: run.tags, + }; + + const requestBody: DatadogLLMObsRequestBody = { + data: { + type: "span", + attributes: { + ml_app: "test", + tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], + spans: [compareSpan], + session_id: "test", + }, + }, + }; + + await tracer.persistRun(run); + + expect(fetch).toBeCalledWith(expect.any(String), { + body: expect.any(String), + headers: expect.any(Object), + method: "POST", + }); + + const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; + expect(parsedBody).toMatchObject( + requestBody as unknown as Record + ); +}); + +test("Test tool span", async () => { + const tracer = new FakeDatadogLLMObsTracer(tracerConfig); + + const run: Run = { + ...baseRun, + run_type: "tool", + inputs: { + input: { query: "test" }, + }, + outputs: { + output: "test", + }, + }; + + const compareSpan: DatadogLLMObsSpan = { + ...createBaseSpan(tracer), + meta: { + kind: "tool", + input: { + value: JSON.stringify(run.inputs), + }, + output: { + value: JSON.stringify(run.outputs?.output), + }, + }, + }; + + const requestBody: DatadogLLMObsRequestBody = { + data: { + type: "span", + attributes: { + ml_app: "test", + tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], + spans: [compareSpan], + session_id: "test", + }, + }, + }; + + await tracer.persistRun(run); + + expect(fetch).toBeCalledWith(expect.any(String), { + body: expect.any(String), + headers: expect.any(Object), + method: "POST", + }); + + const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; + expect(parsedBody).toMatchObject( + requestBody as unknown as Record + ); +}); + +test("Test retrieval span", async () => { + const tracer = new FakeDatadogLLMObsTracer(tracerConfig); + + const run: Run = { + ...baseRun, + run_type: "retriever", + inputs: { + input: { query: "test" }, + }, + outputs: { + documents: [ + { + pageContent: "test", + metadata: { id: "1", name: "test", score: 0.1 }, + }, + ], + }, + }; + + const compareSpan: DatadogLLMObsSpan = { + ...createBaseSpan(tracer), + meta: { + kind: "retrieval", + input: { + value: JSON.stringify(run.inputs), + }, + output: { + documents: [ + { + text: "test", + id: "1", + name: "test", + score: 0.1, + }, + ], + }, + }, + }; + + const requestBody: DatadogLLMObsRequestBody = { + data: { + type: "span", + attributes: { + ml_app: "test", + tags: ["env:test", "service:test", "user_handle:test", "user_id:test"], + spans: [compareSpan], + session_id: "test", + }, + }, + }; + + await tracer.persistRun(run); + + expect(fetch).toBeCalledWith(expect.any(String), { + body: expect.any(String), + headers: expect.any(Object), + method: "POST", + }); + + const { body } = (fetch as jest.Mock).mock.calls[0][1] as { body: string }; + const parsedBody = JSON.parse(body) as DatadogLLMObsRequestBody; + expect(parsedBody).toMatchObject( + requestBody as unknown as Record + ); +}); diff --git a/libs/langchain-community/src/load/import_map.ts b/libs/langchain-community/src/load/import_map.ts index b8b0a18e0564..d06053cbde26 100644 --- a/libs/langchain-community/src/load/import_map.ts +++ b/libs/langchain-community/src/load/import_map.ts @@ -74,5 +74,6 @@ export * as document_loaders__web__searchapi from "../document_loaders/web/searc export * as document_loaders__web__serpapi from "../document_loaders/web/serpapi.js"; export * as document_loaders__web__sort_xyz_blockchain from "../document_loaders/web/sort_xyz_blockchain.js"; export * as utils__event_source_parse from "../utils/event_source_parse.js"; +export * as experimental__callbacks__handlers__datadog from "../experimental/callbacks/handlers/datadog.js"; export * as experimental__graph_transformers__llm from "../experimental/graph_transformers/llm.js"; export * as experimental__chat_models__ollama_functions from "../experimental/chat_models/ollama_functions.js";