community-experimental[minor]: feat: implement DatadogLLMObsTracer (#…

…5867) * chore: implement DatadogLLMObsTracer * chore: declare module into langchain-community/langchain.config.js * fix: remove dangling comma from langchain.config.js * fix: wrong import on datadog tracer example * chore: add autogenerated artifacts * fix: typo on UsageExmaple import * chore: move datadog llmobs from tracers to callbacks for consistency * chore: handle propagating tags to spans * chore: update session_id based on recent DD API Schema update * Update imports, move to experimental --------- Co-authored-by: jacoblee93 <jacoblee93@gmail.com>
langchain-ai · Jun 25, 2024 · 16fed47 · 16fed47
1 parent bbb7628
commit 16fed47
Show file tree

Hide file tree

Showing 10 changed files with 851 additions and 2 deletions.
diff --git a/docs/core_docs/.gitignore b/docs/core_docs/.gitignore
@@ -53,6 +53,8 @@ docs/tutorials/classification.md
 docs/tutorials/classification.mdx
 docs/tutorials/chatbot.md
 docs/tutorials/chatbot.mdx
+docs/how_to/trim_messages.md
+docs/how_to/trim_messages.mdx
 docs/how_to/tools_prompting.md
 docs/how_to/tools_prompting.mdx
 docs/how_to/tools_builtin.md
@@ -117,6 +119,8 @@ docs/how_to/multimodal_inputs.md
 docs/how_to/multimodal_inputs.mdx
 docs/how_to/migrate_agent.md
 docs/how_to/migrate_agent.mdx
+docs/how_to/merge_message_runs.md
+docs/how_to/merge_message_runs.mdx
 docs/how_to/logprobs.md
 docs/how_to/logprobs.mdx
 docs/how_to/lcel_cheatsheet.md
@@ -131,6 +135,8 @@ docs/how_to/graph_constructing.md
 docs/how_to/graph_constructing.mdx
 docs/how_to/functions.md
 docs/how_to/functions.mdx
+docs/how_to/filter_messages.md
+docs/how_to/filter_messages.mdx
 docs/how_to/few_shot_examples_chat.md
 docs/how_to/few_shot_examples_chat.mdx
 docs/how_to/few_shot_examples.md

diff --git a/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx b/docs/core_docs/docs/integrations/callbacks/datadog_tracer.mdx
@@ -0,0 +1,31 @@
+---
+sidebar_class_name: beta
+---
+
+import CodeBlock from "@theme/CodeBlock";
+
+# Datadog LLM Observability
+
+:::warning
+LLM Observability is in public beta, and its API is subject to change.
+:::
+
+With [Datadog LLM Observability](https://docs.datadoghq.com/llm_observability/), you can monitor, troubleshoot, and evaluate your LLM-powered applications, such as chatbots. You can investigate the root cause of issues, monitor operational performance, and evaluate the quality, privacy, and safety of your LLM applications.
+
+This is an experimental community implementation, and it is not officially supported by Datadog. It is based on the [Datadog LLM Observability API](https://docs.datadoghq.com/llm_observability/api).
+
+## Setup
+
+import IntegrationInstallTooltip from "@mdx_components/integration_install_tooltip.mdx";
+
+<IntegrationInstallTooltip></IntegrationInstallTooltip>
+
+```bash npm2yarn
+npm install @langchain/community
+```
+
+## Usage
+
+import UsageExample from "@examples/callbacks/datadog.ts";
+
+<CodeBlock language="typescript">{UsageExample}</CodeBlock>
diff --git a/docs/core_docs/src/css/custom.css b/docs/core_docs/src/css/custom.css
@@ -101,7 +101,8 @@ nav, h1, h2, h3, h4 {
 /* Override `beta` color */
 .beta::after {
   content: "Beta";
-  color: #58006e;
+  color: #fff;
+  background: #58006e;
   border: 1px solid #58006e;
 }
 
@@ -114,7 +115,8 @@ nav, h1, h2, h3, h4 {
 /* Override `web-only` color */
 .web-only::after {
   content: "Web-only";
-  color: #0a0072;
+  color: #fff;
+  background: #0a0072;
   border: 1px solid #0a0072;
 }
 

diff --git a/examples/src/callbacks/datadog.ts b/examples/src/callbacks/datadog.ts
@@ -0,0 +1,30 @@
+import { OpenAI } from "@langchain/openai";
+import { DatadogLLMObsTracer } from "@langchain/community/experimental/callbacks/handlers/datadog";
+
+/**
+ * This example demonstrates how to use the DatadogLLMObsTracer with the OpenAI model.
+ * It will produce a "llm" span with the input and output of the model inside the meta field.
+ *
+ * To run this example, you need to have a valid Datadog API key and OpenAI API key.
+ */
+export const run = async () => {
+  const model = new OpenAI({
+    model: "gpt-4",
+    temperature: 0.7,
+    maxTokens: 1000,
+    maxRetries: 5,
+  });
+
+  const res = await model.invoke(
+    "Question: What would be a good company name a company that makes colorful socks?\nAnswer:",
+    {
+      callbacks: [
+        new DatadogLLMObsTracer({
+          mlApp: "my-ml-app",
+        }),
+      ],
+    }
+  );
+
+  console.log({ res });
+};
diff --git a/libs/langchain-community/.gitignore b/libs/langchain-community/.gitignore
@@ -1010,6 +1010,10 @@ utils/cassandra.cjs
 utils/cassandra.js
 utils/cassandra.d.ts
 utils/cassandra.d.cts
+experimental/callbacks/handlers/datadog.cjs
+experimental/callbacks/handlers/datadog.js
+experimental/callbacks/handlers/datadog.d.ts
+experimental/callbacks/handlers/datadog.d.cts
 experimental/graph_transformers/llm.cjs
 experimental/graph_transformers/llm.js
 experimental/graph_transformers/llm.d.ts

diff --git a/libs/langchain-community/langchain.config.js b/libs/langchain-community/langchain.config.js
@@ -308,6 +308,7 @@ export const config = {
     "utils/event_source_parse": "utils/event_source_parse",
     "utils/cassandra": "utils/cassandra",
     // experimental
+    "experimental/callbacks/handlers/datadog": "experimental/callbacks/handlers/datadog",
     "experimental/graph_transformers/llm":
       "experimental/graph_transformers/llm",
     "experimental/multimodal_embeddings/googlevertexai":

diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json
@@ -2977,6 +2977,15 @@
       "import": "./utils/cassandra.js",
       "require": "./utils/cassandra.cjs"
     },
+    "./experimental/callbacks/handlers/datadog": {
+      "types": {
+        "import": "./experimental/callbacks/handlers/datadog.d.ts",
+        "require": "./experimental/callbacks/handlers/datadog.d.cts",
+        "default": "./experimental/callbacks/handlers/datadog.d.ts"
+      },
+      "import": "./experimental/callbacks/handlers/datadog.js",
+      "require": "./experimental/callbacks/handlers/datadog.cjs"
+    },
     "./experimental/graph_transformers/llm": {
       "types": {
         "import": "./experimental/graph_transformers/llm.d.ts",
@@ -4038,6 +4047,10 @@
     "utils/cassandra.js",
     "utils/cassandra.d.ts",
     "utils/cassandra.d.cts",
+    "experimental/callbacks/handlers/datadog.cjs",
+    "experimental/callbacks/handlers/datadog.js",
+    "experimental/callbacks/handlers/datadog.d.ts",
+    "experimental/callbacks/handlers/datadog.d.cts",
     "experimental/graph_transformers/llm.cjs",
     "experimental/graph_transformers/llm.js",
     "experimental/graph_transformers/llm.d.ts",