Merge branch 'py/rke/pytest-plugin' of github.com:rkechols/langsmith-…

…sdk-pytest into 1472

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [

js/src/client.ts

@@ -575,6 +575,9 @@ export class Client implements LangSmithTracingClientInterface {
     } else if (isLocalhost(this.apiUrl)) {
       this.webUrl = "http://localhost:3000";
       return this.webUrl;
+    } else if (this.apiUrl.endsWith("/api/v1")) {
+      this.webUrl = this.apiUrl.replace("/api/v1", "");
+      return this.webUrl;
     } else if (
       this.apiUrl.includes("/api") &&
       !this.apiUrl.split(".", 1)[0].endsWith("api")
@@ -587,6 +590,9 @@ export class Client implements LangSmithTracingClientInterface {
     } else if (this.apiUrl.split(".", 1)[0].includes("eu")) {
       this.webUrl = "https://eu.smith.langchain.com";
       return this.webUrl;
+    } else if (this.apiUrl.split(".", 1)[0].includes("beta")) {
+      this.webUrl = "https://beta.smith.langchain.com";
+      return this.webUrl;
     } else {
       this.webUrl = "https://smith.langchain.com";
       return this.webUrl;

js/src/index.ts

@@ -18,4 +18,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.3.3";
+export const __version__ = "0.3.4";

js/src/jest/reporter.ts

@@ -5,6 +5,9 @@ import { printReporterTable } from "../utils/jestlike/reporter.js";
 
 class LangSmithEvalReporter extends DefaultReporter {
   async onTestResult(test: any, testResult: any, aggregatedResults: any) {
+    if (testResult.failureMessage) {
+      console.log(testResult.failureMessage);
+    }
     const groupedTestResults = testResult.testResults.reduce(
       (groups: Record<string, any>, testResult: any) => {
         const ancestorTitle = testResult.ancestorTitles.join(" > ");
@@ -19,7 +22,16 @@ class LangSmithEvalReporter extends DefaultReporter {
     try {
       for (const testGroupName of Object.keys(groupedTestResults)) {
         const resultGroup = groupedTestResults[testGroupName];
-        await printReporterTable(resultGroup, testResult.failureMessage);
+        const unskippedTests = resultGroup.filter(
+          (result: any) => result.status !== "pending"
+        );
+        const overallResult =
+          unskippedTests.length === 0
+            ? "skip"
+            : unskippedTests.every((result: any) => result.status === "passed")
+            ? "pass"
+            : "fail";
+        await printReporterTable(testGroupName, resultGroup, overallResult);
       }
     } catch (e: any) {
       console.log("Failed to display LangSmith eval results:", e.message);

js/src/tests/client.test.ts

@@ -122,6 +122,14 @@ describe("Client", () => {
       expect(result).toBe("https://example.com");
     });
 
+    it("should trim /api/v1 from the apiUrl", () => {
+      const client = new Client({
+        apiUrl: "https://my-other-domain.com/api/v1",
+      });
+      const result = (client as any).getHostUrl();
+      expect(result).toBe("https://my-other-domain.com");
+    });
+
     it("should return 'https://dev.smith.langchain.com' if apiUrl contains 'dev'", () => {
       const client = new Client({
         apiUrl: "https://dev.smith.langchain.com/api",
@@ -131,6 +139,15 @@ describe("Client", () => {
       expect(result).toBe("https://dev.smith.langchain.com");
     });
 
+    it("should return 'https://beta.smith.langchain.com' if apiUrl contains 'beta'", () => {
+      const client = new Client({
+        apiUrl: "https://beta.smith.langchain.com/api",
+        apiKey: "test-api-key",
+      });
+      const result = (client as any).getHostUrl();
+      expect(result).toBe("https://beta.smith.langchain.com");
+    });
+
     it("should return 'https://eu.smith.langchain.com' if apiUrl contains 'eu'", () => {
       const client = new Client({
         apiUrl: "https://eu.smith.langchain.com/api",

js/src/tests/jestlike/vitest_separate_file.vitesteval.ts

@@ -49,3 +49,43 @@ ls.describe(
     },
   }
 );
+
+ls.describe(
+  "js vitest 3",
+  () => {
+    ls.test.each(
+      [
+        {
+          inputs: {
+            one: "uno",
+          },
+          referenceOutputs: {
+            ein: "un",
+          },
+        },
+        {
+          inputs: {
+            two: "dos",
+          },
+          referenceOutputs: {
+            zwei: "deux",
+          },
+        },
+      ],
+      { iterations: 3, metadata: { something: "cool" } }
+    )("Does the thing", async ({ inputs, referenceOutputs }) => {
+      const myApp = () => {
+        return { bar: "bad" };
+      };
+      const res = myApp();
+      const evaluator = ls.wrapEvaluator(myEvaluator);
+      await evaluator({ inputs, referenceOutputs, outputs: res });
+      return res;
+    });
+  },
+  {
+    metadata: {
+      model: "test-model",
+    },
+  }
+);

js/src/utils/jestlike/reporter.ts

@@ -30,11 +30,11 @@ function formatTestName(name: string, duration: number) {
 
 function getFormattedStatus(status: string) {
   const s = status.toLowerCase();
-  if (s === "pending") {
+  if (s === "pending" || s === "skipped") {
     return chalk.yellow("○ Skipped");
-  } else if (s === "passed") {
+  } else if (s.includes("pass")) {
     return chalk.green("✓ Passed");
-  } else if (s === "failed") {
+  } else if (s.includes("fail")) {
     return chalk.red("✕ Failed");
   } else {
     return status;
@@ -43,11 +43,11 @@ function getFormattedStatus(status: string) {
 
 function getColorParam(status: string) {
   const s = status.toLowerCase();
-  if (s === "pending") {
+  if (s === "pending" || s === "skipped") {
     return { color: "yellow" };
-  } else if (s === "passed") {
+  } else if (s.includes("pass")) {
     return { color: "grey" };
-  } else if (s === "failed") {
+  } else if (s.includes("fail")) {
     return { color: "red" };
   } else {
     return {};
@@ -75,7 +75,13 @@ function formatValue(value: unknown) {
 }
 
 export async function printReporterTable(
-  results: { title: string; duration: number; status: string }[],
+  testSuiteName: string,
+  results: {
+    title: string;
+    duration: number;
+    status: "pass" | "passed" | "fail" | "failed" | "pending" | "skipped";
+  }[],
+  testStatus: "pass" | "skip" | "fail",
   failureMessage?: string
 ) {
   const rows = [];
@@ -101,7 +107,7 @@ export async function printReporterTable(
         },
         getColorParam(status),
       ]);
-    } else if (status === "pending") {
+    } else if (status === "pending" || status === "skipped") {
       // Skipped
       rows.push([
         {
@@ -265,6 +271,14 @@ export async function printReporterTable(
   for (const row of rows) {
     table.addRow(row[0], row[1]);
   }
+  const testStatusColor = testStatus.includes("pass")
+    ? chalk.green
+    : testStatus.includes("fail")
+    ? chalk.red
+    : chalk.yellow;
+  if (testSuiteName) {
+    console.log(testStatusColor(`› ${testSuiteName}`));
+  }
   if (failureMessage) {
     console.log(failureMessage);
   }

js/src/vitest/reporter.ts

@@ -10,15 +10,22 @@ class LangSmithEvalReporter extends DefaultReporter {
   async onFinished(files: RunnerTestFile[], errors: unknown[]) {
     super.onFinished(files, errors);
     for (const file of files) {
-      const testModule = this.ctx.state.getReportedEntity(file) as TestModule;
-      const tests = [...testModule.children.allTests()].map((test) => {
-        return {
-          title: test.name,
-          status: test.result()?.state ?? "skipped",
-          duration: Math.round(test.diagnostic()?.duration ?? 0),
-        };
-      });
-      await printReporterTable(tests);
+      for (const task of file.tasks) {
+        const testModule = this.ctx.state.getReportedEntity(task) as TestModule;
+        const tests = [...testModule.children.allTests()].map((test) => {
+          return {
+            title: test.name,
+            status: test.result()?.state ?? "skipped",
+            duration: Math.round(test.diagnostic()?.duration ?? 0),
+          };
+        });
+        const result = ["pass", "fail", "skip"].includes(
+          task.result?.state ?? ""
+        )
+          ? (task.result?.state as "pass" | "fail" | "skip")
+          : "skip";
+        await printReporterTable(task.name, tests, result);
+      }
     }
   }
 }

python/README.md

@@ -422,14 +422,7 @@ my_function("Jason is 25 years old")
 The LangSmith pytest plugin lets Python developers define their datasets and evaluations as pytest test cases.
 See [online docs](https://docs.smith.langchain.com/evaluation/how_to_guides/pytest) for more information.
 
-This plugin is installed as part of the LangSmith SDK, but is not enabled by default.  
-To enable the plugin for your project, add `"langsmith.pytest_plugin"` to the `pytest_plugins` list in your root `conftest.py` file:
-
-```python
-# root `conftest.py` file
-pytest_plugins = ["langsmith.pytest_plugin"]
-```
-
+This plugin is installed as part of the LangSmith SDK, and is enabled by default.
 See also official pytest docs: [How to install and use plugins](https://docs.pytest.org/en/stable/how-to/plugins.html)
 
 ## Additional Documentation

python/langsmith/utils.py

@@ -770,10 +770,15 @@ def get_host_url(web_url: Optional[str], api_url: str):
     elif str(parsed_url.path).endswith("/api"):
         new_path = str(parsed_url.path).rsplit("/api", 1)[0]
         link = urllib_parse.urlunparse(parsed_url._replace(path=new_path))
+    elif str(parsed_url.path).endswith("/api/v1"):
+        new_path = str(parsed_url.path).rsplit("/api/v1", 1)[0]
+        link = urllib_parse.urlunparse(parsed_url._replace(path=new_path))
     elif str(parsed_url.netloc).startswith("eu."):
         link = "https://eu.smith.langchain.com"
     elif str(parsed_url.netloc).startswith("dev."):
         link = "https://dev.smith.langchain.com"
+    elif str(parsed_url.netloc).startswith("beta."):
+        link = "https://beta.smith.langchain.com"
     else:
         link = "https://smith.langchain.com"
     return link

python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langsmith"
-version = "0.3.2"
+version = "0.3.3"
 description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
 authors = ["LangChain <support@langchain.dev>"]
 license = "MIT"

python/tests/unit_tests/test_utils.py

@@ -409,3 +409,62 @@ class BarClass:
     assert ls_utils._get_function_name(print) == "print"
 
     assert ls_utils._get_function_name("not_a_function") == "not_a_function"
+
+
+def test_get_host_url():
+    # If web_url is explicitly provided, it takes precedence over api_url.
+    assert (
+        ls_utils.get_host_url(
+            "https://my-custom-web.com", "https://api.smith.langchain.com"
+        )
+        == "https://my-custom-web.com"
+    )
+
+    # When web_url is None and api_url is localhost.
+    assert ls_utils.get_host_url(None, "http://localhost:5000") == "http://localhost"
+    # A port variation on localhost.
+    assert (
+        ls_utils.get_host_url(None, "http://127.0.0.1:8080") == "http://localhost"
+    ), "Should recognize 127.x.x.x as localhost."
+
+    # If api_url path ends with /api, trimmed back to netloc.
+    assert (
+        ls_utils.get_host_url(None, "https://my-awesome-domain.com/api")
+        == "https://my-awesome-domain.com"
+    )
+
+    # If api_url path ends with /api/v1, trimmed back to netloc.
+    assert (
+        ls_utils.get_host_url(None, "https://my-other-domain.com/api/v1")
+        == "https://my-other-domain.com"
+    )
+
+    # If netloc begins with dev.
+    assert (
+        ls_utils.get_host_url(None, "https://dev.smith.langchain.com/api/v1")
+        == "https://dev.smith.langchain.com"
+    )
+
+    # If netloc begins with eu.
+    assert (
+        ls_utils.get_host_url(None, "https://eu.smith.langchain.com/api")
+        == "https://eu.smith.langchain.com"
+    )
+
+    # If netloc begins with beta.
+    assert (
+        ls_utils.get_host_url(None, "https://beta.smith.langchain.com")
+        == "https://beta.smith.langchain.com"
+    )
+
+    # If netloc begins with api.
+    assert (
+        ls_utils.get_host_url(None, "https://api.smith.langchain.com")
+        == "https://smith.langchain.com"
+    )
+
+    # Otherwise, returns https://smith.langchain.com for unknown host.
+    assert (
+        ls_utils.get_host_url(None, "https://unknownhost.com")
+        == "https://smith.langchain.com"
+    )