From 157ab3fa52fd0367f6a6cfafe2d525e0da5d5e3f Mon Sep 17 00:00:00 2001 From: Bagatur <22008038+baskaryan@users.noreply.github.com> Date: Wed, 29 Jan 2025 15:44:47 -0800 Subject: [PATCH 1/5] python[patch]: fix example link [LS-2764] (#1473) --- python/langsmith/utils.py | 5 +++ python/tests/unit_tests/test_utils.py | 59 +++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/python/langsmith/utils.py b/python/langsmith/utils.py index d18222909..7fdf7f5b0 100644 --- a/python/langsmith/utils.py +++ b/python/langsmith/utils.py @@ -770,10 +770,15 @@ def get_host_url(web_url: Optional[str], api_url: str): elif str(parsed_url.path).endswith("/api"): new_path = str(parsed_url.path).rsplit("/api", 1)[0] link = urllib_parse.urlunparse(parsed_url._replace(path=new_path)) + elif str(parsed_url.path).endswith("/api/v1"): + new_path = str(parsed_url.path).rsplit("/api/v1", 1)[0] + link = urllib_parse.urlunparse(parsed_url._replace(path=new_path)) elif str(parsed_url.netloc).startswith("eu."): link = "https://eu.smith.langchain.com" elif str(parsed_url.netloc).startswith("dev."): link = "https://dev.smith.langchain.com" + elif str(parsed_url.netloc).startswith("beta."): + link = "https://beta.smith.langchain.com" else: link = "https://smith.langchain.com" return link diff --git a/python/tests/unit_tests/test_utils.py b/python/tests/unit_tests/test_utils.py index a0ea16db4..17d512de6 100644 --- a/python/tests/unit_tests/test_utils.py +++ b/python/tests/unit_tests/test_utils.py @@ -409,3 +409,62 @@ class BarClass: assert ls_utils._get_function_name(print) == "print" assert ls_utils._get_function_name("not_a_function") == "not_a_function" + + +def test_get_host_url(): + # If web_url is explicitly provided, it takes precedence over api_url. + assert ( + ls_utils.get_host_url( + "https://my-custom-web.com", "https://api.smith.langchain.com" + ) + == "https://my-custom-web.com" + ) + + # When web_url is None and api_url is localhost. + assert ls_utils.get_host_url(None, "http://localhost:5000") == "http://localhost" + # A port variation on localhost. + assert ( + ls_utils.get_host_url(None, "http://127.0.0.1:8080") == "http://localhost" + ), "Should recognize 127.x.x.x as localhost." + + # If api_url path ends with /api, trimmed back to netloc. + assert ( + ls_utils.get_host_url(None, "https://my-awesome-domain.com/api") + == "https://my-awesome-domain.com" + ) + + # If api_url path ends with /api/v1, trimmed back to netloc. + assert ( + ls_utils.get_host_url(None, "https://my-other-domain.com/api/v1") + == "https://my-other-domain.com" + ) + + # If netloc begins with dev. + assert ( + ls_utils.get_host_url(None, "https://dev.smith.langchain.com/api/v1") + == "https://dev.smith.langchain.com" + ) + + # If netloc begins with eu. + assert ( + ls_utils.get_host_url(None, "https://eu.smith.langchain.com/api") + == "https://eu.smith.langchain.com" + ) + + # If netloc begins with beta. + assert ( + ls_utils.get_host_url(None, "https://beta.smith.langchain.com") + == "https://beta.smith.langchain.com" + ) + + # If netloc begins with api. + assert ( + ls_utils.get_host_url(None, "https://api.smith.langchain.com") + == "https://smith.langchain.com" + ) + + # Otherwise, returns https://smith.langchain.com for unknown host. + assert ( + ls_utils.get_host_url(None, "https://unknownhost.com") + == "https://smith.langchain.com" + ) From b9ec9793e7aff3ed04e0e78199101f2bd1e99d38 Mon Sep 17 00:00:00 2001 From: Bagatur <22008038+baskaryan@users.noreply.github.com> Date: Wed, 29 Jan 2025 20:16:27 -0800 Subject: [PATCH 2/5] python[patch]: Release 0.3.3 (#1474) --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 44b5bb0c9..5712ed336 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langsmith" -version = "0.3.2" +version = "0.3.3" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." authors = ["LangChain "] license = "MIT" From e143f921dd61d0b2ce4abf14a7d81caf0af1b3e6 Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 30 Jan 2025 08:35:34 -0800 Subject: [PATCH 3/5] fix(js): Evals link should use user-defined endpoint (#1476) --- js/src/client.ts | 6 ++++++ js/src/tests/client.test.ts | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/js/src/client.ts b/js/src/client.ts index 83c428e85..8509120e6 100644 --- a/js/src/client.ts +++ b/js/src/client.ts @@ -575,6 +575,9 @@ export class Client implements LangSmithTracingClientInterface { } else if (isLocalhost(this.apiUrl)) { this.webUrl = "http://localhost:3000"; return this.webUrl; + } else if (this.apiUrl.endsWith("/api/v1")) { + this.webUrl = this.apiUrl.replace("/api/v1", ""); + return this.webUrl; } else if ( this.apiUrl.includes("/api") && !this.apiUrl.split(".", 1)[0].endsWith("api") @@ -587,6 +590,9 @@ export class Client implements LangSmithTracingClientInterface { } else if (this.apiUrl.split(".", 1)[0].includes("eu")) { this.webUrl = "https://eu.smith.langchain.com"; return this.webUrl; + } else if (this.apiUrl.split(".", 1)[0].includes("beta")) { + this.webUrl = "https://beta.smith.langchain.com"; + return this.webUrl; } else { this.webUrl = "https://smith.langchain.com"; return this.webUrl; diff --git a/js/src/tests/client.test.ts b/js/src/tests/client.test.ts index d86c0dc24..3481b793c 100644 --- a/js/src/tests/client.test.ts +++ b/js/src/tests/client.test.ts @@ -122,6 +122,14 @@ describe("Client", () => { expect(result).toBe("https://example.com"); }); + it("should trim /api/v1 from the apiUrl", () => { + const client = new Client({ + apiUrl: "https://my-other-domain.com/api/v1", + }); + const result = (client as any).getHostUrl(); + expect(result).toBe("https://my-other-domain.com"); + }); + it("should return 'https://dev.smith.langchain.com' if apiUrl contains 'dev'", () => { const client = new Client({ apiUrl: "https://dev.smith.langchain.com/api", @@ -131,6 +139,15 @@ describe("Client", () => { expect(result).toBe("https://dev.smith.langchain.com"); }); + it("should return 'https://beta.smith.langchain.com' if apiUrl contains 'beta'", () => { + const client = new Client({ + apiUrl: "https://beta.smith.langchain.com/api", + apiKey: "test-api-key", + }); + const result = (client as any).getHostUrl(); + expect(result).toBe("https://beta.smith.langchain.com"); + }); + it("should return 'https://eu.smith.langchain.com' if apiUrl contains 'eu'", () => { const client = new Client({ apiUrl: "https://eu.smith.langchain.com/api", From 232618ba4a8f5994a7a04518ac381bdab37b546e Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 30 Jan 2025 08:47:24 -0800 Subject: [PATCH 4/5] fix(js): Fix multiple datasets in same test file for Vitest, improve reporter output (#1475) Please merge after #1476 since this has a version bump --- js/package.json | 2 +- js/src/index.ts | 2 +- js/src/jest/reporter.ts | 14 ++++++- .../vitest_separate_file.vitesteval.ts | 40 +++++++++++++++++++ js/src/utils/jestlike/reporter.ts | 30 ++++++++++---- js/src/vitest/reporter.ts | 25 +++++++----- 6 files changed, 93 insertions(+), 20 deletions(-) diff --git a/js/package.json b/js/package.json index 7c9300862..dae2fd525 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "langsmith", - "version": "0.3.3", + "version": "0.3.4", "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.", "packageManager": "yarn@1.22.19", "files": [ diff --git a/js/src/index.ts b/js/src/index.ts index 1d05820cb..7e92141da 100644 --- a/js/src/index.ts +++ b/js/src/index.ts @@ -18,4 +18,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js"; export { overrideFetchImplementation } from "./singletons/fetch.js"; // Update using yarn bump-version -export const __version__ = "0.3.3"; +export const __version__ = "0.3.4"; diff --git a/js/src/jest/reporter.ts b/js/src/jest/reporter.ts index ad9d57243..31bc62973 100644 --- a/js/src/jest/reporter.ts +++ b/js/src/jest/reporter.ts @@ -5,6 +5,9 @@ import { printReporterTable } from "../utils/jestlike/reporter.js"; class LangSmithEvalReporter extends DefaultReporter { async onTestResult(test: any, testResult: any, aggregatedResults: any) { + if (testResult.failureMessage) { + console.log(testResult.failureMessage); + } const groupedTestResults = testResult.testResults.reduce( (groups: Record, testResult: any) => { const ancestorTitle = testResult.ancestorTitles.join(" > "); @@ -19,7 +22,16 @@ class LangSmithEvalReporter extends DefaultReporter { try { for (const testGroupName of Object.keys(groupedTestResults)) { const resultGroup = groupedTestResults[testGroupName]; - await printReporterTable(resultGroup, testResult.failureMessage); + const unskippedTests = resultGroup.filter( + (result: any) => result.status !== "pending" + ); + const overallResult = + unskippedTests.length === 0 + ? "skip" + : unskippedTests.every((result: any) => result.status === "passed") + ? "pass" + : "fail"; + await printReporterTable(testGroupName, resultGroup, overallResult); } } catch (e: any) { console.log("Failed to display LangSmith eval results:", e.message); diff --git a/js/src/tests/jestlike/vitest_separate_file.vitesteval.ts b/js/src/tests/jestlike/vitest_separate_file.vitesteval.ts index 0bd3cfef6..15c3c7c27 100644 --- a/js/src/tests/jestlike/vitest_separate_file.vitesteval.ts +++ b/js/src/tests/jestlike/vitest_separate_file.vitesteval.ts @@ -49,3 +49,43 @@ ls.describe( }, } ); + +ls.describe( + "js vitest 3", + () => { + ls.test.each( + [ + { + inputs: { + one: "uno", + }, + referenceOutputs: { + ein: "un", + }, + }, + { + inputs: { + two: "dos", + }, + referenceOutputs: { + zwei: "deux", + }, + }, + ], + { iterations: 3, metadata: { something: "cool" } } + )("Does the thing", async ({ inputs, referenceOutputs }) => { + const myApp = () => { + return { bar: "bad" }; + }; + const res = myApp(); + const evaluator = ls.wrapEvaluator(myEvaluator); + await evaluator({ inputs, referenceOutputs, outputs: res }); + return res; + }); + }, + { + metadata: { + model: "test-model", + }, + } +); diff --git a/js/src/utils/jestlike/reporter.ts b/js/src/utils/jestlike/reporter.ts index 649977950..2ef6c83b2 100644 --- a/js/src/utils/jestlike/reporter.ts +++ b/js/src/utils/jestlike/reporter.ts @@ -30,11 +30,11 @@ function formatTestName(name: string, duration: number) { function getFormattedStatus(status: string) { const s = status.toLowerCase(); - if (s === "pending") { + if (s === "pending" || s === "skipped") { return chalk.yellow("○ Skipped"); - } else if (s === "passed") { + } else if (s.includes("pass")) { return chalk.green("✓ Passed"); - } else if (s === "failed") { + } else if (s.includes("fail")) { return chalk.red("✕ Failed"); } else { return status; @@ -43,11 +43,11 @@ function getFormattedStatus(status: string) { function getColorParam(status: string) { const s = status.toLowerCase(); - if (s === "pending") { + if (s === "pending" || s === "skipped") { return { color: "yellow" }; - } else if (s === "passed") { + } else if (s.includes("pass")) { return { color: "grey" }; - } else if (s === "failed") { + } else if (s.includes("fail")) { return { color: "red" }; } else { return {}; @@ -75,7 +75,13 @@ function formatValue(value: unknown) { } export async function printReporterTable( - results: { title: string; duration: number; status: string }[], + testSuiteName: string, + results: { + title: string; + duration: number; + status: "pass" | "passed" | "fail" | "failed" | "pending" | "skipped"; + }[], + testStatus: "pass" | "skip" | "fail", failureMessage?: string ) { const rows = []; @@ -101,7 +107,7 @@ export async function printReporterTable( }, getColorParam(status), ]); - } else if (status === "pending") { + } else if (status === "pending" || status === "skipped") { // Skipped rows.push([ { @@ -265,6 +271,14 @@ export async function printReporterTable( for (const row of rows) { table.addRow(row[0], row[1]); } + const testStatusColor = testStatus.includes("pass") + ? chalk.green + : testStatus.includes("fail") + ? chalk.red + : chalk.yellow; + if (testSuiteName) { + console.log(testStatusColor(`› ${testSuiteName}`)); + } if (failureMessage) { console.log(failureMessage); } diff --git a/js/src/vitest/reporter.ts b/js/src/vitest/reporter.ts index 7f43187d5..796e717e4 100644 --- a/js/src/vitest/reporter.ts +++ b/js/src/vitest/reporter.ts @@ -10,15 +10,22 @@ class LangSmithEvalReporter extends DefaultReporter { async onFinished(files: RunnerTestFile[], errors: unknown[]) { super.onFinished(files, errors); for (const file of files) { - const testModule = this.ctx.state.getReportedEntity(file) as TestModule; - const tests = [...testModule.children.allTests()].map((test) => { - return { - title: test.name, - status: test.result()?.state ?? "skipped", - duration: Math.round(test.diagnostic()?.duration ?? 0), - }; - }); - await printReporterTable(tests); + for (const task of file.tasks) { + const testModule = this.ctx.state.getReportedEntity(task) as TestModule; + const tests = [...testModule.children.allTests()].map((test) => { + return { + title: test.name, + status: test.result()?.state ?? "skipped", + duration: Math.round(test.diagnostic()?.duration ?? 0), + }; + }); + const result = ["pass", "fail", "skip"].includes( + task.result?.state ?? "" + ) + ? (task.result?.state as "pass" | "fail" | "skip") + : "skip"; + await printReporterTable(task.name, tests, result); + } } } } From fbfc8bbd191ea95e217d5bb31e11af25c94be7dc Mon Sep 17 00:00:00 2001 From: Ryan Echols Date: Fri, 31 Jan 2025 10:18:00 -0700 Subject: [PATCH 5/5] chore: update readme info regarding pytest plugin --- python/README.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/python/README.md b/python/README.md index 90a4676c6..3076e168f 100644 --- a/python/README.md +++ b/python/README.md @@ -422,14 +422,7 @@ my_function("Jason is 25 years old") The LangSmith pytest plugin lets Python developers define their datasets and evaluations as pytest test cases. See [online docs](https://docs.smith.langchain.com/evaluation/how_to_guides/pytest) for more information. -This plugin is installed as part of the LangSmith SDK, but is not enabled by default. -To enable the plugin for your project, add `"langsmith.pytest_plugin"` to the `pytest_plugins` list in your root `conftest.py` file: - -```python -# root `conftest.py` file -pytest_plugins = ["langsmith.pytest_plugin"] -``` - +This plugin is installed as part of the LangSmith SDK, and is enabled by default. See also official pytest docs: [How to install and use plugins](https://docs.pytest.org/en/stable/how-to/plugins.html) ## Additional Documentation