fix: js script scorer interface

empirical-run · May 21, 2024 · b0b5f23 · b0b5f23
1 parent 7f56d50
commit b0b5f23
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 20 deletions.
diff --git a/packages/scorer/src/provider/deterministic/js-script.ts b/packages/scorer/src/provider/deterministic/js-script.ts
@@ -0,0 +1,26 @@
+import {
+  DatasetSample,
+  JSScriptScorer,
+  RunOutput,
+  Score,
+} from "@empiricalrun/types";
+
+export default async function scoreWithJSScript({
+  output,
+  config,
+  sample,
+}: {
+  output: RunOutput;
+  config: JSScriptScorer;
+  sample: DatasetSample;
+}): Promise<Score[]> {
+  let score = config({ output, config, sample });
+  if ("then" in score) {
+    score = await score;
+  }
+  if (!Array.isArray(score)) {
+    return [score];
+  } else {
+    return score;
+  }
+}
diff --git a/packages/scorer/src/provider/deterministic/sql.test.ts b/packages/scorer/src/provider/deterministic/sql.test.ts
@@ -4,13 +4,9 @@ import { checkSqlSyntax } from "./sql";
 test("sql-syntax works with backticks", async () => {
   expect(
     await checkSqlSyntax({
-      sample: { id: "1", inputs: {} },
       output: {
         value: "SELECT `column` FROM `table`",
       },
-      config: {
-        type: "sql-syntax",
-      },
     }),
   ).toStrictEqual([
     {
@@ -24,13 +20,9 @@ test("sql-syntax works with backticks", async () => {
 test("sql-syntax fails with empty output", async () => {
   expect(
     await checkSqlSyntax({
-      sample: { id: "1", inputs: {} },
       output: {
         value: "",
       },
-      config: {
-        type: "sql-syntax",
-      },
     }),
   ).toStrictEqual([
     {
@@ -44,11 +36,7 @@ test("sql-syntax fails with empty output", async () => {
 test("sql-syntax works with markdown", async () => {
   expect(
     await checkSqlSyntax({
-      sample: { id: "1", inputs: {} },
       output: { value: "```SELECT table * FROM" },
-      config: {
-        type: "sql-syntax",
-      },
     }),
   ).toStrictEqual([
     {
@@ -73,13 +61,9 @@ WHERE stadium_id = (
 );`;
   expect(
     await checkSqlSyntax({
-      sample: { id: "1", inputs: {} },
       output: {
         value: query,
       },
-      config: {
-        type: "sql-syntax",
-      },
     }),
   ).toStrictEqual([
     {

diff --git a/packages/scorer/src/provider/index.ts b/packages/scorer/src/provider/index.ts
@@ -4,6 +4,7 @@ import { scoreWithPythonScript } from "./deterministic/py-script";
 import { checkSqlSyntax } from "./deterministic/sql";
 import { checkLlmCriteria } from "./model-graded/llm";
 import { ScorerError, ScorerErrorEnum } from "../error";
+import scoreWithJSScript from "./deterministic/js-script";
 
 function buildErrorMessage(config: Scorer): string {
   let recommendation: string =
@@ -46,6 +47,13 @@ export default async function scoreUsingConfig(
       return checkSqlSyntax({ output });
     }
   }
+  if (typeof config === "function") {
+    return scoreWithJSScript({
+      sample,
+      output,
+      config
+    })
+  }
   throw new ScorerError(
     ScorerErrorEnum.INCORRECT_PARAMETERS,
     buildErrorMessage(config),

diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts
@@ -46,11 +46,15 @@ export interface ScoringInputBase {
   options?: any;
 }
 
-export interface ScoringFn {
-  (args: ScoringInputBase): Promise<Score[]>;
+export interface AsyncScoringFn {
+  (args: ScoringInputBase): Promise<Score[] | Score>;
 }
 
-export interface JSScriptScorer extends ScoringFn {}
+export interface SyncScoringFn {
+  (args: ScoringInputBase): Score[] | Score;
+}
+
+export type JSScriptScorer = SyncScoringFn | AsyncScoringFn;
 
 export type Scorer = LLMScorer | SyntaxScorer | ScriptScorer | JSScriptScorer;
 
@@ -236,7 +240,7 @@ export type RunSampleOutput = {
 export type Score = {
   score: number;
   name: string;
-  message: string;
+  message?: string;
 };
 
 export interface RunMetadataUpdate {