feat: add support for passthrough script executor config (#74)

Co-authored-by: Arjun Attam <arjunattam@gmail.com>

.changeset/warm-ravens-raise.md

@@ -0,0 +1,7 @@
+---
+"@empiricalrun/types": minor
+"@empiricalrun/core": minor
+"@empiricalrun/cli": patch
+---
+
+feat: add support for passthrough config for script executor

development/schema/schema.json

@@ -131,7 +131,7 @@
                             "path": {
                                 "type": "string"
                             },
-                            "config": {
+                            "parameters": {
                                 "type": "object"
                             },
                             "scorers": {

docs/models/custom.mdx

@@ -10,7 +10,7 @@ does pre or post-processing around the LLM call or chains multiple LLM calls tog
 ## Run configuration
 
 In your config file, set `type` as `py-script` and specify the Python file
-path in the `value` field.
+path in the `path` field.
 
 ```json
 "runs": [
@@ -20,12 +20,17 @@ path in the `value` field.
   }
 ]
 ```
+You can additional pass following properties in run configuration:
+`name`: Name of the custom run.
+`parameters`: Parameters to be passed to the script file to modify its behavior.
+
 
 The Python file is expected to have a method called `execute` with the following
 signature:
 
 - **Arguments**
   - inputs: dict of key-value pairs with [sample inputs](../dataset/basics)
+  - parameters: dict of key-value pairs with the run parameters
 - **Returns**: a dict with
   - output (string): The response from the model/application
   - metadata (dict): Custom key-value pairs that are passed on to the scorer and

examples/basic/empiricalrc.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://assets.empirical.run/config/schema/v1.12.json",
+  "$schema": "https://assets.empirical.run/config/schema/v1.13.json",
   "runs": [
     {
       "type": "model",

examples/chatbot/empiricalrc.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://assets.empirical.run/config/schema/v1.12.json",
+  "$schema": "https://assets.empirical.run/config/schema/v1.13.json",
   "runs": [
     {
       "name": "less context setting",

examples/humaneval/empiricalrc.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://assets.empirical.run/config/schema/v1.12.json",
+  "$schema": "https://assets.empirical.run/config/schema/v1.13.json",
   "runs": [
     {
       "type": "model",

examples/rag/empiricalrc.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://assets.empirical.run/config/schema/v1.12.json",
+  "$schema": "https://assets.empirical.run/config/schema/v1.13.json",
   "runs": [
     {
       "name": "rag script run",

examples/rag/rag.py

@@ -6,7 +6,7 @@
 nest_asyncio.apply()
 
 
-def execute(inputs):
+def execute(inputs, parameters):
     # load documents
     question = inputs["question"]
     reader = SimpleDirectoryReader("./arxiv-papers/", num_files_limit=30)

packages/cli/src/bin/index.ts

@@ -100,9 +100,8 @@ program
     );
     const completion = await Promise.all(
       runs.map((r) => {
-        if (r.type === "py-script") {
-          r.pythonPath = options.pythonPath;
-        }
+        r.parameters = r.parameters ? r.parameters : {};
+        r.parameters.pythonPath = options.pythonPath;
         return execute(r, dataset, () => {
           progressBar.increment();
         });

packages/core/src/python/executor_wrapper.py

@@ -6,5 +6,5 @@
 sys.path.append(sys.argv[1])
 user_module = importlib.import_module(sys.argv[2])
 
-result = user_module.execute(json.loads(sys.argv[3]))
+result = user_module.execute(json.loads(sys.argv[3]), json.loads(sys.argv[4]))
 print("execution_output:", json.dumps(result))

packages/core/src/run/executors/script.ts

@@ -32,12 +32,17 @@ export const scriptExecutor: Executor = async (runConfig, sample) => {
 
   let basePath = path.dirname(scriptPath);
   let moduleName = path.basename(scriptPath).replace(".py", "");
-  let pythonArgs = [basePath, moduleName, JSON.stringify(sample.inputs)];
+  let pythonArgs = [
+    basePath,
+    moduleName,
+    JSON.stringify(sample.inputs),
+    JSON.stringify(runConfig.parameters || {}),
+  ];
 
   const runOutput = await new Promise<string[]>((resolve) => {
     let output: string[] = [];
     const shell = new PythonShell(wrapperScriptFile, {
-      pythonPath: runConfig.pythonPath || undefined,
+      pythonPath: runConfig.parameters?.pythonPath || undefined,
       scriptPath: wrapperScriptDirectory,
       args: pythonArgs,
     });

packages/core/src/run/index.ts

@@ -10,21 +10,21 @@ import score from "@empiricalrun/scorer";
 import { getExecutor } from "./executors";
 
 export async function execute(
-  run: RunConfig,
+  runConfig: RunConfig,
   dataset: Dataset,
   progressCallback?: (sample: RunOutputSample) => void,
 ): Promise<RunCompletion> {
   const runCreationDate = new Date();
   const sampleCompletions: RunOutputSample[] = [];
   const runId = generateHex(4);
-  const { scorers } = run;
+  const { scorers } = runConfig;
   const completionsPromises = [];
   for (const datasetSample of dataset.samples) {
-    const executor = getExecutor(run);
+    const executor = getExecutor(runConfig);
     if (executor) {
       // if llm error then add to the completion object but if something else throw error and stop the run
       completionsPromises.push(
-        executor(run, datasetSample).then(({ output, error }) => {
+        executor(runConfig, datasetSample).then(({ output, error }) => {
           const sample: RunOutputSample = {
             inputs: datasetSample.inputs,
             output,
@@ -65,7 +65,7 @@ export async function execute(
             output: sampleCompletion.output,
             scorers,
             options: {
-              pythonPath: run.type === "py-script" ? run.pythonPath : undefined,
+              pythonPath: runConfig.parameters?.pythonPath,
             },
           }).then((scores) => {
             sampleCompletion.scores = scores;
@@ -79,7 +79,10 @@ export async function execute(
   }
   return {
     id: runId,
-    run_config: { ...run, name: run.name || getDefaultRunName(run, runId) },
+    run_config: {
+      ...runConfig,
+      name: runConfig.name || getDefaultRunName(runConfig, runId),
+    },
     dataset_config: {
       id: dataset.id,
     },
@@ -88,12 +91,12 @@ export async function execute(
   };
 }
 
-function getDefaultRunName(run: RunConfig, id: string): string {
+function getDefaultRunName(runConfig: RunConfig, id: string): string {
   let name = "";
-  if (run.type === "model") {
-    name = run.model;
-  } else if (run.type === "py-script" || run.type === "js-script") {
-    name = run.path;
+  if (runConfig.type === "model") {
+    name = runConfig.model;
+  } else if (runConfig.type === "py-script" || runConfig.type === "js-script") {
+    name = runConfig.path;
   }
   return `Run #${id}: ${name}`;
 }

packages/types/src/index.ts

@@ -45,6 +45,9 @@ interface RunConfigBase {
   type: string;
   name?: string;
   scorers?: Scorer[];
+  parameters?: {
+    [key: string]: any;
+  };
 }
 
 export interface ModelRunConfig extends RunConfigBase {
@@ -62,7 +65,10 @@ export interface JSScriptRunConfig extends RunConfigBase {
 export interface PyScriptRunConfig extends RunConfigBase {
   type: "py-script";
   path: string;
-  pythonPath?: string;
+  parameters?: {
+    pythonPath?: string;
+    [key: string]: any;
+  };
 }
 
 export type RunConfig = ModelRunConfig | PyScriptRunConfig | JSScriptRunConfig;