feat: add support for passthrough script executor config

.changeset/warm-ravens-raise.md

@@ -0,0 +1,7 @@
+---
+"@empiricalrun/types": minor
+"@empiricalrun/core": minor
+"@empiricalrun/cli": patch
+---
+
+feat: add support for passthrough config for script executor

examples/basic/empiricalrc.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://assets.empirical.run/config/schema/v1.12.json",
+  "$schema": "https://assets.empirical.run/config/schema/v1.13.json",
   "runs": [
     {
       "type": "model",

examples/chatbot/empiricalrc.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://assets.empirical.run/config/schema/v1.12.json",
+  "$schema": "https://assets.empirical.run/config/schema/v1.13.json",
   "runs": [
     {
       "name": "less context setting",

examples/humaneval/empiricalrc.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://assets.empirical.run/config/schema/v1.12.json",
+  "$schema": "https://assets.empirical.run/config/schema/v1.13.json",
   "runs": [
     {
       "type": "model",

examples/rag/empiricalrc.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://assets.empirical.run/config/schema/v1.12.json",
+  "$schema": "https://assets.empirical.run/config/schema/v1.13.json",
   "runs": [
     {
       "name": "rag script run",

examples/rag/rag.py

@@ -6,7 +6,7 @@
 nest_asyncio.apply()
 
 
-def execute(inputs):
+def execute(inputs, config):
     # load documents
     question = inputs["question"]
     reader = SimpleDirectoryReader("./arxiv-papers/", num_files_limit=30)

packages/core/src/python/executor_wrapper.py

@@ -6,5 +6,5 @@
 sys.path.append(sys.argv[1])
 user_module = importlib.import_module(sys.argv[2])
 
-result = user_module.execute(json.loads(sys.argv[3]))
+result = user_module.execute(json.loads(sys.argv[3]), json.loads(sys.argv[4]))
 print("execution_output:", json.dumps(result))

packages/core/src/run/executors/script.ts

@@ -32,7 +32,12 @@ export const scriptExecutor: Executor = async (runConfig, sample) => {
 
   let basePath = path.dirname(scriptPath);
   let moduleName = path.basename(scriptPath).replace(".py", "");
-  let pythonArgs = [basePath, moduleName, JSON.stringify(sample.inputs)];
+  let pythonArgs = [
+    basePath,
+    moduleName,
+    JSON.stringify(sample.inputs),
+    JSON.stringify(runConfig.config || {}),
+  ];
 
   const runOutput = await new Promise<string[]>((resolve) => {
     let output: string[] = [];

packages/types/src/index.ts

@@ -57,12 +57,14 @@ export interface ModelRunConfig extends RunConfigBase {
 export interface JSScriptRunConfig extends RunConfigBase {
   type: "js-script";
   path: string;
+  config?: any;
 }
 
 export interface PyScriptRunConfig extends RunConfigBase {
   type: "py-script";
   path: string;
   pythonPath?: string;
+  config?: any;
 }
 
 export type RunConfig = ModelRunConfig | PyScriptRunConfig | JSScriptRunConfig;