feat: add runtime configuration support in execution

.changeset/warm-tigers-grab.md

@@ -0,0 +1,7 @@
+---
+"@empiricalrun/cli": patch
+"@empiricalrun/core": patch
+"@empiricalrun/types": patch
+---
+
+feat: add support for runtime config options

packages/cli/src/bin/index.ts

@@ -21,6 +21,7 @@ import {
   RunConfig,
   RunCompletion,
   RunStatsUpdate,
+  RuntimeOptions,
 } from "@empiricalrun/types";
 import {
   failedOutputsSummary,
@@ -45,6 +46,7 @@ const config = getDefaultRunsConfig(DefaultRunsConfigType.DEFAULT);
 const outputFileName = "output.json";
 const cacheDir = ".empiricalrun";
 const outputFilePath = `${cwd}/${cacheDir}/${outputFileName}`;
+const runtimeOptionsPath = `${cwd}/${cacheDir}/runtime.json`;
 
 program
   .name("Empirical.run CLI")
@@ -75,7 +77,12 @@ program
     "Provide path to .env file to load environment variables",
   )
   .action(async (options) => {
-    dotenv.config({ path: options.envFile || [".env.local", ".env"] });
+    const envFilePath = options.envFile || [".env.local", ".env"];
+    const runTimeOptions: RuntimeOptions = {
+      envFilePath,
+      pythonPath: options.pythonPath,
+    };
+    dotenv.config({ path: runTimeOptions.envFilePath });
     console.log(yellow("Initiating run..."));
 
     let data;
@@ -125,7 +132,6 @@ program
     const completion = await Promise.all(
       runs.map((r) => {
         r.parameters = r.parameters ? r.parameters : {};
-        r.parameters.pythonPath = options.pythonPath;
         return execute(
           r,
           dataset,
@@ -138,6 +144,7 @@ program
             }
           },
           store,
+          runTimeOptions,
         );
       }),
     );
@@ -161,6 +168,7 @@ program
       };
       await fs.mkdir(`${cwd}/${cacheDir}`, { recursive: true });
       await fs.writeFile(outputFilePath, JSON.stringify(data, null, 2));
+      await fs.writeFile(runtimeOptionsPath, JSON.stringify(runTimeOptions));
     } else {
       await reportOnCI(completion, dataset);
     }
@@ -201,6 +209,18 @@ program
         ),
       );
     }
+
+    let runtimeOptions: RuntimeOptions | undefined;
+    try {
+      const dataStr = await fs.readFile(runtimeOptionsPath);
+      runtimeOptions = JSON.parse(dataStr.toString());
+      if (runtimeOptions?.envFilePath) {
+        dotenv.config({ path: runtimeOptions.envFilePath! });
+      }
+    } catch (e) {
+      runtimeOptions = undefined;
+    }
+
     // TODO: get rid of this with dataset id support
     app.use(express.json({ limit: "50mb" }));
     app.use(express.static(path.join(__dirname, "../webapp")));
@@ -274,7 +294,13 @@ program
       const streamUpdate = (obj: any) => res.write(JSON.stringify(obj) + `\n`);
       // This endpoint expects to execute only one run
       let store = persistToFile ? new EmpiricalStore() : undefined;
-      const completion = await execute(runs[0]!, dataset, streamUpdate, store);
+      const completion = await execute(
+        runs[0]!,
+        dataset,
+        streamUpdate,
+        store,
+        runtimeOptions,
+      );
       setRunSummary([completion]);
       const statsUpdate: RunStatsUpdate = {
         type: "run_stats",

packages/core/src/executors/run/index.ts

@@ -8,6 +8,7 @@ import {
   RunMetadataUpdate,
   RunSampleUpdate,
   RunSampleScoreUpdate,
+  RuntimeOptions,
 } from "@empiricalrun/types";
 import { generateHex } from "../../utils";
 import score from "@empiricalrun/scorer";
@@ -23,6 +24,7 @@ export async function execute(
   dataset: Dataset,
   progressCallback?: (sample: RunUpdateType) => void,
   store?: EmpiricalStore,
+  runtimeOptions?: RuntimeOptions,
 ): Promise<RunCompletion> {
   const runCreationDate = new Date();
   const runId = generateRunId();
@@ -53,7 +55,7 @@ export async function execute(
     if (transform) {
       // if llm error then add to the completion object but if something else throw error and stop the run
       completionsPromises.push(
-        transform(runConfig, datasetSample)
+        transform(runConfig, datasetSample, runtimeOptions)
           .then(({ output, error }) => {
             const data: RunSampleOutput = {
               inputs: datasetSample.inputs,
@@ -85,9 +87,7 @@ export async function execute(
                 sample: datasetSample!,
                 output: sample.output,
                 scorers,
-                options: {
-                  pythonPath: runConfig.parameters?.pythonPath,
-                },
+                options: runtimeOptions,
               });
             }
             sample.scores = scores;

packages/core/src/executors/run/transformers/interface.ts

@@ -1,9 +1,15 @@
-import { DatasetSample, RunConfig, RunOutput } from "@empiricalrun/types";
+import {
+  DatasetSample,
+  RunConfig,
+  RunOutput,
+  RuntimeOptions,
+} from "@empiricalrun/types";
 
 export interface Transformer {
   (
     runConfig: RunConfig,
     sample: DatasetSample,
+    runtimeOptions?: RuntimeOptions,
   ): Promise<{
     output: RunOutput;
     error?: {

packages/core/src/executors/run/transformers/script.ts

@@ -8,7 +8,11 @@ const wrapperScriptDirectory = path.join(__dirname, "..", "..", "..", "python");
 const wrapperScriptFile = "executor_wrapper.py";
 const executionOutputIdentifier = "execution_output:";
 
-export const scriptExecutor: Transformer = async (runConfig, sample) => {
+export const scriptExecutor: Transformer = async (
+  runConfig,
+  sample,
+  runtimeOptions,
+) => {
   let output = { value: "" };
   if (runConfig.type !== "py-script") {
     return {
@@ -42,7 +46,7 @@ export const scriptExecutor: Transformer = async (runConfig, sample) => {
   const runOutput = await new Promise<string[]>((resolve) => {
     let output: string[] = [];
     const shell = new PythonShell(wrapperScriptFile, {
-      pythonPath: runConfig.parameters?.pythonPath || undefined,
+      pythonPath: runtimeOptions?.pythonPath || undefined,
       scriptPath: wrapperScriptDirectory,
       args: pythonArgs,
     });

packages/scorer/src/index.ts

@@ -1,4 +1,10 @@
-import { Scorer, Score, DatasetSample, RunOutput } from "@empiricalrun/types";
+import {
+  Scorer,
+  Score,
+  DatasetSample,
+  RunOutput,
+  RuntimeOptions,
+} from "@empiricalrun/types";
 import { ScorerError, ScorerErrorEnum } from "./error";
 import getScoringFn from "./provider";
 
@@ -11,7 +17,7 @@ export default async function score({
   sample: DatasetSample;
   output: RunOutput;
   scorers: Scorer[] | undefined;
-  options?: object;
+  options?: RuntimeOptions;
 }): Promise<Score[]> {
   if (!scorers) {
     return [];

packages/types/src/index.ts

@@ -222,3 +222,8 @@ export type RunUpdateType =
   | RunSampleUpdate
   | RunSampleScoreUpdate
   | RunStatsUpdate;
+
+export interface RuntimeOptions {
+  envFilePath: string | string[];
+  pythonPath: string;
+}