From d45cc706ca94c439d9fee7013a881e961dd37c5f Mon Sep 17 00:00:00 2001 From: Saikat Mitra Date: Fri, 17 May 2024 10:30:35 +0530 Subject: [PATCH] feat: add js and ts config support (#218) --- .changeset/ten-cheetahs-mate.md | 9 + .eslintrc.js | 6 + apps/web/next.config.js | 2 +- examples/using-js-config/empiricalrc.js | 30 ++ examples/using-js-config/package.json | 18 ++ .../ai/src/providers/fireworks/index.test.ts | 2 +- packages/cli/.npmignore | 6 - packages/cli/CHANGELOG.md | 289 ------------------ packages/cli/src/types/index.ts | 8 - packages/cli/tsconfig.lint.json | 8 - packages/core/.eslintrc.js | 11 - packages/core/CHANGELOG.md | 230 -------------- packages/core/README.md | 1 - packages/core/package.json | 35 --- packages/core/tsconfig.json | 9 - packages/{cli => empiricalrun}/.eslintrc.js | 0 packages/{core => empiricalrun}/.npmignore | 0 packages/{cli => empiricalrun}/README.md | 6 +- packages/{cli => empiricalrun}/package.json | 20 +- .../{core => empiricalrun}/scripts/load.js | 0 .../{core => empiricalrun}/scripts/revert.js | 0 packages/empiricalrun/src/bin/config.ts | 28 ++ .../src/bin/dataset/dataset.test.ts | 0 .../src/bin/dataset/index.ts | 2 +- .../src/bin/dataset/loaders.ts | 2 +- .../src/bin}/error/index.ts | 0 .../{cli => empiricalrun}/src/bin/index.ts | 44 +-- .../src/bin/loaders/file/index.ts | 31 ++ .../src/bin/loaders/file/interface.ts | 4 + .../empiricalrun/src/bin/loaders/file/js.ts | 17 ++ .../empiricalrun/src/bin/loaders/file/json.ts | 18 ++ .../empiricalrun/src/bin/loaders/file/ts.ts | 11 + .../src/bin/logger/cli-logger.ts | 0 .../src/bin}/reporters/ci.ts | 0 .../src/bin}/runs/config/defaults/index.ts | 4 +- .../src/bin}/runs/config/index.ts | 8 +- .../src/bin}/runs/index.ts | 0 .../src/bin}/stats/index.test.ts | 0 .../src/bin}/stats/index.ts | 2 +- .../{core => empiricalrun}/src/db/index.ts | 0 .../src/executors/index.ts | 0 .../src/executors/run/index.ts | 0 .../executors/run/transformers/assistant.ts | 0 .../src/executors/run/transformers/index.ts | 0 .../executors/run/transformers/interface.ts | 0 .../executors/run/transformers/model.test.ts | 0 .../src/executors/run/transformers/model.ts | 0 .../src/executors/run/transformers/script.ts | 0 packages/{core => empiricalrun}/src/index.ts | 1 + .../src/python/executor_wrapper.py | 0 .../{core => empiricalrun}/src/stats/index.ts | 0 .../src/store/dataset-metadata/local.ts | 0 .../src/store/dataset/local.ts | 0 .../{core => empiricalrun}/src/store/index.ts | 0 .../{core => empiricalrun}/src/store/local.ts | 0 .../src/store/run-metadata/index.ts | 0 .../src/store/run/index.ts | 0 .../src/store/user/local.ts | 0 .../src/telemetry/constants.ts | 0 .../src/telemetry/index.ts | 0 .../{core => empiricalrun}/src/utils/index.ts | 0 packages/{cli => empiricalrun}/tsconfig.json | 0 .../{core => empiricalrun}/tsconfig.lint.json | 0 packages/json-schema/src/index.ts | 8 +- packages/types/src/index.ts | 7 + packages/typescript-config/base.json | 4 +- pnpm-lock.yaml | 120 +++----- tsconfig.json | 3 +- 68 files changed, 268 insertions(+), 736 deletions(-) create mode 100644 .changeset/ten-cheetahs-mate.md create mode 100644 examples/using-js-config/empiricalrc.js create mode 100644 examples/using-js-config/package.json delete mode 100644 packages/cli/.npmignore delete mode 100644 packages/cli/CHANGELOG.md delete mode 100644 packages/cli/src/types/index.ts delete mode 100644 packages/cli/tsconfig.lint.json delete mode 100644 packages/core/.eslintrc.js delete mode 100644 packages/core/CHANGELOG.md delete mode 100644 packages/core/README.md delete mode 100644 packages/core/package.json delete mode 100644 packages/core/tsconfig.json rename packages/{cli => empiricalrun}/.eslintrc.js (100%) rename packages/{core => empiricalrun}/.npmignore (100%) rename packages/{cli => empiricalrun}/README.md (96%) rename packages/{cli => empiricalrun}/package.json (69%) rename packages/{core => empiricalrun}/scripts/load.js (100%) rename packages/{core => empiricalrun}/scripts/revert.js (100%) create mode 100644 packages/empiricalrun/src/bin/config.ts rename packages/{cli => empiricalrun}/src/bin/dataset/dataset.test.ts (100%) rename packages/{cli => empiricalrun}/src/bin/dataset/index.ts (98%) rename packages/{cli => empiricalrun}/src/bin/dataset/loaders.ts (97%) rename packages/{cli/src => empiricalrun/src/bin}/error/index.ts (100%) rename packages/{cli => empiricalrun}/src/bin/index.ts (90%) create mode 100644 packages/empiricalrun/src/bin/loaders/file/index.ts create mode 100644 packages/empiricalrun/src/bin/loaders/file/interface.ts create mode 100644 packages/empiricalrun/src/bin/loaders/file/js.ts create mode 100644 packages/empiricalrun/src/bin/loaders/file/json.ts create mode 100644 packages/empiricalrun/src/bin/loaders/file/ts.ts rename packages/{cli => empiricalrun}/src/bin/logger/cli-logger.ts (100%) rename packages/{cli/src => empiricalrun/src/bin}/reporters/ci.ts (100%) rename packages/{cli/src => empiricalrun/src/bin}/runs/config/defaults/index.ts (92%) rename packages/{cli/src => empiricalrun/src/bin}/runs/config/index.ts (62%) rename packages/{cli/src => empiricalrun/src/bin}/runs/index.ts (100%) rename packages/{cli/src => empiricalrun/src/bin}/stats/index.test.ts (100%) rename packages/{cli/src => empiricalrun/src/bin}/stats/index.ts (98%) rename packages/{core => empiricalrun}/src/db/index.ts (100%) rename packages/{core => empiricalrun}/src/executors/index.ts (100%) rename packages/{core => empiricalrun}/src/executors/run/index.ts (100%) rename packages/{core => empiricalrun}/src/executors/run/transformers/assistant.ts (100%) rename packages/{core => empiricalrun}/src/executors/run/transformers/index.ts (100%) rename packages/{core => empiricalrun}/src/executors/run/transformers/interface.ts (100%) rename packages/{core => empiricalrun}/src/executors/run/transformers/model.test.ts (100%) rename packages/{core => empiricalrun}/src/executors/run/transformers/model.ts (100%) rename packages/{core => empiricalrun}/src/executors/run/transformers/script.ts (100%) rename packages/{core => empiricalrun}/src/index.ts (77%) rename packages/{core => empiricalrun}/src/python/executor_wrapper.py (100%) rename packages/{core => empiricalrun}/src/stats/index.ts (100%) rename packages/{core => empiricalrun}/src/store/dataset-metadata/local.ts (100%) rename packages/{core => empiricalrun}/src/store/dataset/local.ts (100%) rename packages/{core => empiricalrun}/src/store/index.ts (100%) rename packages/{core => empiricalrun}/src/store/local.ts (100%) rename packages/{core => empiricalrun}/src/store/run-metadata/index.ts (100%) rename packages/{core => empiricalrun}/src/store/run/index.ts (100%) rename packages/{core => empiricalrun}/src/store/user/local.ts (100%) rename packages/{core => empiricalrun}/src/telemetry/constants.ts (100%) rename packages/{core => empiricalrun}/src/telemetry/index.ts (100%) rename packages/{core => empiricalrun}/src/utils/index.ts (100%) rename packages/{cli => empiricalrun}/tsconfig.json (100%) rename packages/{core => empiricalrun}/tsconfig.lint.json (100%) diff --git a/.changeset/ten-cheetahs-mate.md b/.changeset/ten-cheetahs-mate.md new file mode 100644 index 00000000..f9e5996d --- /dev/null +++ b/.changeset/ten-cheetahs-mate.md @@ -0,0 +1,9 @@ +--- +"empiricalrun": minor +"@empiricalrun/json-schema": minor +"@empiricalrun/types": minor +"@empiricalrun/typescript-config": patch +"web": patch +--- + +feat: add support for js and ts config diff --git a/.eslintrc.js b/.eslintrc.js index 72b39653..6f76e458 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -3,6 +3,12 @@ module.exports = { ignorePatterns: ["apps/**", "packages/**"], extends: ["@empiricalrun/eslint-config/library.js"], + ignorePatterns: [ + // Ignore dotfiles + ".*.js?(x)", + "node_modules/", + "examples/" + ], parser: "@typescript-eslint/parser", parserOptions: { project: true, diff --git a/apps/web/next.config.js b/apps/web/next.config.js index e6b1794e..97d3f7ba 100644 --- a/apps/web/next.config.js +++ b/apps/web/next.config.js @@ -1,7 +1,7 @@ /** @type {import('next').NextConfig} */ module.exports = { output: "export", - distDir: "../../packages/cli/dist/webapp", + distDir: "../../packages/empiricalrun/dist/webapp", productionBrowserSourceMaps: true, optimizeFonts: false, }; diff --git a/examples/using-js-config/empiricalrc.js b/examples/using-js-config/empiricalrc.js new file mode 100644 index 00000000..04cabf34 --- /dev/null +++ b/examples/using-js-config/empiricalrc.js @@ -0,0 +1,30 @@ +/** @type {import("empiricalrun").Config} */ +export default { + runs: [ + { + type: "model", + provider: "openai", + model: "gpt-3.5-turbo", + prompt: "Extract the name, age and location from the message, and respond with a JSON object. If an entity is missing, respond with null.\n\nMessage: {{user_message}}" + } + ], + dataset: { + samples: [ + { + inputs: { + user_message: "Hi my name is John Doe. I'm 26 years old and I work in real estate." + } + }, + { + inputs: { + user_message: "This is Alice. I am a nurse from Maryland. I was born in 1990." + } + } + ] + }, + scorers: [ + { + type: "json-syntax" + } + ] +}; \ No newline at end of file diff --git a/examples/using-js-config/package.json b/examples/using-js-config/package.json new file mode 100644 index 00000000..688b38d2 --- /dev/null +++ b/examples/using-js-config/package.json @@ -0,0 +1,18 @@ +{ + "name": "using-js-config", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@empiricalrun/types": "workspace:^" + }, + "dependencies": { + "empiricalrun": "workspace:^" + } +} diff --git a/packages/ai/src/providers/fireworks/index.test.ts b/packages/ai/src/providers/fireworks/index.test.ts index a6cda968..f793bfe4 100644 --- a/packages/ai/src/providers/fireworks/index.test.ts +++ b/packages/ai/src/providers/fireworks/index.test.ts @@ -9,7 +9,7 @@ describe("Fireworks provider tool calls", () => { { role: "user", content: - "Extract data from message: Hi my name is John Doe. I'm 26 years old and I work in real estate.", + "Extract name from the following message: \n\n Message: Hi my name is John Doe. I'm 26 years old and I work in real estate.", }, ], tools: [ diff --git a/packages/cli/.npmignore b/packages/cli/.npmignore deleted file mode 100644 index 97846de7..00000000 --- a/packages/cli/.npmignore +++ /dev/null @@ -1,6 +0,0 @@ -.turbo -src -node_modules -tsconfig*.json -.eslintrc.js -*.js.map \ No newline at end of file diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md deleted file mode 100644 index e3bdf578..00000000 --- a/packages/cli/CHANGELOG.md +++ /dev/null @@ -1,289 +0,0 @@ -# @empiricalrun/cli - -## 0.12.1 - -### Patch Changes - -- a9c04d8: fix: google ai provider throwing 503 service unavailable - - @empiricalrun/core@0.8.2 - -## 0.12.0 - -### Minor Changes - -- 740a844: feat: add support for merging inputs and add multi-turn chat example - -### Patch Changes - -- Updated dependencies [952baa1] -- Updated dependencies [740a844] - - @empiricalrun/types@0.9.0 - - @empiricalrun/core@0.8.1 - -## 0.11.0 - -### Minor Changes - -- 973c9b2: feat: add support for tool calls - -### Patch Changes - -- 73e35c5: feat: support editing of dataset sample in UI -- 9fae264: fix: scorer row missing in cli summary if scorer was not configured for all runs -- Updated dependencies [973c9b2] - - @empiricalrun/types@0.8.0 - - @empiricalrun/core@0.8.0 - -## 0.10.0 - -### Minor Changes - -- 93e12e0: feat: add support for assistant tool calls - -### Patch Changes - -- 2b03d24: feat: dedicated edit UI for assistant instructions -- bde6bf0: feat: add support for configuring assistant tools -- Updated dependencies [2b03d24] -- Updated dependencies [bde6bf0] -- Updated dependencies [93e12e0] - - @empiricalrun/core@0.7.0 - - @empiricalrun/types@0.7.0 - -## 0.9.1 - -### Patch Changes - -- 2b0cc32: feat: add support for assistant run config defaults -- 51f2f52: fix: add retry logs for model api calls -- 5f043b6: feat: csv loader should parse numbers -- Updated dependencies [c7e5a4a] -- Updated dependencies [2b0cc32] -- Updated dependencies [51f2f52] - - @empiricalrun/types@0.6.1 - - @empiricalrun/core@0.6.1 - -## 0.9.0 - -### Minor Changes - -- 2517c74: feat: add support for openai assistants -- db945c2: feat: ability to add global scorers - -### Patch Changes - -- 3ed659b: feat: add telemetry for usage analytics -- bc3aa56: fix: py-script getting timed out -- 215c54a: chore: change scorer names to json-syntax and llm-critic -- Updated dependencies [3ed659b] -- Updated dependencies [2517c74] -- Updated dependencies [bc3aa56] -- Updated dependencies [215c54a] - - @empiricalrun/core@0.6.0 - - @empiricalrun/types@0.6.0 - -## 0.8.0 - -### Minor Changes - -- ba98ebb: feat: add support for chat format prompt - -### Patch Changes - -- d81cff4: fix: error messaging in case of request timeout -- 4150b1f: fix: typo in chat prompt type -- Updated dependencies [ba98ebb] -- Updated dependencies [d81cff4] -- Updated dependencies [4150b1f] - - @empiricalrun/types@0.5.1 - - @empiricalrun/core@0.5.6 - - @empiricalrun/fetch@0.3.1 - -## 0.7.0 - -### Minor Changes - -- 7a67267: feat: add azure-openai model provider - -### Patch Changes - -- Updated dependencies [7a67267] - - @empiricalrun/fetch@0.3.0 - - @empiricalrun/types@0.5.0 - - @empiricalrun/core@0.5.5 - -## 0.6.0 - -### Minor Changes - -- 65eec6a: feat: get aggregate latency and token numbers for the run - -### Patch Changes - -- Updated dependencies [65eec6a] - - @empiricalrun/types@0.4.0 - - @empiricalrun/core@0.5.4 - -## 0.5.6 - -### Patch Changes - -- 837528d: feat: add support for sharing results using empirical link - -## 0.5.5 - -### Patch Changes - -- @empiricalrun/core@0.5.3 - -## 0.5.4 - -### Patch Changes - -- ebdc566: fix: ui breaking in case of too many sample inputs - -## 0.5.3 - -### Patch Changes - -- d62108b: feat: add cachedir to gitignore on init -- f51254f: feat: add support for runtime config options -- Updated dependencies [f51254f] - - @empiricalrun/core@0.5.2 - - @empiricalrun/types@0.3.3 - -## 0.5.2 - -### Patch Changes - -- 709f1f5: fix: cli not package not able to resolve fetch package -- Updated dependencies [709f1f5] - - @empiricalrun/fetch@0.2.0 - -## 0.5.1 - -### Patch Changes - -- 82739a9: feat: add fetch library which supports retry and timeouts -- a4805df: fix: duck db table error creating more than 1 run from web app -- Updated dependencies [82739a9] -- Updated dependencies [a4805df] - - @empiricalrun/fetch@0.1.0 - - @empiricalrun/core@0.5.1 - -## 0.5.0 - -### Minor Changes - -- d087119: feat: add support for dataset using csv and google sheet -- a8bc03a: feat: add support for duckdb for filtering and score summary - -### Patch Changes - -- 036c015: fix: update the tabs UI to a more minimal design -- 8ffc64e: feat: add or remove samples and run them on the comparison ui -- d3a24a6: feat: update scores UI to accomodate score message -- 2bc5465: fix: minor improvements for execution accuracy in spider example -- Updated dependencies [8ffc64e] -- Updated dependencies [9822db6] -- Updated dependencies [dd7af1f] -- Updated dependencies [a8bc03a] - - @empiricalrun/types@0.3.2 - - @empiricalrun/core@0.5.0 - -## 0.4.1 - -### Patch Changes - -- 0964efd: fix: added scorer progress bar and fix cli progress bar ui -- 3558afb: fix: python script logs are unreadable and breaks cli logs ui -- Updated dependencies [a94aa16] - - @empiricalrun/core@0.4.0 - - @empiricalrun/types@0.3.1 - -## 0.4.0 - -### Minor Changes - -- 879b971: feat: add support for interactivity in the UI - -### Patch Changes - -- 2866cef: feat: add support for showing output metadata on UI -- Updated dependencies [879b971] - - @empiricalrun/types@0.3.0 - - @empiricalrun/core@0.3.0 - -## 0.3.1 - -### Patch Changes - -- c8868d9: fix: top bar in UI is not sticky - -## 0.3.0 - -### Minor Changes - -- df65e07: feat: execution and reporting for github actions - -### Patch Changes - -- b354507: fix: change script value property to path in run -- 5ed6ca9: fix: add support for alternate port for web UI -- 17860b0: feat: add support for passthrough config for script executor -- Updated dependencies [b354507] -- Updated dependencies [90082c8] -- Updated dependencies [17860b0] - - @empiricalrun/types@0.2.0 - - @empiricalrun/core@0.2.0 - -## 0.2.4 - -### Patch Changes - -- f7de133: fix: reduce bundled size of cli package - -## 0.2.3 - -### Patch Changes - -- 571986a: chore: update npm package license -- Updated dependencies [571986a] - - @empiricalrun/types@0.1.1 - - @empiricalrun/core@0.1.3 - -## 0.2.2 - -### Patch Changes - -- @empiricalrun/core@0.1.2 - -## 0.2.1 - -### Patch Changes - -- Updated dependencies [41e3e94] - - @empiricalrun/core@0.1.1 - -## 0.2.0 - -### Minor Changes - -- 844981c: feat: cli can use env vars from dotenv files in cwd or custom path - -## 0.1.0 - -### Minor Changes - -- 8787bf0: feat: add run summary in cli -- dbb2abb: first version of empirical package -- 234a5fd: feat: dataset loaders and auto-fill for missing sample ids - -### Patch Changes - -- 9aa825e: fix: update default config for init command -- Updated dependencies [8787bf0] -- Updated dependencies [dbb2abb] -- Updated dependencies [234a5fd] - - @empiricalrun/types@0.1.0 - - @empiricalrun/core@0.1.0 diff --git a/packages/cli/src/types/index.ts b/packages/cli/src/types/index.ts deleted file mode 100644 index fc1f7f7b..00000000 --- a/packages/cli/src/types/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { RunConfig, DatasetConfig, Scorer } from "@empiricalrun/types"; - -export type RunsConfig = { - $schema?: string; - runs: RunConfig[]; - dataset: DatasetConfig; - scorers?: Scorer[]; -}; diff --git a/packages/cli/tsconfig.lint.json b/packages/cli/tsconfig.lint.json deleted file mode 100644 index 398a613b..00000000 --- a/packages/cli/tsconfig.lint.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "@empiricalrun/typescript-config/base.json", - "compilerOptions": { - "outDir": "dist" - }, - "include": ["src"], - "exclude": ["node_modules", "dist"] -} diff --git a/packages/core/.eslintrc.js b/packages/core/.eslintrc.js deleted file mode 100644 index 053927f1..00000000 --- a/packages/core/.eslintrc.js +++ /dev/null @@ -1,11 +0,0 @@ -// This configuration only applies to the package manager root. -/** @type {import("eslint").Linter.Config} */ -module.exports = { - root: true, - extends: ["@empiricalrun/eslint-config/library.js"], - parser: "@typescript-eslint/parser", - parserOptions: { - project: "./tsconfig.lint.json", - tsconfigRootDir: __dirname, - }, -}; diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md deleted file mode 100644 index 330fb345..00000000 --- a/packages/core/CHANGELOG.md +++ /dev/null @@ -1,230 +0,0 @@ -# @empiricalrun/core - -## 0.8.2 - -### Patch Changes - -- Updated dependencies [a9c04d8] -- Updated dependencies [b379e7f] - - @empiricalrun/ai@0.10.1 - - @empiricalrun/scorer@0.3.1 - -## 0.8.1 - -### Patch Changes - -- Updated dependencies [740a844] - - @empiricalrun/scorer@0.3.0 - - @empiricalrun/ai@0.10.0 - -## 0.8.0 - -### Minor Changes - -- 973c9b2: feat: add support for tool calls - -### Patch Changes - -- Updated dependencies [973c9b2] -- Updated dependencies [f3b632e] -- Updated dependencies [0b2d1c0] - - @empiricalrun/ai@0.10.0 - - @empiricalrun/scorer@0.2.3 - -## 0.7.0 - -### Minor Changes - -- bde6bf0: feat: add support for configuring assistant tools -- 93e12e0: feat: add support for assistant tool calls - -### Patch Changes - -- 2b03d24: feat: dedicated edit UI for assistant instructions -- Updated dependencies [d6d8b5c] -- Updated dependencies [bde6bf0] -- Updated dependencies [93e12e0] - - @empiricalrun/ai@0.9.0 - - @empiricalrun/scorer@0.2.2 - -## 0.6.1 - -### Patch Changes - -- 2b0cc32: feat: add support for assistant run config defaults -- 51f2f52: fix: add retry logs for model api calls -- Updated dependencies [c7e5a4a] -- Updated dependencies [2b0cc32] -- Updated dependencies [9f15c2d] -- Updated dependencies [51f2f52] - - @empiricalrun/scorer@0.2.1 - - @empiricalrun/ai@0.8.0 - -## 0.6.0 - -### Minor Changes - -- 3ed659b: feat: add telemetry for usage analytics -- 2517c74: feat: add support for openai assistants - -### Patch Changes - -- bc3aa56: fix: py-script getting timed out -- Updated dependencies [2517c74] -- Updated dependencies [bc3aa56] -- Updated dependencies [215c54a] -- Updated dependencies [2ad19be] - - @empiricalrun/scorer@0.2.0 - - @empiricalrun/ai@0.7.0 - -## 0.5.6 - -### Patch Changes - -- ba98ebb: feat: add support for chat format prompt -- 4150b1f: fix: typo in chat prompt type -- Updated dependencies [ba98ebb] -- Updated dependencies [d81cff4] -- Updated dependencies [4150b1f] - - @empiricalrun/ai@0.6.1 - - @empiricalrun/scorer@0.1.10 - -## 0.5.5 - -### Patch Changes - -- Updated dependencies [7a67267] - - @empiricalrun/ai@0.6.0 - - @empiricalrun/scorer@0.1.9 - -## 0.5.4 - -### Patch Changes - -- 65eec6a: feat: get aggregate latency and token numbers for the run -- Updated dependencies [66cddb3] - - @empiricalrun/ai@0.5.0 - - @empiricalrun/scorer@0.1.8 - -## 0.5.3 - -### Patch Changes - -- Updated dependencies [ac10032] - - @empiricalrun/ai@0.4.2 - - @empiricalrun/scorer@0.1.7 - -## 0.5.2 - -### Patch Changes - -- f51254f: feat: add support for runtime config options - - @empiricalrun/ai@0.4.1 - - @empiricalrun/scorer@0.1.6 - -## 0.5.1 - -### Patch Changes - -- a4805df: fix: duck db table error creating more than 1 run from web app -- Updated dependencies [a952bd9] - - @empiricalrun/ai@0.4.1 - - @empiricalrun/scorer@0.1.6 - -## 0.5.0 - -### Minor Changes - -- dd7af1f: fix: py-script execute's return type should be consistent with scorer's return type - -### Patch Changes - -- a8bc03a: feat: add support for duckdb for filtering and score summary -- Updated dependencies [e77b76e] -- Updated dependencies [9822db6] -- Updated dependencies [06b1667] -- Updated dependencies [5079335] -- Updated dependencies [bbd5cd0] -- Updated dependencies [b065dad] -- Updated dependencies [2bc5465] - - @empiricalrun/ai@0.4.0 - - @empiricalrun/scorer@0.1.5 - -## 0.4.0 - -### Minor Changes - -- a94aa16: feat: add stop reason and token usage metrics to run output - -### Patch Changes - -- Updated dependencies [a94aa16] - - @empiricalrun/ai@0.3.0 - - @empiricalrun/scorer@0.1.4 - -## 0.3.0 - -### Minor Changes - -- 879b971: feat: add support for interactivity in the UI - -### Patch Changes - -- @empiricalrun/ai@0.2.0 -- @empiricalrun/scorer@0.1.3 - -## 0.2.0 - -### Minor Changes - -- 90082c8: feat: support model parameters with passthrough -- 17860b0: feat: add support for passthrough config for script executor - -### Patch Changes - -- b354507: fix: change script value property to path in run -- Updated dependencies [b354507] -- Updated dependencies [90082c8] -- Updated dependencies [a9e840a] -- Updated dependencies [b3cb527] -- Updated dependencies [d7f4b53] -- Updated dependencies [a4cfb49] - - @empiricalrun/scorer@0.1.3 - - @empiricalrun/ai@0.2.0 - -## 0.1.3 - -### Patch Changes - -- 571986a: chore: update npm package license -- Updated dependencies [571986a] -- Updated dependencies [4a858f5] - - @empiricalrun/scorer@0.1.2 - - @empiricalrun/ai@0.1.1 - -## 0.1.2 - -### Patch Changes - -- Updated dependencies [424f994] - - @empiricalrun/scorer@0.1.1 - -## 0.1.1 - -### Patch Changes - -- 41e3e94: feat: default name for run if not specified - -## 0.1.0 - -### Minor Changes - -- dbb2abb: first version of empirical package - -### Patch Changes - -- 8787bf0: feat: add run summary in cli -- Updated dependencies [8787bf0] -- Updated dependencies [dbb2abb] - - @empiricalrun/scorer@0.1.0 - - @empiricalrun/ai@0.1.0 diff --git a/packages/core/README.md b/packages/core/README.md deleted file mode 100644 index cbe84543..00000000 --- a/packages/core/README.md +++ /dev/null @@ -1 +0,0 @@ -# `core` \ No newline at end of file diff --git a/packages/core/package.json b/packages/core/package.json deleted file mode 100644 index c4052bd4..00000000 --- a/packages/core/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "@empiricalrun/core", - "version": "0.8.2", - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, - "main": "dist/index.js", - "author": "Empirical Team ", - "license": "MIT", - "repository": { - "type": "git", - "url": "https://github.com/empirical-run/empirical.git" - }, - "scripts": { - "build": "node scripts/load.js && tsc --build && cp -r src/python/ dist/python/ && node scripts/revert.js", - "clean": "tsc --build --clean", - "lint": "eslint .", - "test": "vitest run", - "test:watch": "vitest" - }, - "devDependencies": { - "@empiricalrun/types": "workspace:*", - "@empiricalrun/typescript-config": "workspace:*", - "dotenv": "^16.4.5" - }, - "dependencies": { - "@empiricalrun/ai": "workspace:^", - "@empiricalrun/scorer": "workspace:*", - "duckdb-async": "^0.10.0", - "openai": "^4.29.0", - "posthog-node": "^4.0.1", - "python-shell": "^5.0.0" - } -} \ No newline at end of file diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json deleted file mode 100644 index 6f2437b0..00000000 --- a/packages/core/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "@empiricalrun/typescript-config/base.json", - "compilerOptions": { - "outDir": "dist", - "rootDir": "src", - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} diff --git a/packages/cli/.eslintrc.js b/packages/empiricalrun/.eslintrc.js similarity index 100% rename from packages/cli/.eslintrc.js rename to packages/empiricalrun/.eslintrc.js diff --git a/packages/core/.npmignore b/packages/empiricalrun/.npmignore similarity index 100% rename from packages/core/.npmignore rename to packages/empiricalrun/.npmignore diff --git a/packages/cli/README.md b/packages/empiricalrun/README.md similarity index 96% rename from packages/cli/README.md rename to packages/empiricalrun/README.md index 16a47b7e..b14c57ec 100644 --- a/packages/cli/README.md +++ b/packages/empiricalrun/README.md @@ -39,7 +39,7 @@ Our test will succeed if the model outputs valid JSON. 1. Use the CLI to create a sample configuration file called `empiricalrc.json`. ```sh - npx @empiricalrun/cli init + npx empiricalrun init cat empiricalrc.json ``` @@ -48,13 +48,13 @@ Our test will succeed if the model outputs valid JSON. execution will cost $0.0026, based on the selected models. ```sh - npx @empiricalrun/cli run + npx empiricalrun ``` 3. Use the `ui` command to open the reporter web app and see side-by-side results. ```sh - npx @empiricalrun/cli ui + npx empiricalrun ui ``` ### Make it yours diff --git a/packages/cli/package.json b/packages/empiricalrun/package.json similarity index 69% rename from packages/cli/package.json rename to packages/empiricalrun/package.json index 30a89551..bf3d2353 100644 --- a/packages/cli/package.json +++ b/packages/empiricalrun/package.json @@ -1,14 +1,14 @@ { - "name": "@empiricalrun/cli", + "name": "empiricalrun", "version": "0.12.1", - "main": "dist/bin/index.js", "publishConfig": { "registry": "https://registry.npmjs.org/", "access": "public" }, "bin": { - "@empiricalrun/cli": "./dist/bin/index.js" + "empiricalrun": "dist/bin/index.js" }, + "main": "dist/index.js", "author": "Empirical Team ", "license": "MIT", "repository": { @@ -16,13 +16,14 @@ "url": "https://github.com/empirical-run/empirical.git" }, "scripts": { - "build": "tsc --build", + "build": "node scripts/load.js && tsc --build && cp -r src/python/ dist/python/ && node scripts/revert.js", "clean": "tsc --build --clean", "lint": "eslint .", "test": "vitest run", "test:watch": "vitest" }, "devDependencies": { + "@empiricalrun/types": "workspace:*", "@empiricalrun/typescript-config": "workspace:*", "@types/cli-progress": "^3.11.5", "@types/detect-port": "^1.3.5", @@ -30,18 +31,23 @@ "@types/opener": "^1.4.3" }, "dependencies": { - "@empiricalrun/core": "workspace:*", + "@empiricalrun/ai": "workspace:*", "@empiricalrun/fetch": "workspace:*", - "@empiricalrun/types": "workspace:*", + "@empiricalrun/scorer": "workspace:*", "@types/express": "^4.17.21", "cli-progress": "^3.12.0", "commander": "^12.0.0", "csvtojson": "^2.0.10", "detect-port": "^1.5.1", "dotenv": "^16.4.5", + "duckdb-async": "^0.10.0", "express": "^4.19.2", + "openai": "^4.29.0", "opener": "^1.5.2", "picocolors": "^1.0.0", - "table": "^6.8.2" + "posthog-node": "^4.0.1", + "python-shell": "^5.0.0", + "table": "^6.8.2", + "tsx": "^4.10.2" } } \ No newline at end of file diff --git a/packages/core/scripts/load.js b/packages/empiricalrun/scripts/load.js similarity index 100% rename from packages/core/scripts/load.js rename to packages/empiricalrun/scripts/load.js diff --git a/packages/core/scripts/revert.js b/packages/empiricalrun/scripts/revert.js similarity index 100% rename from packages/core/scripts/revert.js rename to packages/empiricalrun/scripts/revert.js diff --git a/packages/empiricalrun/src/bin/config.ts b/packages/empiricalrun/src/bin/config.ts new file mode 100644 index 00000000..28a9accb --- /dev/null +++ b/packages/empiricalrun/src/bin/config.ts @@ -0,0 +1,28 @@ +import { Config } from "@empiricalrun/types"; +import { getFileLoaderForFileName } from "./loaders/file"; +import { + buildErrorLog, + buildSuccessLog, + buildWarningLog, +} from "./logger/cli-logger"; + +export async function readEmpiricalConfig(): Promise { + const cwd = process.cwd(); + const fileName = "empiricalrc"; + try { + const loader = await getFileLoaderForFileName(fileName, cwd); + const config = await loader(); + console.log(buildSuccessLog(`read ${fileName} file successfully`)); + config.runs.forEach((r) => { + // if scorers are not set for a run, then override it with the global scorers + if (!r.scorers && config.scorers) { + r.scorers = config.scorers; + } + }); + return config; + } catch (err: any) { + console.log(buildErrorLog(`Failed to read ${fileName} file`)); + console.log(buildWarningLog("Please ensure running init command first")); + process.exit(1); + } +} diff --git a/packages/cli/src/bin/dataset/dataset.test.ts b/packages/empiricalrun/src/bin/dataset/dataset.test.ts similarity index 100% rename from packages/cli/src/bin/dataset/dataset.test.ts rename to packages/empiricalrun/src/bin/dataset/dataset.test.ts diff --git a/packages/cli/src/bin/dataset/index.ts b/packages/empiricalrun/src/bin/dataset/index.ts similarity index 98% rename from packages/cli/src/bin/dataset/index.ts rename to packages/empiricalrun/src/bin/dataset/index.ts index 51809c4e..6a0fa433 100644 --- a/packages/cli/src/bin/dataset/index.ts +++ b/packages/empiricalrun/src/bin/dataset/index.ts @@ -1,7 +1,7 @@ import { Dataset, DatasetConfig, DatasetSample } from "@empiricalrun/types"; import { promises as fs } from "fs"; import { loaders, hashContents, LoaderType } from "./loaders"; -import { DatasetError, DatasetErrorEnum } from "../../error"; +import { DatasetError, DatasetErrorEnum } from "../error"; import { fetchWithRetry } from "@empiricalrun/fetch"; const googleSheetIdentifier = "https://docs.google.com/spreadsheets/d/"; diff --git a/packages/cli/src/bin/dataset/loaders.ts b/packages/empiricalrun/src/bin/dataset/loaders.ts similarity index 97% rename from packages/cli/src/bin/dataset/loaders.ts rename to packages/empiricalrun/src/bin/dataset/loaders.ts index 947febf3..5e9b99ea 100644 --- a/packages/cli/src/bin/dataset/loaders.ts +++ b/packages/empiricalrun/src/bin/dataset/loaders.ts @@ -1,6 +1,6 @@ import { DatasetSample } from "@empiricalrun/types"; import crypto from "crypto"; -import { DatasetError, DatasetErrorEnum } from "../../error"; +import { DatasetError, DatasetErrorEnum } from "../error"; import csv from "csvtojson"; type LoaderFunction = (contents: string) => Promise; diff --git a/packages/cli/src/error/index.ts b/packages/empiricalrun/src/bin/error/index.ts similarity index 100% rename from packages/cli/src/error/index.ts rename to packages/empiricalrun/src/bin/error/index.ts diff --git a/packages/cli/src/bin/index.ts b/packages/empiricalrun/src/bin/index.ts similarity index 90% rename from packages/cli/src/bin/index.ts rename to packages/empiricalrun/src/bin/index.ts index 9f04cb8e..13c65482 100644 --- a/packages/cli/src/bin/index.ts +++ b/packages/empiricalrun/src/bin/index.ts @@ -11,11 +11,12 @@ import { EmpiricalStore, execute, getLocalDBInstance, -} from "@empiricalrun/core"; -import { RunsConfig } from "../types"; + Telemetry, + runEventProperties, +} from "../index"; import { loadDataset } from "./dataset"; -import { DatasetError } from "../error"; -import { DefaultRunsConfigType, getDefaultRunsConfig } from "../runs"; +import { DatasetError } from "./error"; +import { DefaultRunsConfigType, getDefaultRunsConfig } from "./runs"; import { Dataset, RunConfig, @@ -23,13 +24,12 @@ import { RunStatsUpdate, RuntimeOptions, } from "@empiricalrun/types"; -import { Telemetry, runEventProperties } from "@empiricalrun/core"; import { failedOutputsSummary, printStatsSummary, setRunSummary, -} from "../stats"; -import { reportOnCI } from "../reporters/ci"; +} from "./stats"; +import { reportOnCI } from "./reporters/ci"; import detect from "detect-port"; import { ProgressBar, @@ -38,6 +38,7 @@ import { buildWarningLog, getCliProgressLoggerInstance, } from "./logger/cli-logger"; +import { readEmpiricalConfig } from "./config"; const configFileName = "empiricalrc.json"; const cwd = process.cwd(); @@ -50,31 +51,6 @@ const runtimeOptionsPath = `${cwd}/${cacheDir}/runtime.json`; const telemetry = new Telemetry(); -const readConfig = async (): Promise => { - let data: string; - try { - data = (await fs.readFile(configFileFullPath)).toString(); - console.log(buildSuccessLog(`read ${configFileName} file successfully`)); - } catch (err) { - console.log(buildErrorLog(`Failed to read ${configFileName} file`)); - console.log(yellow("Please ensure running init command first")); - process.exit(1); - } - const { runs, dataset, scorers } = JSON.parse(data) as RunsConfig; - - runs.forEach((r) => { - // if scorers are not set for a run, then override it with the global scorers - if (!r.scorers && scorers) { - r.scorers = scorers; - } - }); - - return { - runs, - dataset, - }; -}; - program .name("Empirical.run CLI") .description( @@ -100,7 +76,6 @@ program }); program - .command("run") .description("initiate a run to evaluate model completions") .option( "-pyp, --python-path ", @@ -121,9 +96,8 @@ program telemetry.logEvent("run.start"); const startTime = performance.now(); - const { runs, dataset: datasetConfig } = await readConfig(); + const { runs, dataset: datasetConfig } = await readEmpiricalConfig(); - // TODO: add check here for empty runs config. Add validator of the file let dataset: Dataset; const store = new EmpiricalStore(); try { diff --git a/packages/empiricalrun/src/bin/loaders/file/index.ts b/packages/empiricalrun/src/bin/loaders/file/index.ts new file mode 100644 index 00000000..31bc9ccf --- /dev/null +++ b/packages/empiricalrun/src/bin/loaders/file/index.ts @@ -0,0 +1,31 @@ +import fs from "fs/promises"; +import path from "path"; +import json from "./json"; +import js from "./js"; +import { FileLoader } from "./interface"; +import ts from "./ts"; + +const loaders: FileLoader[] = [json, js, ts]; + +export async function getFileLoaderForFileName( + name: string, + directory: string, +) { + let matchingLoader: FileLoader | undefined; + for (const loader of loaders) { + try { + const fullPath = path.join(directory, `${name}${loader.type}`); + await fs.access(fullPath, fs.constants.F_OK); + matchingLoader = loader; + break; + } catch (e) { + // + } + } + if (!matchingLoader) { + throw new Error( + `Failed to find ${name} file having either of the extensions ${loaders.map((l) => l.type).join(" ")}`, + ); + } + return matchingLoader.getLoader(name, directory); +} diff --git a/packages/empiricalrun/src/bin/loaders/file/interface.ts b/packages/empiricalrun/src/bin/loaders/file/interface.ts new file mode 100644 index 00000000..0529e832 --- /dev/null +++ b/packages/empiricalrun/src/bin/loaders/file/interface.ts @@ -0,0 +1,4 @@ +export interface FileLoader { + type: string; + getLoader: (file: string, directory: string) => () => Promise; +} diff --git a/packages/empiricalrun/src/bin/loaders/file/js.ts b/packages/empiricalrun/src/bin/loaders/file/js.ts new file mode 100644 index 00000000..c498605c --- /dev/null +++ b/packages/empiricalrun/src/bin/loaders/file/js.ts @@ -0,0 +1,17 @@ +import tsx from "tsx/cjs/api"; + +export default { + type: ".js", + getLoader(file: string, directory: string = "") { + return async function loader(): Promise { + const loaded = await tsx.require(file, `${directory}/${file}`); + // support for ESM modules + if (loaded.default) { + return loaded.default as T; + } else { + // support for commonjs modules + return loaded as T; + } + }; + }, +}; diff --git a/packages/empiricalrun/src/bin/loaders/file/json.ts b/packages/empiricalrun/src/bin/loaders/file/json.ts new file mode 100644 index 00000000..ac99420a --- /dev/null +++ b/packages/empiricalrun/src/bin/loaders/file/json.ts @@ -0,0 +1,18 @@ +import fs from "fs/promises"; + +export default { + type: ".json", + getLoader(file: string, directory: string) { + const fileType = this.type; + return async function loader(): Promise { + try { + const data = ( + await fs.readFile(`${directory}/${file}${fileType}`) + ).toString(); + return JSON.parse(data) as T; + } catch (e: any) { + throw Error(`Failed to parse JSON file: ${e.message}`); + } + }; + }, +}; diff --git a/packages/empiricalrun/src/bin/loaders/file/ts.ts b/packages/empiricalrun/src/bin/loaders/file/ts.ts new file mode 100644 index 00000000..b5793ee3 --- /dev/null +++ b/packages/empiricalrun/src/bin/loaders/file/ts.ts @@ -0,0 +1,11 @@ +import tsx from "tsx/cjs/api"; + +export default { + type: ".ts", + getLoader(file: string, directory: string = "") { + return async function loader(): Promise { + const loaded = await tsx.require(file, `${directory}/${file}`); + return loaded.default as T; + }; + }, +}; diff --git a/packages/cli/src/bin/logger/cli-logger.ts b/packages/empiricalrun/src/bin/logger/cli-logger.ts similarity index 100% rename from packages/cli/src/bin/logger/cli-logger.ts rename to packages/empiricalrun/src/bin/logger/cli-logger.ts diff --git a/packages/cli/src/reporters/ci.ts b/packages/empiricalrun/src/bin/reporters/ci.ts similarity index 100% rename from packages/cli/src/reporters/ci.ts rename to packages/empiricalrun/src/bin/reporters/ci.ts diff --git a/packages/cli/src/runs/config/defaults/index.ts b/packages/empiricalrun/src/bin/runs/config/defaults/index.ts similarity index 92% rename from packages/cli/src/runs/config/defaults/index.ts rename to packages/empiricalrun/src/bin/runs/config/defaults/index.ts index 154d8fe6..8fa9ad96 100644 --- a/packages/cli/src/runs/config/defaults/index.ts +++ b/packages/empiricalrun/src/bin/runs/config/defaults/index.ts @@ -1,6 +1,6 @@ -import { RunsConfig } from "../../../types"; +import { Config } from "@empiricalrun/types"; -export const config: RunsConfig = { +export const config: Config = { $schema: "https://assets.empirical.run/config/schema/latest.json", runs: [ { diff --git a/packages/cli/src/runs/config/index.ts b/packages/empiricalrun/src/bin/runs/config/index.ts similarity index 62% rename from packages/cli/src/runs/config/index.ts rename to packages/empiricalrun/src/bin/runs/config/index.ts index 8f6f67e5..2079c462 100644 --- a/packages/cli/src/runs/config/index.ts +++ b/packages/empiricalrun/src/bin/runs/config/index.ts @@ -1,17 +1,15 @@ -import { RunsConfig } from "../../types"; +import { Config } from "@empiricalrun/types"; import { config as OpenAIRunsConfig } from "./defaults"; export enum DefaultRunsConfigType { DEFAULT = "default", } -const map = new Map([ +const map = new Map([ [DefaultRunsConfigType.DEFAULT, OpenAIRunsConfig], ]); -export const getDefaultRunsConfig = ( - name: DefaultRunsConfigType, -): RunsConfig => { +export const getDefaultRunsConfig = (name: DefaultRunsConfigType): Config => { if (map.has(name)) { return { ...map.get(name)! }; } else { diff --git a/packages/cli/src/runs/index.ts b/packages/empiricalrun/src/bin/runs/index.ts similarity index 100% rename from packages/cli/src/runs/index.ts rename to packages/empiricalrun/src/bin/runs/index.ts diff --git a/packages/cli/src/stats/index.test.ts b/packages/empiricalrun/src/bin/stats/index.test.ts similarity index 100% rename from packages/cli/src/stats/index.test.ts rename to packages/empiricalrun/src/bin/stats/index.test.ts diff --git a/packages/cli/src/stats/index.ts b/packages/empiricalrun/src/bin/stats/index.ts similarity index 98% rename from packages/cli/src/stats/index.ts rename to packages/empiricalrun/src/bin/stats/index.ts index e4bb13a2..2e7dcefe 100644 --- a/packages/cli/src/stats/index.ts +++ b/packages/empiricalrun/src/bin/stats/index.ts @@ -1,4 +1,4 @@ -import { getStatsForRun } from "@empiricalrun/core"; +import { getStatsForRun } from "../../index"; import { RunCompletion } from "@empiricalrun/types"; import { green, yellow, bold, cyan, red } from "picocolors"; import { table } from "table"; diff --git a/packages/core/src/db/index.ts b/packages/empiricalrun/src/db/index.ts similarity index 100% rename from packages/core/src/db/index.ts rename to packages/empiricalrun/src/db/index.ts diff --git a/packages/core/src/executors/index.ts b/packages/empiricalrun/src/executors/index.ts similarity index 100% rename from packages/core/src/executors/index.ts rename to packages/empiricalrun/src/executors/index.ts diff --git a/packages/core/src/executors/run/index.ts b/packages/empiricalrun/src/executors/run/index.ts similarity index 100% rename from packages/core/src/executors/run/index.ts rename to packages/empiricalrun/src/executors/run/index.ts diff --git a/packages/core/src/executors/run/transformers/assistant.ts b/packages/empiricalrun/src/executors/run/transformers/assistant.ts similarity index 100% rename from packages/core/src/executors/run/transformers/assistant.ts rename to packages/empiricalrun/src/executors/run/transformers/assistant.ts diff --git a/packages/core/src/executors/run/transformers/index.ts b/packages/empiricalrun/src/executors/run/transformers/index.ts similarity index 100% rename from packages/core/src/executors/run/transformers/index.ts rename to packages/empiricalrun/src/executors/run/transformers/index.ts diff --git a/packages/core/src/executors/run/transformers/interface.ts b/packages/empiricalrun/src/executors/run/transformers/interface.ts similarity index 100% rename from packages/core/src/executors/run/transformers/interface.ts rename to packages/empiricalrun/src/executors/run/transformers/interface.ts diff --git a/packages/core/src/executors/run/transformers/model.test.ts b/packages/empiricalrun/src/executors/run/transformers/model.test.ts similarity index 100% rename from packages/core/src/executors/run/transformers/model.test.ts rename to packages/empiricalrun/src/executors/run/transformers/model.test.ts diff --git a/packages/core/src/executors/run/transformers/model.ts b/packages/empiricalrun/src/executors/run/transformers/model.ts similarity index 100% rename from packages/core/src/executors/run/transformers/model.ts rename to packages/empiricalrun/src/executors/run/transformers/model.ts diff --git a/packages/core/src/executors/run/transformers/script.ts b/packages/empiricalrun/src/executors/run/transformers/script.ts similarity index 100% rename from packages/core/src/executors/run/transformers/script.ts rename to packages/empiricalrun/src/executors/run/transformers/script.ts diff --git a/packages/core/src/index.ts b/packages/empiricalrun/src/index.ts similarity index 77% rename from packages/core/src/index.ts rename to packages/empiricalrun/src/index.ts index 1732b48b..06d74b26 100644 --- a/packages/core/src/index.ts +++ b/packages/empiricalrun/src/index.ts @@ -3,3 +3,4 @@ export * from "./stats"; export * from "./store"; export * from "./db"; export * from "./telemetry"; +export * from "@empiricalrun/types"; diff --git a/packages/core/src/python/executor_wrapper.py b/packages/empiricalrun/src/python/executor_wrapper.py similarity index 100% rename from packages/core/src/python/executor_wrapper.py rename to packages/empiricalrun/src/python/executor_wrapper.py diff --git a/packages/core/src/stats/index.ts b/packages/empiricalrun/src/stats/index.ts similarity index 100% rename from packages/core/src/stats/index.ts rename to packages/empiricalrun/src/stats/index.ts diff --git a/packages/core/src/store/dataset-metadata/local.ts b/packages/empiricalrun/src/store/dataset-metadata/local.ts similarity index 100% rename from packages/core/src/store/dataset-metadata/local.ts rename to packages/empiricalrun/src/store/dataset-metadata/local.ts diff --git a/packages/core/src/store/dataset/local.ts b/packages/empiricalrun/src/store/dataset/local.ts similarity index 100% rename from packages/core/src/store/dataset/local.ts rename to packages/empiricalrun/src/store/dataset/local.ts diff --git a/packages/core/src/store/index.ts b/packages/empiricalrun/src/store/index.ts similarity index 100% rename from packages/core/src/store/index.ts rename to packages/empiricalrun/src/store/index.ts diff --git a/packages/core/src/store/local.ts b/packages/empiricalrun/src/store/local.ts similarity index 100% rename from packages/core/src/store/local.ts rename to packages/empiricalrun/src/store/local.ts diff --git a/packages/core/src/store/run-metadata/index.ts b/packages/empiricalrun/src/store/run-metadata/index.ts similarity index 100% rename from packages/core/src/store/run-metadata/index.ts rename to packages/empiricalrun/src/store/run-metadata/index.ts diff --git a/packages/core/src/store/run/index.ts b/packages/empiricalrun/src/store/run/index.ts similarity index 100% rename from packages/core/src/store/run/index.ts rename to packages/empiricalrun/src/store/run/index.ts diff --git a/packages/core/src/store/user/local.ts b/packages/empiricalrun/src/store/user/local.ts similarity index 100% rename from packages/core/src/store/user/local.ts rename to packages/empiricalrun/src/store/user/local.ts diff --git a/packages/core/src/telemetry/constants.ts b/packages/empiricalrun/src/telemetry/constants.ts similarity index 100% rename from packages/core/src/telemetry/constants.ts rename to packages/empiricalrun/src/telemetry/constants.ts diff --git a/packages/core/src/telemetry/index.ts b/packages/empiricalrun/src/telemetry/index.ts similarity index 100% rename from packages/core/src/telemetry/index.ts rename to packages/empiricalrun/src/telemetry/index.ts diff --git a/packages/core/src/utils/index.ts b/packages/empiricalrun/src/utils/index.ts similarity index 100% rename from packages/core/src/utils/index.ts rename to packages/empiricalrun/src/utils/index.ts diff --git a/packages/cli/tsconfig.json b/packages/empiricalrun/tsconfig.json similarity index 100% rename from packages/cli/tsconfig.json rename to packages/empiricalrun/tsconfig.json diff --git a/packages/core/tsconfig.lint.json b/packages/empiricalrun/tsconfig.lint.json similarity index 100% rename from packages/core/tsconfig.lint.json rename to packages/empiricalrun/tsconfig.lint.json diff --git a/packages/json-schema/src/index.ts b/packages/json-schema/src/index.ts index c80790e3..2352d78b 100644 --- a/packages/json-schema/src/index.ts +++ b/packages/json-schema/src/index.ts @@ -1,15 +1,15 @@ import { createGenerator, Config } from "ts-json-schema-generator"; import path from "path"; import fs from "fs"; -import packageJson from "./../../cli/package.json"; +import packageJson from "./../../empiricalrun/package.json"; -const tsconfigPath = path.join(__dirname, "./../../cli/tsconfig.json"); -const typesPath = path.join(__dirname, "./../../cli/src/types/index.ts"); +const tsconfigPath = path.join(__dirname, "./../../types/tsconfig.json"); +const typesPath = path.join(__dirname, "./../../types/src/index.ts"); const config: Config = { path: typesPath, tsconfig: tsconfigPath, - type: "RunsConfig", + type: "Config", }; const schema = createGenerator(config).createSchema(config.type); const schemaString = JSON.stringify(schema, null, 2); diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 6b82e15c..fe54bd3d 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -266,3 +266,10 @@ export interface RuntimeOptions { envFilePath: string | string[]; pythonPath: string; } + +export type Config = { + $schema?: string; + runs: RunConfig[]; + dataset: DatasetConfig; + scorers?: Scorer[]; +}; diff --git a/packages/typescript-config/base.json b/packages/typescript-config/base.json index 0f80cfd6..a3c48709 100644 --- a/packages/typescript-config/base.json +++ b/packages/typescript-config/base.json @@ -8,13 +8,13 @@ "incremental": false, "isolatedModules": true, "lib": ["es2022", "DOM", "DOM.Iterable"], - "module": "NodeNext", "moduleDetection": "force", "moduleResolution": "NodeNext", "noUncheckedIndexedAccess": true, "resolveJsonModule": true, "skipLibCheck": true, "strict": true, - "target": "ES2022" + "target": "ES2022", + "module": "NodeNext" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c71d9cd6..63926520 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -169,17 +169,17 @@ importers: specifier: ^1.1.6 version: 1.1.6 - packages/cli: + packages/empiricalrun: dependencies: - '@empiricalrun/core': + '@empiricalrun/ai': specifier: workspace:* - version: link:../core + version: link:../ai '@empiricalrun/fetch': specifier: workspace:* version: link:../fetch - '@empiricalrun/types': + '@empiricalrun/scorer': specifier: workspace:* - version: link:../types + version: link:../scorer '@types/express': specifier: ^4.17.21 version: 4.17.21 @@ -198,19 +198,37 @@ importers: dotenv: specifier: ^16.4.5 version: 16.4.5 + duckdb-async: + specifier: ^0.10.0 + version: 0.10.0 express: specifier: ^4.19.2 version: 4.19.2 + openai: + specifier: ^4.29.0 + version: 4.38.5 opener: specifier: ^1.5.2 version: 1.5.2 picocolors: specifier: ^1.0.0 version: 1.0.0 + posthog-node: + specifier: ^4.0.1 + version: 4.0.1 + python-shell: + specifier: ^5.0.0 + version: 5.0.0 table: specifier: ^6.8.2 version: 6.8.2 + tsx: + specifier: ^4.10.2 + version: 4.10.2 devDependencies: + '@empiricalrun/types': + specifier: workspace:* + version: link:../types '@empiricalrun/typescript-config': specifier: workspace:* version: link:../typescript-config @@ -227,37 +245,6 @@ importers: specifier: ^1.4.3 version: 1.4.3 - packages/core: - dependencies: - '@empiricalrun/ai': - specifier: workspace:^ - version: link:../ai - '@empiricalrun/scorer': - specifier: workspace:* - version: link:../scorer - duckdb-async: - specifier: ^0.10.0 - version: 0.10.0 - openai: - specifier: ^4.29.0 - version: 4.29.0 - posthog-node: - specifier: ^4.0.1 - version: 4.0.1 - python-shell: - specifier: ^5.0.0 - version: 5.0.0 - devDependencies: - '@empiricalrun/types': - specifier: workspace:* - version: link:../types - '@empiricalrun/typescript-config': - specifier: workspace:* - version: link:../typescript-config - dotenv: - specifier: ^16.4.5 - version: 16.4.5 - packages/eslint-config: devDependencies: '@typescript-eslint/eslint-plugin': @@ -784,7 +771,6 @@ packages: cpu: [ppc64] os: [aix] requiresBuild: true - dev: true optional: true /@esbuild/android-arm64@0.20.2: @@ -793,7 +779,6 @@ packages: cpu: [arm64] os: [android] requiresBuild: true - dev: true optional: true /@esbuild/android-arm@0.20.2: @@ -802,7 +787,6 @@ packages: cpu: [arm] os: [android] requiresBuild: true - dev: true optional: true /@esbuild/android-x64@0.20.2: @@ -811,7 +795,6 @@ packages: cpu: [x64] os: [android] requiresBuild: true - dev: true optional: true /@esbuild/darwin-arm64@0.20.2: @@ -820,7 +803,6 @@ packages: cpu: [arm64] os: [darwin] requiresBuild: true - dev: true optional: true /@esbuild/darwin-x64@0.20.2: @@ -829,7 +811,6 @@ packages: cpu: [x64] os: [darwin] requiresBuild: true - dev: true optional: true /@esbuild/freebsd-arm64@0.20.2: @@ -838,7 +819,6 @@ packages: cpu: [arm64] os: [freebsd] requiresBuild: true - dev: true optional: true /@esbuild/freebsd-x64@0.20.2: @@ -847,7 +827,6 @@ packages: cpu: [x64] os: [freebsd] requiresBuild: true - dev: true optional: true /@esbuild/linux-arm64@0.20.2: @@ -856,7 +835,6 @@ packages: cpu: [arm64] os: [linux] requiresBuild: true - dev: true optional: true /@esbuild/linux-arm@0.20.2: @@ -865,7 +843,6 @@ packages: cpu: [arm] os: [linux] requiresBuild: true - dev: true optional: true /@esbuild/linux-ia32@0.20.2: @@ -874,7 +851,6 @@ packages: cpu: [ia32] os: [linux] requiresBuild: true - dev: true optional: true /@esbuild/linux-loong64@0.20.2: @@ -883,7 +859,6 @@ packages: cpu: [loong64] os: [linux] requiresBuild: true - dev: true optional: true /@esbuild/linux-mips64el@0.20.2: @@ -892,7 +867,6 @@ packages: cpu: [mips64el] os: [linux] requiresBuild: true - dev: true optional: true /@esbuild/linux-ppc64@0.20.2: @@ -901,7 +875,6 @@ packages: cpu: [ppc64] os: [linux] requiresBuild: true - dev: true optional: true /@esbuild/linux-riscv64@0.20.2: @@ -910,7 +883,6 @@ packages: cpu: [riscv64] os: [linux] requiresBuild: true - dev: true optional: true /@esbuild/linux-s390x@0.20.2: @@ -919,7 +891,6 @@ packages: cpu: [s390x] os: [linux] requiresBuild: true - dev: true optional: true /@esbuild/linux-x64@0.20.2: @@ -928,7 +899,6 @@ packages: cpu: [x64] os: [linux] requiresBuild: true - dev: true optional: true /@esbuild/netbsd-x64@0.20.2: @@ -937,7 +907,6 @@ packages: cpu: [x64] os: [netbsd] requiresBuild: true - dev: true optional: true /@esbuild/openbsd-x64@0.20.2: @@ -946,7 +915,6 @@ packages: cpu: [x64] os: [openbsd] requiresBuild: true - dev: true optional: true /@esbuild/sunos-x64@0.20.2: @@ -955,7 +923,6 @@ packages: cpu: [x64] os: [sunos] requiresBuild: true - dev: true optional: true /@esbuild/win32-arm64@0.20.2: @@ -964,7 +931,6 @@ packages: cpu: [arm64] os: [win32] requiresBuild: true - dev: true optional: true /@esbuild/win32-ia32@0.20.2: @@ -973,7 +939,6 @@ packages: cpu: [ia32] os: [win32] requiresBuild: true - dev: true optional: true /@esbuild/win32-x64@0.20.2: @@ -982,7 +947,6 @@ packages: cpu: [x64] os: [win32] requiresBuild: true - dev: true optional: true /@eslint-community/eslint-utils@4.4.0(eslint@8.57.0): @@ -2281,16 +2245,10 @@ packages: resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} dev: true - /@types/node@18.19.24: - resolution: {integrity: sha512-eghAz3gnbQbvnHqB+mgB2ZR3aH6RhdEmHGS48BnV75KceQPHqabkxKI0BbUSsqhqy2Ddhc2xD/VAR9ySZd57Lw==} - dependencies: - undici-types: 5.26.5 - /@types/node@18.19.31: resolution: {integrity: sha512-ArgCD39YpyyrtFKIqMDvjz79jto5fcI/SVUs2HwB+f0dAzq68yqOdyaSivLiLugSziTpNXLQrVb7RZFmdZzbhA==} dependencies: undici-types: 5.26.5 - dev: false /@types/node@20.11.24: resolution: {integrity: sha512-Kza43ewS3xoLgCEpQrsT+xRo/EJej1y0kVYGiLFE1NEODXGzTfwiC6tXTLMQskn1X4/Rjlh0MQUvx9W+L9long==} @@ -3817,6 +3775,7 @@ packages: /dotenv@16.4.5: resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} engines: {node: '>=12'} + dev: false /duckdb-async@0.10.0: resolution: {integrity: sha512-YMDYulx5JLzT01CkqfG813h3ikCF3Pf7nV2xH1OZdVB6q0A6Rofrtf0x1CIotXY33fifgFY1jte3WPzD+3708Q==} @@ -4027,7 +3986,6 @@ packages: '@esbuild/win32-arm64': 0.20.2 '@esbuild/win32-ia32': 0.20.2 '@esbuild/win32-x64': 0.20.2 - dev: true /escalade@3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} @@ -4098,7 +4056,7 @@ packages: eslint-module-utils: 2.8.0(@typescript-eslint/parser@6.17.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) eslint-plugin-import: 2.29.0(@typescript-eslint/parser@6.17.0)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) fast-glob: 3.3.1 - get-tsconfig: 4.7.2 + get-tsconfig: 4.7.5 is-core-module: 2.13.1 is-glob: 4.0.3 transitivePeerDependencies: @@ -4958,11 +4916,10 @@ packages: get-intrinsic: 1.2.2 dev: true - /get-tsconfig@4.7.2: - resolution: {integrity: sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==} + /get-tsconfig@4.7.5: + resolution: {integrity: sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==} dependencies: resolve-pkg-maps: 1.0.0 - dev: true /git-hooks-list@3.1.0: resolution: {integrity: sha512-LF8VeHeR7v+wAbXqfgRlTSX/1BJR9Q1vEMR8JAz1cEg6GX07+zyj3sAdDvYjj/xnlIfVuGgj4qBei1K3hKH+PA==} @@ -6363,7 +6320,7 @@ packages: resolution: {integrity: sha512-ic6C681bSow1XQdKhADthM/OOKqNL05M1gCFLx1mRqLJ+yH49v6qnvaWQ76kwqI/IieCuVTXfRfTk3sz4cB45w==} hasBin: true dependencies: - '@types/node': 18.19.24 + '@types/node': 18.19.31 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -6380,7 +6337,7 @@ packages: resolution: {integrity: sha512-mIL9WtrFNOanpx98mJ+X/wkoepcxdqqu0noWFoNQHl/yODQ47YM7NEYda7qp8JfjqpLFVxY9mQhshoS/Fqac0A==} hasBin: true dependencies: - '@types/node': 18.19.24 + '@types/node': 18.19.31 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -6396,7 +6353,7 @@ packages: resolution: {integrity: sha512-Ym5GJL98ZhLJJ7enBx53jjG3vwN/fsB+Ozh46nnRZZS9W1NiYqbwkJ+sXd3dkCIiWIgcyyOPL2Zr8SQAzbpj3g==} hasBin: true dependencies: - '@types/node': 18.19.24 + '@types/node': 18.19.31 '@types/node-fetch': 2.6.11 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -7068,7 +7025,6 @@ packages: /resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - dev: true /resolve@1.19.0: resolution: {integrity: sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==} @@ -7196,10 +7152,12 @@ packages: /semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} + hasBin: true dev: true /semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} + hasBin: true /semver@7.5.4: resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} @@ -7893,6 +7851,17 @@ packages: typescript: 5.3.3 dev: true + /tsx@4.10.2: + resolution: {integrity: sha512-gOfACgv1ElsIjvt7Fp0rMJKGnMGjox0JfGOfX3kmZCV/yZumaNqtHGKBXt1KgaYS9KjDOmqGeI8gHk/W7kWVZg==} + engines: {node: '>=18.0.0'} + hasBin: true + dependencies: + esbuild: 0.20.2 + get-tsconfig: 4.7.5 + optionalDependencies: + fsevents: 2.3.3 + dev: false + /tty-table@4.2.3: resolution: {integrity: sha512-Fs15mu0vGzCrj8fmJNP7Ynxt5J7praPXqFN0leZeZBXJwkMxv9cb2D454k1ltrtUSJbZ4yH4e0CynsHLxmUfFA==} engines: {node: '>=8.0.0'} @@ -8436,6 +8405,7 @@ packages: /which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} + hasBin: true dependencies: isexe: 2.0.0 @@ -8600,7 +8570,7 @@ packages: dependencies: '@types/fs-extra': 11.0.4 '@types/minimist': 1.2.5 - '@types/node': 18.19.24 + '@types/node': 18.19.31 '@types/ps-tree': 1.1.6 '@types/which': 3.0.3 chalk: 5.3.0 diff --git a/tsconfig.json b/tsconfig.json index 2f9214e1..abd937e4 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,3 +1,4 @@ { - "extends": "@empiricalrun/typescript-config/base.json" + "extends": "@empiricalrun/typescript-config/base.json", + "exclude": ["examples", "node_modules"] }