diff --git a/packages/create-empiricalrun/package.json b/packages/create-empiricalrun/package.json index 6976f6a0..217de8a6 100644 --- a/packages/create-empiricalrun/package.json +++ b/packages/create-empiricalrun/package.json @@ -1,11 +1,14 @@ { - "name": "@empiricalrun/create-empiricalrun", + "name": "create-empiricalrun", "version": "0.0.0", "publishConfig": { "registry": "https://registry.npmjs.org/", "access": "public" }, "main": "dist/index.js", + "bin": { + "create-empiricalrun": "dist/index.js" + }, "repository": { "type": "git", "url": "https://github.com/empirical-run/empirical.git" @@ -18,6 +21,11 @@ "test": "vitest run", "test:watch": "vitest" }, - "devDependencies": {}, - "dependencies": {} + "dependencies": { + "@types/node": "^20.11.24", + "enquirer": "^2.4.1" + }, + "devDependencies": { + "@empiricalrun/types": "workspace:^" + } } \ No newline at end of file diff --git a/packages/create-empiricalrun/src/constant.ts b/packages/create-empiricalrun/src/constant.ts new file mode 100644 index 00000000..d3a6bb37 --- /dev/null +++ b/packages/create-empiricalrun/src/constant.ts @@ -0,0 +1,3 @@ +// export const +export const CACHE_DIR = ".empiricalrun"; +export const GEN_CONFIG_FILE_NAME = "empiricalrc.json"; diff --git a/packages/create-empiricalrun/src/generator/config.ts b/packages/create-empiricalrun/src/generator/config.ts new file mode 100644 index 00000000..ff0b4ac7 --- /dev/null +++ b/packages/create-empiricalrun/src/generator/config.ts @@ -0,0 +1,41 @@ +import { Config } from "@empiricalrun/types"; +export const config: Config = { + $schema: "https://assets.empirical.run/config/schema/latest.json", + runs: [ + { + type: "model", + provider: "openai", + model: "gpt-3.5-turbo", + prompt: + "Extract the name, age and location from the message, and respond with a JSON object. If an entity is missing, respond with null.\n\nMessage: {{user_message}}", + }, + { + type: "model", + provider: "openai", + model: "gpt-4-turbo-preview", + prompt: + "Extract the name, age and location from the message, and respond with a JSON object. If an entity is missing, respond with null.\n\nMessage: {{user_message}}", + }, + ], + dataset: { + samples: [ + { + inputs: { + user_message: + "Hi my name is John Doe. I'm 26 years old and I work in real estate.", + }, + }, + { + inputs: { + user_message: + "This is Alice. I am a nurse from Maryland. I was born in 1990.", + }, + }, + ], + }, + scorers: [ + { + type: "json-syntax", + }, + ], +}; diff --git a/packages/create-empiricalrun/src/generator/index.ts b/packages/create-empiricalrun/src/generator/index.ts new file mode 100644 index 00000000..5f495c32 --- /dev/null +++ b/packages/create-empiricalrun/src/generator/index.ts @@ -0,0 +1,17 @@ +import { PackageManager } from "../pkg-managers/interface"; +import { Generator } from "./interface"; +import { JSGenerator } from "./js"; +import { JSONGenerator } from "./json"; +import { TSGenerator } from "./ts"; + +export function getGenerator( + format: string, + packageManager: PackageManager, +): Generator { + if (format.includes("Javascript")) { + return new JSGenerator({ packageManager }); + } else if (format.includes("Typescript")) { + return new TSGenerator(); + } + return new JSONGenerator(); +} diff --git a/packages/create-empiricalrun/src/generator/interface.ts b/packages/create-empiricalrun/src/generator/interface.ts new file mode 100644 index 00000000..3d538f26 --- /dev/null +++ b/packages/create-empiricalrun/src/generator/interface.ts @@ -0,0 +1,4 @@ +export interface Generator { + format: string; + generate(): Promise; +} diff --git a/packages/create-empiricalrun/src/generator/js/index.ts b/packages/create-empiricalrun/src/generator/js/index.ts new file mode 100644 index 00000000..5c09845b --- /dev/null +++ b/packages/create-empiricalrun/src/generator/js/index.ts @@ -0,0 +1,16 @@ +import { PackageManager } from "../../pkg-managers/interface"; +import { Generator } from "../interface"; +import { generateGitIgnore } from "../utils"; +export class JSGenerator implements Generator { + format = "js"; + packageManager: PackageManager; + constructor({ packageManager }: { packageManager: PackageManager }) { + this.packageManager = packageManager; + } + + async generate() { + // generate json file + const cwd = process.cwd(); + await generateGitIgnore(cwd); + } +} diff --git a/packages/create-empiricalrun/src/generator/json/index.ts b/packages/create-empiricalrun/src/generator/json/index.ts new file mode 100644 index 00000000..e5e26d48 --- /dev/null +++ b/packages/create-empiricalrun/src/generator/json/index.ts @@ -0,0 +1,20 @@ +import { config } from "../config"; +import { Generator } from "../interface"; +import fs from "fs/promises"; +import { generateGitIgnore } from "../utils"; +import { GEN_CONFIG_FILE_NAME } from "../../constant"; +export class JSONGenerator implements Generator { + format = "json"; + + async #createJSONConfig(dir: string) { + const configFileFullPath = `${dir}/${GEN_CONFIG_FILE_NAME}`; + await fs.writeFile(configFileFullPath, JSON.stringify(config, null, 2)); + } + + async generate() { + const cwd = process.cwd(); + await this.#createJSONConfig(cwd); + await generateGitIgnore(cwd); + console.log(`created ${GEN_CONFIG_FILE_NAME} in ${cwd}`); + } +} diff --git a/packages/create-empiricalrun/src/generator/ts/index.ts b/packages/create-empiricalrun/src/generator/ts/index.ts new file mode 100644 index 00000000..71c06824 --- /dev/null +++ b/packages/create-empiricalrun/src/generator/ts/index.ts @@ -0,0 +1,9 @@ +import { Generator } from "../interface"; +export class TSGenerator implements Generator { + format = "ts"; + + async generate() { + // generate json file + console.log("generate ts files"); + } +} diff --git a/packages/create-empiricalrun/src/generator/utils.ts b/packages/create-empiricalrun/src/generator/utils.ts new file mode 100644 index 00000000..d3b82c41 --- /dev/null +++ b/packages/create-empiricalrun/src/generator/utils.ts @@ -0,0 +1,10 @@ +import { CACHE_DIR } from "../constant"; +import fs from "fs/promises"; + +export const generateGitIgnore = async (dir: string) => { + const gitIgnoreFullPath = `${dir}/.gitignore`; + await fs.appendFile( + gitIgnoreFullPath, + `\n# Ignore outputs from Empirical\n${CACHE_DIR}\n`, + ); +}; diff --git a/packages/create-empiricalrun/src/index.ts b/packages/create-empiricalrun/src/index.ts index e69de29b..d37bf06d 100644 --- a/packages/create-empiricalrun/src/index.ts +++ b/packages/create-empiricalrun/src/index.ts @@ -0,0 +1,32 @@ +#!/usr/bin/env node +import { prompt } from "enquirer"; +import { getPackageManager } from "./pkg-managers"; +import { getGenerator } from "./generator"; + +interface Config { + format: "JSON" | "Javascript" | "Typescript"; +} + +// ask which format +// if the format is just JSON, use JSON generator +// if the format is js then use JS generator +// in the JS generator have steps for installing dependencies for JS +// TS generator extends JS but with one step overriden for file generation + +async function getConfig() { + const config: Config = await prompt([ + { + name: "format", + type: "select", + message: "What kind of format do you want for your configuration file?", + choices: ["JSON", "Javascript", "Typescript"], + }, + ]); + const packageManager = getPackageManager(); + console.log(`Using ${packageManager.name} for installation`); + console.log(`Selected format: ${config.format}`); + const generator = getGenerator(config.format, packageManager); + await generator.generate(); +} + +getConfig(); diff --git a/packages/create-empiricalrun/src/pkg-managers/index.ts b/packages/create-empiricalrun/src/pkg-managers/index.ts new file mode 100644 index 00000000..3d76f4af --- /dev/null +++ b/packages/create-empiricalrun/src/pkg-managers/index.ts @@ -0,0 +1,14 @@ +import { PackageManager, PackageManagerConstructor } from "./interface"; +import { NPM } from "./npm"; + +const supportedPkgManager = [NPM]; + +export function getPackageManager(): PackageManager { + const userAgent = process.env.npm_config_user_agent; + const PkgMngr: PackageManagerConstructor | undefined = + supportedPkgManager.find((manager) => manager.check(userAgent!)); + if (!PkgMngr) { + throw new Error("Package manager not supported"); + } + return new PkgMngr(); +} diff --git a/packages/create-empiricalrun/src/pkg-managers/interface.ts b/packages/create-empiricalrun/src/pkg-managers/interface.ts new file mode 100644 index 00000000..9e6cc690 --- /dev/null +++ b/packages/create-empiricalrun/src/pkg-managers/interface.ts @@ -0,0 +1,8 @@ +export interface PackageManager { + name: string; +} + +export interface PackageManagerConstructor { + new (): PackageManager; + check(userAgent: string): boolean; +} diff --git a/packages/create-empiricalrun/src/pkg-managers/npm.ts b/packages/create-empiricalrun/src/pkg-managers/npm.ts new file mode 100644 index 00000000..9b5ec12d --- /dev/null +++ b/packages/create-empiricalrun/src/pkg-managers/npm.ts @@ -0,0 +1,16 @@ +import { PackageManager } from "./interface"; +import { exec } from "child_process"; + +export class NPM implements PackageManager { + static check(userAgent: string): boolean { + return userAgent.includes("npm"); + } + + get name() { + return "npm"; + } + + async installDependency(pkg: string) { + await exec(`npm install ${pkg} --save`); + } +} diff --git a/packages/create-empiricalrun/src/pkg-managers/utils/index.ts b/packages/create-empiricalrun/src/pkg-managers/utils/index.ts new file mode 100644 index 00000000..e69de29b diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63926520..9a58a4d5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -169,6 +169,19 @@ importers: specifier: ^1.1.6 version: 1.1.6 + packages/create-empiricalrun: + dependencies: + '@types/node': + specifier: ^20.11.24 + version: 20.11.24 + enquirer: + specifier: ^2.4.1 + version: 2.4.1 + devDependencies: + '@empiricalrun/types': + specifier: workspace:^ + version: link:../types + packages/empiricalrun: dependencies: '@empiricalrun/ai': @@ -2837,7 +2850,6 @@ packages: /ansi-colors@4.1.3: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} - dev: true /ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} @@ -3852,7 +3864,6 @@ packages: dependencies: ansi-colors: 4.1.3 strip-ansi: 6.0.1 - dev: true /env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} diff --git a/turbo.json b/turbo.json index 94ae182a..bf8884c1 100644 --- a/turbo.json +++ b/turbo.json @@ -14,7 +14,8 @@ "AZURE_OPENAI_RESOURCE_NAME", "AZURE_OPENAI_BASE_URL", "POSTHOG_API_KEY", - "EMPIRICAL_TELEMETRY" + "EMPIRICAL_TELEMETRY", + "npm_config_user_agent" ], "pipeline": { "build": {