Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,13 +135,13 @@ import {
ToolResponse,
} from './resources/agents/agents';
import {
BenchmarkConfig,
Eval,
EvalCandidate,
EvalEvaluateRowsAlphaParams,
EvalEvaluateRowsParams,
EvalRunEvalAlphaParams,
EvalRunEvalParams,
EvalTaskConfig,
EvaluateResponse,
Job,
} from './resources/eval/eval';
Expand Down Expand Up @@ -415,8 +415,8 @@ export declare namespace LlamaStackClient {

export {
Eval as Eval,
type BenchmarkConfig as BenchmarkConfig,
type EvalCandidate as EvalCandidate,
type EvalTaskConfig as EvalTaskConfig,
type EvaluateResponse as EvaluateResponse,
type Job as Job,
type EvalEvaluateRowsParams as EvalEvaluateRowsParams,
Expand Down
30 changes: 15 additions & 15 deletions src/resources/eval/eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,16 @@ export class Eval extends APIResource {
}
}

export interface BenchmarkConfig {
eval_candidate: EvalCandidate;

scoring_params: Record<string, ScoringFunctionsAPI.ScoringFnParams>;

type: 'benchmark';

num_examples?: number;
}

export type EvalCandidate = EvalCandidate.ModelCandidate | EvalCandidate.AgentCandidate;

export namespace EvalCandidate {
Expand All @@ -62,16 +72,6 @@ export namespace EvalCandidate {
}
}

export interface EvalTaskConfig {
eval_candidate: EvalCandidate;

scoring_params: Record<string, ScoringFunctionsAPI.ScoringFnParams>;

type: 'benchmark';

num_examples?: number;
}

export interface EvaluateResponse {
generations: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;

Expand All @@ -87,31 +87,31 @@ export interface EvalEvaluateRowsParams {

scoring_functions: Array<string>;

task_config: EvalTaskConfig;
task_config: BenchmarkConfig;
}

export interface EvalEvaluateRowsAlphaParams {
input_rows: Array<Record<string, boolean | number | string | Array<unknown> | unknown | null>>;

scoring_functions: Array<string>;

task_config: EvalTaskConfig;
task_config: BenchmarkConfig;
}

export interface EvalRunEvalParams {
task_config: EvalTaskConfig;
task_config: BenchmarkConfig;
}

export interface EvalRunEvalAlphaParams {
task_config: EvalTaskConfig;
task_config: BenchmarkConfig;
}

Eval.Jobs = Jobs;

export declare namespace Eval {
export {
type BenchmarkConfig as BenchmarkConfig,
type EvalCandidate as EvalCandidate,
type EvalTaskConfig as EvalTaskConfig,
type EvaluateResponse as EvaluateResponse,
type Job as Job,
type EvalEvaluateRowsParams as EvalEvaluateRowsParams,
Expand Down
2 changes: 1 addition & 1 deletion src/resources/eval/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

export {
Eval,
type BenchmarkConfig,
type EvalCandidate,
type EvalTaskConfig,
type EvaluateResponse,
type Job,
type EvalEvaluateRowsParams,
Expand Down
2 changes: 1 addition & 1 deletion src/resources/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ export {
} from './datasets';
export {
Eval,
type BenchmarkConfig,
type EvalCandidate,
type EvalTaskConfig,
type EvaluateResponse,
type Job,
type EvalEvaluateRowsParams,
Expand Down
4 changes: 2 additions & 2 deletions src/resources/shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ import * as InferenceAPI from './inference';
import * as ToolRuntimeAPI from './tool-runtime/tool-runtime';

export interface AgentConfig {
enable_session_persistence: boolean;

instructions: string;

model: string;

client_tools?: Array<ToolRuntimeAPI.ToolDef>;

enable_session_persistence?: boolean;

input_shields?: Array<string>;

max_infer_iters?: number;
Expand Down
4 changes: 2 additions & 2 deletions tests/api-resources/agents/agents.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ const client = new LlamaStackClient({ baseURL: process.env['TEST_API_BASE_URL']
describe('resource agents', () => {
test('create: only required params', async () => {
const responsePromise = client.agents.create({
agent_config: { enable_session_persistence: true, instructions: 'instructions', model: 'model' },
agent_config: { instructions: 'instructions', model: 'model' },
});
const rawResponse = await responsePromise.asResponse();
expect(rawResponse).toBeInstanceOf(Response);
Expand All @@ -22,7 +22,6 @@ describe('resource agents', () => {
test('create: required and optional params', async () => {
const response = await client.agents.create({
agent_config: {
enable_session_persistence: true,
instructions: 'instructions',
model: 'model',
client_tools: [
Expand All @@ -41,6 +40,7 @@ describe('resource agents', () => {
],
},
],
enable_session_persistence: true,
input_shields: ['string'],
max_infer_iters: 0,
output_shields: ['string'],
Expand Down