From 727bd58514ac922b3d6ad8098d7bdb9b09bb3c87 Mon Sep 17 00:00:00 2001 From: Saikat Mitra Date: Tue, 30 Apr 2024 18:31:44 +0530 Subject: [PATCH 1/4] fix: add retry logs --- packages/ai/src/providers/anthropic/index.ts | 5 ++++- packages/ai/src/providers/azure-openai/index.ts | 2 +- packages/ai/src/providers/fireworks/index.ts | 5 ++++- packages/ai/src/providers/google/index.ts | 5 ++++- packages/ai/src/providers/openai/index.ts | 5 ++++- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index af7c319c..94f2bac9 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -69,7 +69,7 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { try { const startedAt = Date.now(); const response = await promiseRetry( - (retry) => { + (retry, attempt) => { return anthropic.messages .create({ model: canonicalModelName(model), @@ -92,6 +92,9 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { err instanceof Anthropic.APIConnectionTimeoutError || err instanceof Anthropic.InternalServerError ) { + console.log( + `Retrying request for anthropic model: ${body.model}. Retry count: ${attempt}`, + ); retry(err); throw err; } diff --git a/packages/ai/src/providers/azure-openai/index.ts b/packages/ai/src/providers/azure-openai/index.ts index 7fa48d69..583073a7 100644 --- a/packages/ai/src/providers/azure-openai/index.ts +++ b/packages/ai/src/providers/azure-openai/index.ts @@ -39,7 +39,7 @@ const createChatCompletion: ICreateChatCompletion = async ( maxRetries: 5, shouldRetry: async (resp, retryCount) => { if (resp instanceof Response) { - console.log( + console.warn( `Retrying request for azure-openai model: ${body.model}. Retry count: ${retryCount}`, ); if (resp.status === 429) { diff --git a/packages/ai/src/providers/fireworks/index.ts b/packages/ai/src/providers/fireworks/index.ts index 02a97e4c..ed1e5f0e 100644 --- a/packages/ai/src/providers/fireworks/index.ts +++ b/packages/ai/src/providers/fireworks/index.ts @@ -40,7 +40,7 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { try { const startedAt = Date.now(); const completion = await promiseRetry( - (retry) => { + (retry, attempt) => { return fetch("https://api.fireworks.ai/inference/v1/chat/completions", { method: "POST", headers: { @@ -64,6 +64,9 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { AIErrorEnum.RATE_LIMITED, "Fireworks API rate limit reached", ); + console.warn( + `Retrying request for fireworks model: ${body.model}. Got response status code ${response.status}. Retry count: ${attempt}`, + ); retry(err); throw err; } diff --git a/packages/ai/src/providers/google/index.ts b/packages/ai/src/providers/google/index.ts index 51bff46b..c2c1acd3 100644 --- a/packages/ai/src/providers/google/index.ts +++ b/packages/ai/src/providers/google/index.ts @@ -90,7 +90,7 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { try { const startedAt = Date.now(); const completion = await promiseRetry( - (retry) => { + (retry, attempt) => { return modelInstance .startChat({ // @ts-ignore same as above @@ -101,6 +101,9 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { .catch((err: Error) => { // TODO: Replace with instanceof checks when the Gemini SDK exports errors if (err.message.includes("[429 Too Many Requests]")) { + console.warn( + `Retrying request for google model: ${body.model}. Retry count: ${attempt}`, + ); retry(err); } throw err; diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index d0af5d78..f6411041 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -33,7 +33,7 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { try { const startedAt = Date.now(); const completions = await promiseRetry( - (retry) => { + (retry, attempt) => { return openai.chat.completions.create(body).catch((err) => { if ( err instanceof OpenAI.RateLimitError && @@ -46,6 +46,9 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { err instanceof OpenAI.APIConnectionTimeoutError || err instanceof OpenAI.InternalServerError ) { + console.log( + `Retrying request for openai model: ${body.model}. Retry count: ${attempt}`, + ); retry(err); throw err; } From 9ad093b9a124bc73376053b1516924c447b11648 Mon Sep 17 00:00:00 2001 From: Saikat Mitra Date: Tue, 30 Apr 2024 18:33:17 +0530 Subject: [PATCH 2/4] fix: review comments --- packages/ai/src/providers/anthropic/index.ts | 2 +- packages/ai/src/providers/openai/index.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index 94f2bac9..49087a39 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -92,7 +92,7 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { err instanceof Anthropic.APIConnectionTimeoutError || err instanceof Anthropic.InternalServerError ) { - console.log( + console.warn( `Retrying request for anthropic model: ${body.model}. Retry count: ${attempt}`, ); retry(err); diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index f6411041..f1764c98 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -46,7 +46,7 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { err instanceof OpenAI.APIConnectionTimeoutError || err instanceof OpenAI.InternalServerError ) { - console.log( + console.warn( `Retrying request for openai model: ${body.model}. Retry count: ${attempt}`, ); retry(err); @@ -161,7 +161,7 @@ const runAssistant: ICreateAndRunAssistantThread = async (body) => { return asstRunResp; })().catch((err: any) => { if ((err.message as string).includes("server_error")) { - console.log( + console.warn( `Retrying request due to server error (attempt ${attempt})`, ); requestStartTime = Date.now(); From 5ffa6e33ff11ab3834256737e0bd0f0a3d3648e5 Mon Sep 17 00:00:00 2001 From: Saikat Mitra Date: Tue, 30 Apr 2024 19:53:22 +0530 Subject: [PATCH 3/4] chore: add changeset --- .changeset/lucky-needles-change.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/lucky-needles-change.md diff --git a/.changeset/lucky-needles-change.md b/.changeset/lucky-needles-change.md new file mode 100644 index 00000000..f50167b7 --- /dev/null +++ b/.changeset/lucky-needles-change.md @@ -0,0 +1,7 @@ +--- +"@empiricalrun/ai": patch +"@empiricalrun/cli": patch +"@empiricalrun/core": patch +--- + +fix: add retry logs for model api calls From da6b6f9b40773bb34368fba9d2da7c02b9eb917d Mon Sep 17 00:00:00 2001 From: Saikat Mitra Date: Tue, 30 Apr 2024 19:56:08 +0530 Subject: [PATCH 4/4] chore: rename retry count to attempt --- packages/ai/src/providers/anthropic/index.ts | 2 +- packages/ai/src/providers/azure-openai/index.ts | 2 +- packages/ai/src/providers/fireworks/index.ts | 2 +- packages/ai/src/providers/google/index.ts | 2 +- packages/ai/src/providers/openai/index.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/ai/src/providers/anthropic/index.ts b/packages/ai/src/providers/anthropic/index.ts index 49087a39..a994a3a3 100644 --- a/packages/ai/src/providers/anthropic/index.ts +++ b/packages/ai/src/providers/anthropic/index.ts @@ -93,7 +93,7 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { err instanceof Anthropic.InternalServerError ) { console.warn( - `Retrying request for anthropic model: ${body.model}. Retry count: ${attempt}`, + `Retrying request for anthropic model: ${body.model}. Retry attempt: ${attempt}`, ); retry(err); throw err; diff --git a/packages/ai/src/providers/azure-openai/index.ts b/packages/ai/src/providers/azure-openai/index.ts index 583073a7..5f5caab7 100644 --- a/packages/ai/src/providers/azure-openai/index.ts +++ b/packages/ai/src/providers/azure-openai/index.ts @@ -40,7 +40,7 @@ const createChatCompletion: ICreateChatCompletion = async ( shouldRetry: async (resp, retryCount) => { if (resp instanceof Response) { console.warn( - `Retrying request for azure-openai model: ${body.model}. Retry count: ${retryCount}`, + `Retrying request for azure-openai model: ${body.model}. Retry attempt: ${retryCount}`, ); if (resp.status === 429) { requestStartTime = new Date().getTime(); diff --git a/packages/ai/src/providers/fireworks/index.ts b/packages/ai/src/providers/fireworks/index.ts index ed1e5f0e..d0cc1f11 100644 --- a/packages/ai/src/providers/fireworks/index.ts +++ b/packages/ai/src/providers/fireworks/index.ts @@ -65,7 +65,7 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { "Fireworks API rate limit reached", ); console.warn( - `Retrying request for fireworks model: ${body.model}. Got response status code ${response.status}. Retry count: ${attempt}`, + `Retrying request for fireworks model: ${body.model}. Got response status code ${response.status}. Retry attempt: ${attempt}`, ); retry(err); throw err; diff --git a/packages/ai/src/providers/google/index.ts b/packages/ai/src/providers/google/index.ts index c2c1acd3..bcbdcafb 100644 --- a/packages/ai/src/providers/google/index.ts +++ b/packages/ai/src/providers/google/index.ts @@ -102,7 +102,7 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { // TODO: Replace with instanceof checks when the Gemini SDK exports errors if (err.message.includes("[429 Too Many Requests]")) { console.warn( - `Retrying request for google model: ${body.model}. Retry count: ${attempt}`, + `Retrying request for google model: ${body.model}. Retry attempt: ${attempt}`, ); retry(err); } diff --git a/packages/ai/src/providers/openai/index.ts b/packages/ai/src/providers/openai/index.ts index f1764c98..189d37f3 100644 --- a/packages/ai/src/providers/openai/index.ts +++ b/packages/ai/src/providers/openai/index.ts @@ -47,7 +47,7 @@ const createChatCompletion: ICreateChatCompletion = async (body) => { err instanceof OpenAI.InternalServerError ) { console.warn( - `Retrying request for openai model: ${body.model}. Retry count: ${attempt}`, + `Retrying request for openai model: ${body.model}. Retry attempt: ${attempt}`, ); retry(err); throw err;