diff --git a/packages/midscene/src/ai-model/coze/index.ts b/packages/midscene/src/ai-model/coze/index.ts deleted file mode 100644 index 6074a2934..000000000 --- a/packages/midscene/src/ai-model/coze/index.ts +++ /dev/null @@ -1,107 +0,0 @@ -import assert from 'node:assert'; -import type { ChatCompletionUserMessageParam } from 'openai/resources'; - -export const COZE_INSPECT_ELEMENT_BOT_ID = - process.env.COZE_INSPECT_ELEMENT_BOT_ID || ''; -export const COZE_AI_ACTION_BOT_ID = process.env.COZE_AI_ACTION_BOT_ID || ''; -export const COZE_AI_ASSERT_BOT_ID = process.env.COZE_AI_ASSERT_BOT_ID || ''; -export const COZE_EXTRACT_INFO_BOT_ID = - process.env.COZE_EXTRACT_INFO_BOT_ID || ''; - -export const COZE_BOT_TOKEN = 'COZE_BOT_TOKEN'; - -export function preferCozeModel(preferVendor?: 'coze' | 'openAI') { - if (preferVendor && preferVendor !== 'coze') return false; - return ( - process.env[COZE_BOT_TOKEN] && - process.env.COZE_INSPECT_ELEMENT_BOT_ID && - process.env.COZE_AI_ACTION_BOT_ID && - process.env.COZE_AI_ASSERT_BOT_ID && - process.env.COZE_EXTRACT_INFO_BOT_ID - ); -} - -export async function callCozeAi(options: { - query: string; - imgs: Array; - botId: string; -}): Promise { - const { query, imgs, botId } = options; - const completion = await fetch('https://api.coze.com/open_api/v2/chat', { - method: 'POST', - headers: { - Authorization: `Bearer ${process.env[COZE_BOT_TOKEN]}`, - 'Content-Type': 'application/json', - Accept: '*/*', - Host: 'api.coze.com', - Connection: 'keep-alive', - }, - body: JSON.stringify({ - conversation_id: '123', - bot_id: botId, - user: '29032201862555', - query, - meta_data: { - img: imgs.map((imgPath) => { - return { - url: imgPath, - }; - }), - }, - stream: false, - }), - }); - if (!completion.ok) { - console.error('CozeAI reponse error', completion); - throw new Error('Network response was not ok'); - } - - const aiResponse = await completion.json(); - if (aiResponse.code !== 0) { - console.error('CozeAI error response', aiResponse.msg); - throw new Error(`CozeAI error response ${aiResponse.msg}`); - } - - if (!aiResponse?.messages || !aiResponse?.messages[0]?.content) { - console.error('aiResponse', aiResponse); - throw new Error('aiResponse is undefined', aiResponse); - } - const parseContent = aiResponse?.messages[0]?.content; - assert(parseContent, 'empty content'); - try { - return JSON.parse(parseContent); - } catch (err) { - console.error("can't parse coze content", aiResponse, err); - throw Error("can't parse coze content"); - } -} - -export function transformOpenAiArgsToCoze(msg: ChatCompletionUserMessageParam) { - if (msg.role !== 'user') throw Error(`can't transform ${msg} to coze args`); - // const query = ''; - // const imgs = msg.content - if (typeof msg.content === 'string') { - return { - query: msg.content, - imgs: [], - }; - } - - return { - query: msg.content.reduce((res, next) => { - if (next.type === 'text') { - res += `\n${next.text}`; - } - return res; - }, ''), - imgs: msg.content.reduce( - (res, next) => { - if (next.type === 'image_url') { - res.push(next.image_url.url); - } - return res; - }, - [] as Array, - ), - }; -} diff --git a/packages/midscene/src/ai-model/inspect.ts b/packages/midscene/src/ai-model/inspect.ts index 5d7f5e525..ce7fd68be 100644 --- a/packages/midscene/src/ai-model/inspect.ts +++ b/packages/midscene/src/ai-model/inspect.ts @@ -124,7 +124,6 @@ export async function AiInspectElement< multi: boolean; targetElementDescription: string; callAI?: typeof callAiFn; - useModel?: 'coze' | 'openAI'; quickAnswer?: AISingleElementResponse; }): Promise<{ parseResult: AIElementResponse; @@ -195,9 +194,8 @@ export async function AiExtractElementInfo< >(options: { dataQuery: string | Record; context: UIContext; - useModel?: 'coze' | 'openAI'; }) { - const { dataQuery, context, useModel } = options; + const { dataQuery, context } = options; const systemPrompt = systemPromptToExtract(); const { screenshotBase64 } = context; @@ -256,9 +254,8 @@ export async function AiAssert< >(options: { assertion: string; context: UIContext; - useModel?: 'coze' | 'openAI'; }) { - const { assertion, context, useModel } = options; + const { assertion, context } = options; assert(assertion, 'assertion should be a string'); diff --git a/packages/midscene/src/ai-model/openai/index.ts b/packages/midscene/src/ai-model/openai/index.ts index c2a96bd5b..fcc8b16df 100644 --- a/packages/midscene/src/ai-model/openai/index.ts +++ b/packages/midscene/src/ai-model/openai/index.ts @@ -41,7 +41,7 @@ import { findElementSchema } from '../prompt/element_inspector'; import { planSchema } from '../prompt/planning'; import { assertSchema } from '../prompt/util'; -export function checkAIConfig(preferVendor?: 'coze' | 'openAI') { +export function checkAIConfig(preferVendor?: 'openAI') { if (preferVendor && preferVendor !== 'openAI') return false; if (getAIConfig(OPENAI_API_KEY)) return true; if (getAIConfig(MIDSCENE_USE_AZURE_OPENAI)) return true; diff --git a/packages/midscene/tests/ai/assert/assert.test.ts b/packages/midscene/tests/ai/assert/assert.test.ts index b03ddd570..4e67c9dcb 100644 --- a/packages/midscene/tests/ai/assert/assert.test.ts +++ b/packages/midscene/tests/ai/assert/assert.test.ts @@ -1,5 +1,4 @@ import { AiAssert } from '@/ai-model'; -import { preferCozeModel } from '@/ai-model/coze'; /* eslint-disable max-lines-per-function */ import { describe, expect, it, vi } from 'vitest'; import { getPageDataOfTestName } from '../evaluate/test-suite/util'; @@ -9,38 +8,28 @@ vi.setConfig({ hookTimeout: 30 * 1000, }); -const modelList: Array<'openAI' | 'coze'> = ['openAI']; +describe('assert', () => { + it('todo pass', async () => { + const { context } = await getPageDataOfTestName('todo'); -if (preferCozeModel('coze')) { - modelList.push('coze'); -} - -modelList.forEach((model) => { - describe('assert', () => { - it('todo pass', async () => { - const { context } = await getPageDataOfTestName('todo'); - - const { - content: { pass }, - } = await AiAssert({ - assertion: 'Three tasks have been added', - context, - useModel: model, - }); - expect(pass).toBe(true); + const { + content: { pass }, + } = await AiAssert({ + assertion: 'Three tasks have been added', + context, }); + expect(pass).toBe(true); + }); - it('todo error', async () => { - const { context } = await getPageDataOfTestName('todo'); + it('todo error', async () => { + const { context } = await getPageDataOfTestName('todo'); - const { - content: { pass, thought }, - } = await AiAssert({ - assertion: 'There are four tasks in the task list', - context, - useModel: model, - }); - expect(pass).toBe(false); + const { + content: { pass, thought }, + } = await AiAssert({ + assertion: 'There are four tasks in the task list', + context, }); + expect(pass).toBe(false); }); }); diff --git a/packages/midscene/tests/ai/evaluate/plan/planning-input.test.ts b/packages/midscene/tests/ai/evaluate/plan/planning-input.test.ts new file mode 100644 index 000000000..7d377e52c --- /dev/null +++ b/packages/midscene/tests/ai/evaluate/plan/planning-input.test.ts @@ -0,0 +1,45 @@ +import { plan } from '@/ai-model'; +/* eslint-disable max-lines-per-function */ +import { describe, expect, it, vi } from 'vitest'; +import { makePlanResultStable } from '../../util'; +import { getPageDataOfTestName, repeat } from './../test-suite/util'; + +vi.setConfig({ + testTimeout: 180 * 1000, + hookTimeout: 30 * 1000, +}); + +describe('automation - planning input', () => { + repeat(5, () => + it('input value', async () => { + const { context } = await getPageDataOfTestName('todo'); + const instructions = [ + 'In the taskbar, type learning english', + 'In the taskbar, type learning english and hit Enter key', + ]; + + for (const instruction of instructions) { + const { actions } = await plan(instruction, { context }); + const res = makePlanResultStable(actions); + expect(res).toMatchSnapshot(); + } + }), + ); + + repeat(5, () => + it('input value Add, delete, correct and check', async () => { + const { context } = await getPageDataOfTestName('todo-input-with-value'); + const instructions = [ + 'Append "tomorrow" to the existing content in the task input box', + 'Replace "English" with "Skiing" in the existing content of the task input box', + 'Delete "English" from the existing content in the task input box', + ]; + + for (const instruction of instructions) { + const { actions } = await plan(instruction, { context }); + const res = makePlanResultStable(actions); + expect(res).toMatchSnapshot(); + } + }), + ); +}); diff --git a/packages/midscene/tests/ai/evaluate/planning.test.ts b/packages/midscene/tests/ai/evaluate/plan/planning.test.ts similarity index 100% rename from packages/midscene/tests/ai/evaluate/planning.test.ts rename to packages/midscene/tests/ai/evaluate/plan/planning.test.ts diff --git a/packages/midscene/tests/ai/evaluate/planning-input.test.ts b/packages/midscene/tests/ai/evaluate/planning-input.test.ts deleted file mode 100644 index cfb311002..000000000 --- a/packages/midscene/tests/ai/evaluate/planning-input.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { plan } from '@/ai-model'; -/* eslint-disable max-lines-per-function */ -import { describe, expect, it, vi } from 'vitest'; -import { makePlanResultStable, modelList } from '../util'; -import { getPageDataOfTestName, repeat } from './test-suite/util'; - -vi.setConfig({ - testTimeout: 180 * 1000, - hookTimeout: 30 * 1000, -}); - -modelList.forEach((model) => { - describe(`automation - planning input - ${model}`, () => { - repeat(5, () => - it('input value', async () => { - const { context } = await getPageDataOfTestName('todo'); - const instructions = [ - 'In the taskbar, type learning english', - 'In the taskbar, type learning english and hit Enter key', - ]; - - for (const instruction of instructions) { - const { actions } = await plan(instruction, { context }, model); - const res = makePlanResultStable(actions); - expect(res).toMatchSnapshot(); - } - }), - ); - - repeat(5, () => - it('input value Add, delete, correct and check', async () => { - const { context } = await getPageDataOfTestName( - 'todo-input-with-value', - ); - const instructions = [ - 'Append "tomorrow" to the existing content in the task input box', - 'Replace "English" with "Skiing" in the existing content of the task input box', - 'Delete "English" from the existing content in the task input box', - ]; - - for (const instruction of instructions) { - const { actions } = await plan(instruction, { context }, model); - const res = makePlanResultStable(actions); - expect(res).toMatchSnapshot(); - } - }), - ); - }); -}); diff --git a/packages/midscene/tests/ai/extract/extract.test.ts b/packages/midscene/tests/ai/extract/extract.test.ts index 6b023a471..18995c4c0 100644 --- a/packages/midscene/tests/ai/extract/extract.test.ts +++ b/packages/midscene/tests/ai/extract/extract.test.ts @@ -1,5 +1,4 @@ import { AiExtractElementInfo } from '@/ai-model'; -import { preferCozeModel } from '@/ai-model/coze'; import { describe, expect, it, vi } from 'vitest'; import { getPageDataOfTestName } from '../evaluate/test-suite/util'; @@ -8,46 +7,35 @@ vi.setConfig({ hookTimeout: 30 * 1000, }); -const modelList: Array<'openAI' | 'coze'> = ['openAI']; +describe('extract', () => { + it('todo', async () => { + const { context } = await getPageDataOfTestName('todo'); -if (preferCozeModel('coze')) { - modelList.push('coze'); -} - -modelList.forEach((model) => { - describe(`assert ${model}`, () => { - it('todo', async () => { - const { context } = await getPageDataOfTestName('todo'); - - const { parseResult } = await AiExtractElementInfo({ - dataQuery: 'Array, Complete task list, string is the task', - context, - useModel: model, - }); - expect(parseResult).toMatchSnapshot(); + const { parseResult } = await AiExtractElementInfo({ + dataQuery: 'Array, Complete task list, string is the task', + context, }); + expect(parseResult).toMatchSnapshot(); + }); - it('online order', async () => { - const { context } = await getPageDataOfTestName('online_order'); + it('online order', async () => { + const { context } = await getPageDataOfTestName('online_order'); - const { parseResult } = await AiExtractElementInfo({ - dataQuery: '{name: string, price: string}[], 饮品名称和价格', - context, - useModel: model, - }); - expect(parseResult).toMatchSnapshot(); + const { parseResult } = await AiExtractElementInfo({ + dataQuery: '{name: string, price: string}[], 饮品名称和价格', + context, }); + expect(parseResult).toMatchSnapshot(); + }); - it('todo obj', async () => { - const { context } = await getPageDataOfTestName('todo'); + it('todo obj', async () => { + const { context } = await getPageDataOfTestName('todo'); - const { parseResult } = await AiExtractElementInfo({ - dataQuery: - '{checked: boolean; text: string}[],Complete task list, string is the task', - context, - useModel: model, - }); - expect(parseResult).toMatchSnapshot(); + const { parseResult } = await AiExtractElementInfo({ + dataQuery: + '{checked: boolean; text: string}[],Complete task list, string is the task', + context, }); + expect(parseResult).toMatchSnapshot(); }); }); diff --git a/packages/midscene/tests/ai/util.ts b/packages/midscene/tests/ai/util.ts index f85e76729..211383676 100644 --- a/packages/midscene/tests/ai/util.ts +++ b/packages/midscene/tests/ai/util.ts @@ -1,4 +1,3 @@ -import { preferCozeModel } from '@/ai-model/coze'; import type { PlanningAction } from '@/types'; export function makePlanResultStable(plans: PlanningAction[]) { @@ -15,7 +14,3 @@ export function makePlanResultStable(plans: PlanningAction[]) { return plan; }); } - -export const modelList: Array<'openAI' | 'coze'> = preferCozeModel('coze') - ? ['openAI', 'coze'] - : ['openAI'];