fix(prompt): fix plan prompt and optimize task background content

web-infra-dev · Jan 3, 2025 · 95c65f0 · 95c65f0
1 parent 691eb6e
commit 95c65f0
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 28 deletions.
diff --git a/packages/midscene/src/ai-model/automation.ts b/packages/midscene/src/ai-model/automation.ts
@@ -4,8 +4,8 @@ import { PromptTemplate } from '@langchain/core/prompts';
 import { AIActionType, type AIArgs, callAiFn } from './common';
 import {
   automationUserPrompt,
+  generateTaskBackgroundContext,
   systemPromptToTaskPlanning,
-  taskBackgroundContext,
 } from './prompt/planning';
 import { describeUserPage } from './prompt/util';
 
@@ -24,13 +24,14 @@ export async function plan(
     await describeUserPage(context);
 
   const systemPrompt = await systemPromptToTaskPlanning();
+  const taskBackgroundContextText = generateTaskBackgroundContext(
+    userPrompt,
+    opts.originalPrompt,
+    opts.whatHaveDone,
+  );
   const userInstructionPrompt = await automationUserPrompt.format({
     pageDescription,
-    userPrompt,
-    taskBackgroundContext: taskBackgroundContext(
-      opts.originalPrompt,
-      opts.whatHaveDone,
-    ),
+    taskBackgroundContext: taskBackgroundContextText,
   });
 
   const msgs: AIArgs = [

diff --git a/packages/midscene/src/ai-model/prompt/planning.ts b/packages/midscene/src/ai-model/prompt/planning.ts
@@ -11,7 +11,8 @@ const quickAnswerFormat = () => {
       description:
         '"position": { x: number; y: number } // Represents the position of the element; replace with actual values in practice (ensure it reflects the element\'s position)',
       format: '"position": { x: number; y: number }',
-      sample: '{"prompt": "the search bar"}',
+      sample:
+        '{"prompt": "the search bar" // Use language consistent with the information on the page}',
       locateParam: `{
         "prompt"?: string // the description of the element to find. It can only be omitted when locate is null.
       } | null // If it's not on the page, the LocateParam should be null`,
@@ -350,36 +351,38 @@ export const planSchema: ResponseFormatJSONSchema = {
   },
 };
 
-export const taskBackgroundContext = async (
+export const generateTaskBackgroundContext = (
+  userPrompt: string,
   originalPrompt?: string,
   whatHaveDone?: string,
 ) => {
-  if (!originalPrompt || !whatHaveDone) {
-    return '';
-  }
-
-  return `
+  if (originalPrompt && whatHaveDone) {
+    return `
     For your information, this is a task that some important person handed to you. Here is the original task description and what have been done after the previous actions:
     =====================================
-    Original task description:
-    ${originalPrompt}
+    Original task description: ${originalPrompt}
     =====================================
-    What have been done:
-    ${whatHaveDone}
+    What have been done: ${whatHaveDone}
     =====================================
-`;
+    `;
+  }
+
+  return `
+  Here is the instruction:
+  =====================================
+  ${userPrompt}
+  =====================================
+  `;
 };
 
 export const automationUserPrompt = new PromptTemplate({
   template: `
-    Here is the instruction:
+    pageDescription:
     =====================================
-    {userPrompt}
+    {pageDescription}
     =====================================
 
     {taskBackgroundContext}
-
-    pageDescription: {pageDescription}
   `,
-  inputVariables: ['pageDescription', 'userPrompt', 'taskBackgroundContext'],
+  inputVariables: ['pageDescription', 'taskBackgroundContext'],
 });
diff --git a/packages/midscene/src/ai-model/prompt/util.ts b/packages/midscene/src/ai-model/prompt/util.ts
@@ -290,10 +290,7 @@ export async function describeUserPage<
   const sizeDescription = describeSize({ width, height });
 
   return {
-    description: `
-  The size of the page: ${sizeDescription} \n
-  ${pageJSONDescription}
-`,
+    description: `The size of the page: ${sizeDescription} \n ${pageJSONDescription}`,
     elementById(id: string) {
       assert(typeof id !== 'undefined', 'id is required for query');
       const item = idElementMap[`${id}`];

diff --git a/packages/midscene/src/env.ts b/packages/midscene/src/env.ts
@@ -15,7 +15,7 @@ export const MIDSCENE_MODEL_TEXT_ONLY = 'MIDSCENE_MODEL_TEXT_ONLY';
 
 export const MIDSCENE_CACHE = 'MIDSCENE_CACHE';
 export const MATCH_BY_POSITION = 'MATCH_BY_POSITION';
-export const MIDSCENE_API_TYPE = 'MIDSCENE_API_TYPE';
+export const MIDSCENE_API_TYPE = 'MIDSCENE-API-TYPE';
 export const MIDSCENE_REPORT_TAG_NAME = 'MIDSCENE_REPORT_TAG_NAME';
 
 export const MIDSCENE_USE_AZURE_OPENAI = 'MIDSCENE_USE_AZURE_OPENAI';