Skyvern-AI · LawyZheng · Nov 21, 2024 · Nov 21, 2024
diff --git a/alembic/versions/2024_11_21_0708-2d79d5fc1baa_add_task_prompt_template_complete_.py b/alembic/versions/2024_11_21_0708-2d79d5fc1baa_add_task_prompt_template_complete_.py
@@ -0,0 +1,35 @@
+"""Add task prompt template, complete criterion and complete criterion
+
+Revision ID: 2d79d5fc1baa
+Revises: 1909715536dc
+Create Date: 2024-11-21 07:08:19.177274+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "2d79d5fc1baa"
+down_revision: Union[str, None] = "1909715536dc"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("tasks", sa.Column("prompt_template", sa.String(), nullable=True))
+    op.add_column("tasks", sa.Column("complete_criterion", sa.String(), nullable=True))
+    op.add_column("tasks", sa.Column("terminate_criterion", sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("tasks", "terminate_criterion")
+    op.drop_column("tasks", "complete_criterion")
+    op.drop_column("tasks", "prompt_template")
+    # ### end Alembic commands ###
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
@@ -39,11 +39,12 @@
 from skyvern.forge.sdk.core import skyvern_context
 from skyvern.forge.sdk.core.security import generate_skyvern_signature
 from skyvern.forge.sdk.core.validators import validate_url
+from skyvern.forge.sdk.db.enums import TaskPromptTemplate
 from skyvern.forge.sdk.models import Organization, Step, StepStatus
 from skyvern.forge.sdk.schemas.tasks import Task, TaskRequest, TaskStatus
 from skyvern.forge.sdk.settings_manager import SettingsManager
 from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
-from skyvern.forge.sdk.workflow.models.block import TaskBlock
+from skyvern.forge.sdk.workflow.models.block import ActionBlock, BaseTaskBlock
 from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus
 from skyvern.webeye.actions.actions import (
     Action,
@@ -98,7 +99,7 @@ def __init__(self) -> None:
 
     async def create_task_and_step_from_block(
         self,
-        task_block: TaskBlock,
+        task_block: BaseTaskBlock,
         workflow: Workflow,
         workflow_run: WorkflowRun,
         workflow_run_context: WorkflowRunContext,
@@ -132,6 +133,9 @@ async def create_task_and_step_from_block(
         task_url = validate_url(task_url)
         task = await app.DATABASE.create_task(
             url=task_url,
+            prompt_template=task_block.prompt_template,
+            complete_criterion=task_block.complete_criterion,
+            terminate_criterion=task_block.terminate_criterion,
             title=task_block.title or task_block.label,
             webhook_callback_url=None,
             totp_verification_url=task_block.totp_verification_url,
@@ -195,6 +199,8 @@ async def create_task(self, task_request: TaskRequest, organization_id: str | No
             totp_verification_url=totp_verification_url,
             totp_identifier=task_request.totp_identifier,
             navigation_goal=task_request.navigation_goal,
+            complete_criterion=task_request.complete_criterion,
+            terminate_criterion=task_request.terminate_criterion,
             data_extraction_goal=task_request.data_extraction_goal,
             navigation_payload=task_request.navigation_payload,
             organization_id=organization_id,
@@ -222,7 +228,7 @@ async def execute_step(
         step: Step,
         api_key: str | None = None,
         close_browser_on_completion: bool = True,
-        task_block: TaskBlock | None = None,
+        task_block: BaseTaskBlock | None = None,
     ) -> Tuple[Step, DetailedAgentStepOutput | None, Step | None]:
         workflow_run: WorkflowRun | None = None
         if task.workflow_run_id:
@@ -362,7 +368,13 @@ async def execute_step(
                     is_task_completed,
                     maybe_last_step,
                     maybe_next_step,
-                ) = await self.handle_completed_step(organization, task, step, await browser_state.get_working_page())
+                ) = await self.handle_completed_step(
+                    organization=organization,
+                    task=task,
+                    step=step,
+                    page=await browser_state.get_working_page(),
+                    task_block=task_block,
+                )
                 if is_task_completed is not None and maybe_last_step:
                     last_step = maybe_last_step
                     await self.clean_up_task(
@@ -582,7 +594,7 @@ async def agent_step(
         step: Step,
         browser_state: BrowserState,
         organization: Organization | None = None,
-        task_block: TaskBlock | None = None,
+        task_block: BaseTaskBlock | None = None,
     ) -> tuple[Step, DetailedAgentStepOutput]:
         detailed_agent_step_output = DetailedAgentStepOutput(
             scraped_page=None,
@@ -620,7 +632,7 @@ async def agent_step(
             actions: list[Action]
 
             using_cached_action_plan = False
-            if not task.navigation_goal:
+            if not task.workflow_run_id and not task.navigation_goal:
                 actions = [
                     CompleteAction(
                         reasoning="Task has no navigation goal.",
@@ -870,7 +882,7 @@ async def agent_step(
                         break
 
             task_completes_on_download = task_block and task_block.complete_on_download and task.workflow_run_id
-            if not has_decisive_action and not task_completes_on_download:
+            if not has_decisive_action and not task_completes_on_download and not isinstance(task_block, ActionBlock):
                 disable_user_goal_check = app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
                     "DISABLE_USER_GOAL_CHECK",
                     task.task_id,
@@ -1225,8 +1237,10 @@ async def _build_extract_action_prompt(
         final_navigation_payload = self._build_navigation_payload(
             task, expire_verification_code=expire_verification_code
         )
+
+        template = task.prompt_template if task.prompt_template else TaskPromptTemplate.ExtractAction
         return prompt_engine.load_prompt(
-            "extract-action",
+            template=template,
             navigation_goal=navigation_goal,
             navigation_payload_str=json.dumps(final_navigation_payload),
             starting_url=starting_url,
@@ -1237,6 +1251,8 @@ async def _build_extract_action_prompt(
             error_code_mapping_str=(json.dumps(task.error_code_mapping) if task.error_code_mapping else None),
             utc_datetime=datetime.utcnow().strftime("%Y-%m-%d %H:%M"),
             verification_code_check=verification_code_check,
+            complete_criterion=task.complete_criterion,
+            terminate_criterion=task.terminate_criterion,
         )
 
     def _build_navigation_payload(
@@ -1770,6 +1786,7 @@ async def handle_completed_step(
         task: Task,
         step: Step,
         page: Page | None,
+        task_block: BaseTaskBlock | None = None,
     ) -> tuple[bool | None, Step | None, Step | None]:
         if step.is_goal_achieved():
             LOG.info(
@@ -1810,6 +1827,24 @@ async def handle_completed_step(
             or organization.max_steps_per_run
             or SettingsManager.get_settings().MAX_STEPS_PER_RUN
         )
+
+        # HACK: action block only have one step to execute without complete action, so we consider the task is completed as long as the step is completed
+        if isinstance(task_block, ActionBlock) and step.is_success():
+            LOG.info(
+                "Step completed for the action block, marking task as completed",
+                task_id=task.task_id,
+                step_id=step.step_id,
+                step_order=step.order,
+                step_retry=step.retry_index,
+                output=step.output,
+            )
+            last_step = await self.update_step(step, is_last=True)
+            await self.update_task(
+                task,
+                status=TaskStatus.completed,
+            )
+            return True, last_step, None
+
         if step.order + 1 >= max_steps_per_run:
             LOG.info(
                 "Step completed but max steps reached, marking task as failed",

diff --git a/skyvern/forge/prompts/skyvern/decisive-criterion-validate.j2 b/skyvern/forge/prompts/skyvern/decisive-criterion-validate.j2
@@ -0,0 +1,55 @@
+Your are here to help the user determine if the current page has met the complete/terminte criterion. Use the criterions of complete/terminate, the content of the elements parsed from the page, the screenshots of the page, and user details to determine whether the criterions has been met.
+
+
+Reply in JSON format with the following keys:
+{
+    "page_info": str, // Think step by step. Describe all the useful information in the page related to the complete/terminate criterion.
+    "thoughts": str, // Think step by step. What information makes you believe whether and which criterion has been met. Use information you see on the site to explain.
+    "actions": array // You are supposed to give only one action("COMPLETE" or "TERMINATE") in the action list. Here's the format of the action:
+    [{
+        "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.
+        "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
+        "action_type": str, // It's a string enum: "COMPLETE", "TERMINATE". "COMPLETE" is used when the current page info has met the complete criterion. If there is no complete criterion, use "COMPLETE" as long as the page info hasn't met the terminate criterion. "TERMINATE" is used to terminate with a failure when the current page info has met the terminate criterion. It there is no terminate criterion, use "TERMINATE" as long as the page info hasn't met the complete criterion.
+{% if error_code_mapping_str %}
+        "errors": array // A list of errors. This is used to surface any errors that matches the current situation for COMPLETE and TERMINATE actions. For other actions or if no error description suits the current situation on the screenshots, return an empty list. You are allowed to return multiple errors if there are multiple errors on the page.
+        [{
+            "error_code": str, // The error code from the user's error code list
+            "reasoning": str, // The reasoning behind the error. Be specific, referencing any user information and their fields in your reasoning. Keep the reasoning short and to the point.
+            "confidence_float": float // The confidence of the error. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
+        }]
+{% endif %}
+    }]
+}
+
+HTML elements from `{{ current_url }}`:
+```
+{{ elements }}
+```
+
+The URL of the page you're on right now is `{{ current_url }}`.
+
+{% if complete_criterion %}
+Complete Criterion:
+```
+{{ complete_criterion }}
+```{% endif %}
+{% if terminate_criterion %}
+Terminate Criterion:
+```
+{{ terminate_criterion }}
+```{% endif %}
+{% if error_code_mapping_str %}
+Use the error codes and their descriptions to surface user-defined errors. Do not return any error that's not defined by the user. User defined errors:
+```
+{{ error_code_mapping_str }}
+```{% endif %}
+
+User details:
+```
+{{ navigation_payload_str }}
+```
+
+Current datetime in UTC, YYYY-MM-DD HH:MM format:
+```
+{{ utc_datetime }}
+```
diff --git a/skyvern/forge/prompts/skyvern/single-click-action.j2 b/skyvern/forge/prompts/skyvern/single-click-action.j2
@@ -0,0 +1,39 @@
+Your are here to help the user to perform a CLICK action on the web page. Use the user instruction, the content of the elements parsed from the page, the screenshots of the page, and user details to determine which element to click.
+Each actionable element is tagged with an ID. Only take the action on the elements provided in the HTML elements, do not image any new element.
+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
+
+Reply in JSON format with the following keys:
+{
+    "page_info": str, // Think step by step. Describe all the useful information in the page related to the user instruction.
+    "thoughts": str, // Think step by step. What information makes you believe which element to click. Use information you see on the site to explain.
+    "actions": array // You are supposed to give only one action("CLICK") in the action list. Here's the format of the action:
+    [{
+        "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.
+        "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
+        "action_type": str, // It's a string enum: "CLICK". "CLICK" is an element you'd like to click.
+        "id": str, // The id of the element to take action on. The id has to be one from the elements list.
+        "download": bool, // If true, the browser will trigger a download by clicking the element. If false, the browser will click the element without triggering a download.
+    }]
+}
+
+The URL of the page you're on right now is `{{ current_url }}`.
+
+HTML elements from `{{ current_url }}`:
+```
+{{ elements }}
+```
+
+User instruction:
+```
+{{ navigation_goal }}
+```
+
+User details:
+```
+{{ navigation_payload_str }}
+```
+
+Current datetime in UTC, YYYY-MM-DD HH:MM format:
+```
+{{ utc_datetime }}
+```
diff --git a/skyvern/forge/prompts/skyvern/single-input-action.j2 b/skyvern/forge/prompts/skyvern/single-input-action.j2
@@ -0,0 +1,41 @@
+Your are here to help the user to perform an INPUT_TEXT action on the web page. Use the user instruction, the content of the elements parsed from the page, the screenshots of the page, and user details to determine which element to input.
+Each actionable element is tagged with an ID. Only take the action on the elements provided in the HTML elements, do not image any new element.
+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
+
+Reply in JSON format with the following keys:
+{
+    "page_info": str, // Think step by step. Describe all the useful information in the page related to the user instruction.
+    "thoughts": str, // Think step by step. What information makes you believe which element to input the value. Use information you see on the site to explain.
+    "actions": array // You are supposed to give only one action("INPUT_TEXT") in the action list. Here's the format of the action:
+    [{
+        "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.
+        "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
+        "action_type": str, // It's a string enum: "INPUT_TEXT". "INPUT_TEXT" is an element you'd like to input text into.
+        "id": str, // The id of the element to take action on. The id has to be one from the elements list.
+        "text": str, // The text to input.
+    }]{% if verification_code_check %}
+    "verification_code_reasoning": str, // Let's think step by step. Describe what you see and think if a verification code is needed for login or any verification step. Explain why you believe a verification code is needed or not. Has the code been sent and is code available somewhere (email, phone or 2FA device)?
+    "need_verification_code": bool, // Whether a verification code is needed to proceed. True only if the code is available to user. If the code is not sent, return false {% endif %}
+}
+
+The URL of the page you're on right now is `{{ current_url }}`.
+
+HTML elements from `{{ current_url }}`:
+```
+{{ elements }}
+```
+
+User instruction:
+```
+{{ navigation_goal }}
+```
+
+User details:
+```
+{{ navigation_payload_str }}
+```
+
+Current datetime in UTC, YYYY-MM-DD HH:MM format:
+```
+{{ utc_datetime }}
+```
diff --git a/skyvern/forge/prompts/skyvern/single-select-action.j2 b/skyvern/forge/prompts/skyvern/single-select-action.j2
@@ -0,0 +1,43 @@
+Your are here to help the user to perform an SELECT_OPTION action on the web page. Use the user instruction, the content of the elements parsed from the page, the screenshots of the page, and user details to determine which element to select.
+Each actionable element is tagged with an ID. Only take the action on the elements provided in the HTML elements, do not image any new element.
+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
+
+Reply in JSON format with the following keys:
+{
+    "page_info": str, // Think step by step. Describe all the useful information in the page related to the user instruction.
+    "thoughts": str, // Think step by step. What information makes you believe which element to select. Use information you see on the site to explain.
+    "actions": array // You are supposed to give only one action("SELECT_OPTION") in the action list. Here's the format of the action:
+    [{
+        "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.
+        "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
+        "action_type": str, // It's a string enum: "SELECT_OPTION". "SELECT_OPTION" is an element you'd like to select an option from.
+        "id": str, // The id of the element to take action on. The id has to be one from the elements list.
+        "option": {  // The option to select.
+            "label": str, // the label of the option if any. MAKE SURE YOU USE THIS LABEL TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION LABEL HERE
+            "index": int, // the index corresponding to the option index under the select element.
+            "value": str // the value of the option. MAKE SURE YOU USE THIS VALUE TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION VALUE HERE
+        },
+    }]
+}
+
+The URL of the page you're on right now is `{{ current_url }}`.
+
+HTML elements from `{{ current_url }}`:
+```
+{{ elements }}
+```
+
+User instruction:
+```
+{{ navigation_goal }}
+```
+
+User details:
+```
+{{ navigation_payload_str }}
+```
+
+Current datetime in UTC, YYYY-MM-DD HH:MM format:
+```
+{{ utc_datetime }}
+```