From 10090e3a7f8892c4893f4d89414e2bf986a6e585 Mon Sep 17 00:00:00 2001 From: Shuchang Zheng Date: Mon, 25 Nov 2024 16:32:15 +0000 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=94=84=20synced=20local=20'skyvern/'?= =?UTF-8?q?=20with=20remote=20'skyvern/'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > [!IMPORTANT] > Add `NavigationNode` component to handle navigation tasks in workflows, integrating it into the workflow editor and YAML serialization. > > - **New Features**: > - Add `NavigationNode` component in `NavigationNode.tsx` to handle navigation tasks in workflows. > - Introduce `NavigationNodeData` and `NavigationNode` types in `NavigationNode/types.ts`. > - Add `NavigationBlock` type in `workflowTypes.ts` and `workflowYamlTypes.ts`. > - **Integration**: > - Update `nodeTypes` in `index.ts` to include `NavigationNode`. > - Add `NavigationNode` to `WorkflowNodeLibraryPanel.tsx` for user selection. > - Modify `workflowEditorUtils.ts` to handle `NavigationNode` in node creation and YAML conversion. > - **Refactoring**: > - Move common tooltip content and placeholders to `constants.ts` for reuse. > - Remove redundant `helpTooltipContent` and `fieldPlaceholders` from `ActionNode/types.ts`. > > This description was created by [Ellipsis](https://www.ellipsis.dev?ref=Skyvern-AI%2Fskyvern-cloud&utm_source=github&utm_medium=referral) for 41bb7fbaa5d94e484642dbbc4434b7bc08100931. It will automatically update as commits are pushed. --- skyvern/forge/agent.py | 6 ++- skyvern/forge/agent_functions.py | 63 ++++++++++++++------------------ 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py index 893c776096..6b646ab19a 100644 --- a/skyvern/forge/agent.py +++ b/skyvern/forge/agent.py @@ -624,6 +624,7 @@ async def agent_step( task, step, browser_state, + organization, ) detailed_agent_step_output.scraped_page = scraped_page detailed_agent_step_output.extract_action_prompt = extract_action_prompt @@ -1097,6 +1098,7 @@ async def _scrape_with_type( step: Step, browser_state: BrowserState, scrape_type: ScrapeType, + organization: Organization | None = None, ) -> ScrapedPage: if scrape_type == ScrapeType.NORMAL: pass @@ -1119,7 +1121,7 @@ async def _scrape_with_type( return await scrape_website( browser_state, task.url, - app.AGENT_FUNCTION.cleanup_element_tree_factory(task=task, step=step), + app.AGENT_FUNCTION.cleanup_element_tree_factory(task=task, step=step, organization=organization), scrape_exclude=app.scrape_exclude, ) @@ -1128,6 +1130,7 @@ async def _build_and_record_step_prompt( task: Task, step: Step, browser_state: BrowserState, + organization: Organization | None = None, ) -> tuple[ScrapedPage, str]: # start the async tasks while running scrape_website self.async_operation_pool.run_operation(task.task_id, AgentPhase.scrape) @@ -1145,6 +1148,7 @@ async def _build_and_record_step_prompt( step=step, browser_state=browser_state, scrape_type=scrape_type, + organization=organization, ) break except FailedToTakeScreenshot as e: diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py index e2692fe4d4..d8152089bb 100644 --- a/skyvern/forge/agent_functions.py +++ b/skyvern/forge/agent_functions.py @@ -96,19 +96,13 @@ def _remove_skyvern_attributes(element: Dict) -> Dict: return element_copied -async def _convert_svg_to_string( - element: Dict, - task: Task | None = None, - step: Step | None = None, -) -> None: +async def _convert_svg_to_string(task: Task, step: Step, organization: Organization | None, element: Dict) -> None: if element.get("tagName") != "svg": return if element.get("isDropped", False): return - task_id = task.task_id if task else None - step_id = step.step_id if step else None element_id = element.get("id", "") svg_element = _remove_skyvern_attributes(element) svg_html = json_to_html(svg_element) @@ -123,8 +117,8 @@ async def _convert_svg_to_string( except Exception: LOG.warning( "Failed to loaded SVG cache", - task_id=task_id, - step_id=step_id, + task_id=task.task_id, + step_id=step.step_id, exc_info=True, key=svg_key, ) @@ -137,8 +131,8 @@ async def _convert_svg_to_string( LOG.warning( "SVG element is too large to convert, going to drop the svg element.", element_id=element_id, - task_id=task_id, - step_id=step_id, + task_id=task.task_id, + step_id=step.step_id, length=len(svg_html), ) del element["children"] @@ -160,8 +154,8 @@ async def _convert_svg_to_string( except Exception: LOG.exception( "Failed to convert SVG to string shape by secondary llm. Will retry if haven't met the max try attempt after 3s.", - task_id=task_id, - step_id=step_id, + task_id=task.task_id, + step_id=step.step_id, element_id=element_id, retry=retry, ) @@ -176,15 +170,10 @@ async def _convert_svg_to_string( async def _convert_css_shape_to_string( - frame: Page | Frame, - element: Dict, - task: Task | None = None, - step: Step | None = None, + task: Task, step: Step, organization: Organization | None, frame: Page | Frame, element: Dict ) -> None: element_id: str = element.get("id", "") - task_id = task.task_id if task else None - step_id = step.step_id if step else None shape_element = _remove_skyvern_attributes(element) svg_html = json_to_html(shape_element) hash_object = hashlib.sha256() @@ -198,8 +187,8 @@ async def _convert_css_shape_to_string( except Exception: LOG.warning( "Failed to loaded CSS shape cache", - task_id=task_id, - step_id=step_id, + task_id=task.task_id, + step_id=step.step_id, exc_info=True, key=shape_key, ) @@ -212,8 +201,8 @@ async def _convert_css_shape_to_string( if await locater.count() == 0: LOG.info( "No locater found to convert css shape", - task_id=task_id, - step_id=step_id, + task_id=task.task_id, + step_id=step.step_id, element_id=element_id, ) return None @@ -221,8 +210,8 @@ async def _convert_css_shape_to_string( if await locater.count() > 1: LOG.info( "multiple locaters found to convert css shape", - task_id=task_id, - step_id=step_id, + task_id=task.task_id, + step_id=step.step_id, element_id=element_id, ) return None @@ -246,8 +235,8 @@ async def _convert_css_shape_to_string( except Exception: LOG.exception( "Failed to convert css shape to string shape by secondary llm. Will retry if haven't met the max try attempt after 3s.", - task_id=task_id, - step_id=step_id, + task_id=task.task_id, + step_id=step.step_id, element_id=element_id, retry=retry, ) @@ -255,16 +244,16 @@ async def _convert_css_shape_to_string( else: LOG.info( "Max css shape convertion retry, going to abort the convertion.", - task_id=task_id, - step_id=step_id, + task_id=task.task_id, + step_id=step.step_id, element_id=element_id, ) return None except Exception: LOG.warning( "Failed to convert css shape to string shape by LLM", - task_id=task_id, - step_id=step_id, + task_id=task.task_id, + step_id=step.step_id, element_id=element_id, exc_info=True, ) @@ -327,8 +316,9 @@ async def generate_async_operations( def cleanup_element_tree_factory( self, - task: Task | None = None, - step: Step | None = None, + task: Task, + step: Step, + organization: Organization | None = None, ) -> CleanupElementTreeFunc: async def cleanup_element_tree_func(frame: Page | Frame, url: str, element_tree: list[dict]) -> list[dict]: """ @@ -345,14 +335,15 @@ async def cleanup_element_tree_func(frame: Page | Frame, url: str, element_tree: while queue: queue_ele = queue.pop(0) _remove_rect(queue_ele) - await _convert_svg_to_string(queue_ele, task, step) + await _convert_svg_to_string(task, step, organization, queue_ele) if _should_css_shape_convert(element=queue_ele): await _convert_css_shape_to_string( - frame=frame, - element=queue_ele, task=task, step=step, + organization=organization, + frame=frame, + element=queue_ele, ) # TODO: we can come back to test removing the unique_id From 3d54048d74f4735be1918aa32db1a4c0d914d220 Mon Sep 17 00:00:00 2001 From: Shuchang Zheng Date: Mon, 25 Nov 2024 16:32:15 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=94=84=20synced=20local=20'skyvern-fr?= =?UTF-8?q?ontend/src/'=20with=20remote=20'skyvern-frontend/src/'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > [!IMPORTANT] > Add `NavigationNode` component to handle navigation tasks in workflows, integrating it into the workflow editor and YAML serialization. > > - **New Features**: > - Add `NavigationNode` component in `NavigationNode.tsx` to handle navigation tasks in workflows. > - Introduce `NavigationNodeData` and `NavigationNode` types in `NavigationNode/types.ts`. > - Add `NavigationBlock` type in `workflowTypes.ts` and `workflowYamlTypes.ts`. > - **Integration**: > - Update `nodeTypes` in `index.ts` to include `NavigationNode`. > - Add `NavigationNode` to `WorkflowNodeLibraryPanel.tsx` for user selection. > - Modify `workflowEditorUtils.ts` to handle `NavigationNode` in node creation and YAML conversion. > - **Refactoring**: > - Move common tooltip content and placeholders to `constants.ts` for reuse. > - Remove redundant `helpTooltipContent` and `fieldPlaceholders` from `ActionNode/types.ts`. > > This description was created by [Ellipsis](https://www.ellipsis.dev?ref=Skyvern-AI%2Fskyvern-cloud&utm_source=github&utm_medium=referral) for 41bb7fbaa5d94e484642dbbc4434b7bc08100931. It will automatically update as commits are pushed. --- .../src/components/icons/RobotIcon.tsx | 23 ++ .../src/routes/workflows/editor/constants.ts | 30 ++ .../editor/nodes/ActionNode/ActionNode.tsx | 45 +- .../editor/nodes/ActionNode/types.ts | 31 -- .../nodes/NavigationNode/NavigationNode.tsx | 383 ++++++++++++++++++ .../editor/nodes/NavigationNode/types.ts | 39 ++ .../routes/workflows/editor/nodes/index.ts | 8 +- .../panels/WorkflowNodeLibraryPanel.tsx | 7 + .../workflows/editor/workflowEditorUtils.ts | 86 +++- .../routes/workflows/types/workflowTypes.ts | 36 +- .../workflows/types/workflowYamlTypes.ts | 19 +- 11 files changed, 650 insertions(+), 57 deletions(-) create mode 100644 skyvern-frontend/src/components/icons/RobotIcon.tsx create mode 100644 skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx create mode 100644 skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/types.ts diff --git a/skyvern-frontend/src/components/icons/RobotIcon.tsx b/skyvern-frontend/src/components/icons/RobotIcon.tsx new file mode 100644 index 0000000000..05420fffd6 --- /dev/null +++ b/skyvern-frontend/src/components/icons/RobotIcon.tsx @@ -0,0 +1,23 @@ +type Props = { + className?: string; +}; + +function RobotIcon({ className }: Props) { + return ( + + + + ); +} + +export { RobotIcon }; diff --git a/skyvern-frontend/src/routes/workflows/editor/constants.ts b/skyvern-frontend/src/routes/workflows/editor/constants.ts index 4a38ffed27..bc219a474c 100644 --- a/skyvern-frontend/src/routes/workflows/editor/constants.ts +++ b/skyvern-frontend/src/routes/workflows/editor/constants.ts @@ -14,3 +14,33 @@ export const SMTP_USERNAME_AWS_KEY = "SKYVERN_SMTP_USERNAME_SES"; export const SMTP_PASSWORD_AWS_KEY = "SKYVERN_SMTP_PASSWORD_SES"; export const EMAIL_BLOCK_SENDER = "hello@skyvern.com"; + +export const commonHelpTooltipContent = { + maxRetries: + "Specify how many times you would like a task to retry upon failure.", + maxStepsOverride: + "Specify the maximum number of steps a task can take in total.", + completeOnDownload: + "Allow Skyvern to auto-complete the task when it downloads a file.", + fileSuffix: + "A file suffix that's automatically added to all downloaded files.", + errorCodeMapping: + "Knowing about why a task terminated can be important, specify error messages here.", + totpVerificationUrl: + "If you have an internal system for storing TOTP codes, link the endpoint here.", + totpIdentifier: + "If you are running multiple tasks or workflows at once, you will need to give the task an identifier to know that this TOTP goes with this task.", + continueOnFailure: + "Allow the workflow to continue if it encounters a failure.", + cacheActions: "Cache the actions of this task.", +} as const; + +export const commonFieldPlaceholders = { + url: "https://", + navigationGoal: 'Input text into "Name" field.', + maxRetries: "Default: 3", + maxStepsOverride: "Default: 10", + downloadSuffix: "Add an ID for downloaded files", + totpVerificationUrl: "Provide your 2FA endpoint", + totpIdentifier: "Add an ID that links your TOTP to the task", +} as const; diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx index ab8fd53e48..f059f5d23c 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx @@ -13,15 +13,23 @@ import { Handle, NodeProps, Position, useReactFlow } from "@xyflow/react"; import { useState } from "react"; import { EditableNodeTitle } from "../components/EditableNodeTitle"; import { NodeActionMenu } from "../NodeActionMenu"; -import { helpTooltipContent, type ActionNode } from "./types"; +import type { ActionNode } from "./types"; import { HelpTooltip } from "@/components/HelpTooltip"; import { Input } from "@/components/ui/input"; -import { fieldPlaceholders } from "./types"; import { Checkbox } from "@/components/ui/checkbox"; import { errorMappingExampleValue } from "../types"; import { CodeEditor } from "@/routes/workflows/components/CodeEditor"; import { Switch } from "@/components/ui/switch"; import { ClickIcon } from "@/components/icons/ClickIcon"; +import { + commonFieldPlaceholders, + commonHelpTooltipContent, +} from "../../constants"; + +const navigationGoalTooltip = + "Specify a single step or action you'd like Skyvern to complete. Actions are one-off tasks like filling a field or interacting with a specific element on the page.\n\nCurrently supported actions are click, input text, upload file, and select."; + +const navigationGoalPlaceholder = 'Input text into "Name" field.'; function ActionNode({ id, data }: NodeProps) { const { updateNodeData } = useReactFlow(); @@ -91,7 +99,7 @@ function ActionNode({ id, data }: NodeProps) {
- +
{ @@ -101,6 +109,7 @@ function ActionNode({ id, data }: NodeProps) { handleChange("navigationGoal", event.target.value); }} value={inputs.navigationGoal} + placeholder={navigationGoalPlaceholder} className="nopan text-xs" />
@@ -117,11 +126,13 @@ function ActionNode({ id, data }: NodeProps) { - + ) { Error Messages ) { Continue on Failure
@@ -207,7 +218,9 @@ function ActionNode({ id, data }: NodeProps) { - +
) { Complete on Download
@@ -248,11 +261,13 @@ function ActionNode({ id, data }: NodeProps) { - +
{ @@ -270,7 +285,7 @@ function ActionNode({ id, data }: NodeProps) { 2FA Verification URL ) { handleChange("totpVerificationUrl", event.target.value); }} value={inputs.totpVerificationUrl ?? ""} - placeholder={fieldPlaceholders["totpVerificationUrl"]} + placeholder={commonFieldPlaceholders["totpVerificationUrl"]} className="nopan text-xs" /> @@ -291,7 +306,7 @@ function ActionNode({ id, data }: NodeProps) { 2FA Identifier ) { handleChange("totpIdentifier", event.target.value); }} value={inputs.totpIdentifier ?? ""} - placeholder={fieldPlaceholders["totpIdentifier"]} + placeholder={commonFieldPlaceholders["totpIdentifier"]} className="nopan text-xs" /> diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/types.ts index e564c98bec..71cef10780 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/types.ts +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/types.ts @@ -35,34 +35,3 @@ export const actionNodeDefaultData: ActionNodeData = { export function isActionNode(node: Node): node is ActionNode { return node.type === "action"; } - -export const helpTooltipContent = { - navigationGoal: - "Specify a single step or action you'd like Skyvern to complete. Actions are one-off tasks like filling a field or interacting with a specific element on the page.\n\nCurrently supported actions are click, input text, upload file, and select.", - maxRetries: - "Specify how many times you would like a task to retry upon failure.", - maxStepsOverride: - "Specify the maximum number of steps a task can take in total.", - completeOnDownload: - "Allow Skyvern to auto-complete the task when it downloads a file.", - fileSuffix: - "A file suffix that's automatically added to all downloaded files.", - errorCodeMapping: - "Knowing about why a task terminated can be important, specify error messages here.", - totpVerificationUrl: - "If you have an internal system for storing TOTP codes, link the endpoint here.", - totpIdentifier: - "If you are running multiple tasks or workflows at once, you will need to give the task an identifier to know that this TOTP goes with this task.", - continueOnFailure: - "Allow the workflow to continue if it encounters a failure.", - cacheActions: "Cache the actions of this task.", -} as const; - -export const fieldPlaceholders = { - navigationGoal: 'Input text into "Name" field.', - maxRetries: "Default: 3", - maxStepsOverride: "Default: 10", - downloadSuffix: "Add an ID for downloaded files", - totpVerificationUrl: "Provide your 2FA endpoint", - totpIdentifier: "Add an ID that links your TOTP to the task", -}; diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx new file mode 100644 index 0000000000..e780a1c777 --- /dev/null +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx @@ -0,0 +1,383 @@ +import { AutoResizingTextarea } from "@/components/AutoResizingTextarea/AutoResizingTextarea"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@/components/ui/accordion"; +import { Label } from "@/components/ui/label"; +import { Separator } from "@/components/ui/separator"; +import { useDeleteNodeCallback } from "@/routes/workflows/hooks/useDeleteNodeCallback"; +import { useNodeLabelChangeHandler } from "@/routes/workflows/hooks/useLabelChangeHandler"; +import { Handle, NodeProps, Position, useReactFlow } from "@xyflow/react"; +import { useState } from "react"; +import { EditableNodeTitle } from "../components/EditableNodeTitle"; +import { NodeActionMenu } from "../NodeActionMenu"; +import { HelpTooltip } from "@/components/HelpTooltip"; +import { Input } from "@/components/ui/input"; +import { Checkbox } from "@/components/ui/checkbox"; +import { errorMappingExampleValue } from "../types"; +import { CodeEditor } from "@/routes/workflows/components/CodeEditor"; +import { Switch } from "@/components/ui/switch"; +import type { NavigationNode } from "./types"; +import { + commonFieldPlaceholders, + commonHelpTooltipContent, +} from "../../constants"; +import { RobotIcon } from "@/components/icons/RobotIcon"; + +const urlTooltip = + "The URL Skyvern is navigating to. Leave this field blank to pick up from where the last block left off."; +const urlPlaceholder = "https://"; +const navigationGoalTooltip = + "Give Skyvern an objective. Make sure to include when the task is complete, when it should self-terminate, and any guardrails."; +const navigationGoalPlaceholder = "Tell Skyvern what to do."; + +function NavigationNode({ id, data }: NodeProps) { + const { updateNodeData } = useReactFlow(); + const { editable } = data; + const [label, setLabel] = useNodeLabelChangeHandler({ + id, + initialValue: data.label, + }); + const [inputs, setInputs] = useState({ + url: data.url, + navigationGoal: data.navigationGoal, + errorCodeMapping: data.errorCodeMapping, + maxRetries: data.maxRetries, + maxStepsOverride: data.maxStepsOverride, + allowDownloads: data.allowDownloads, + continueOnFailure: data.continueOnFailure, + cacheActions: data.cacheActions, + downloadSuffix: data.downloadSuffix, + totpVerificationUrl: data.totpVerificationUrl, + totpIdentifier: data.totpIdentifier, + }); + const deleteNodeCallback = useDeleteNodeCallback(); + + function handleChange(key: string, value: unknown) { + if (!editable) { + return; + } + setInputs({ ...inputs, [key]: value }); + updateNodeData(id, { [key]: value }); + } + + return ( +
+ + +
+
+
+
+ +
+
+ + Navigation Block +
+
+ { + deleteNodeCallback(id); + }} + /> +
+
+
+
+ + +
+ { + if (!editable) { + return; + } + handleChange("url", event.target.value); + }} + value={inputs.url} + placeholder={urlPlaceholder} + className="nopan text-xs" + /> +
+
+
+ + +
+ { + if (!editable) { + return; + } + handleChange("navigationGoal", event.target.value); + }} + value={inputs.navigationGoal} + placeholder={navigationGoalPlaceholder} + className="nopan text-xs" + /> +
+
+ + + + + Advanced Settings + + +
+
+
+ + +
+ { + if (!editable) { + return; + } + const value = + event.target.value === "" + ? null + : Number(event.target.value); + handleChange("maxRetries", value); + }} + /> +
+
+
+ + +
+ { + if (!editable) { + return; + } + const value = + event.target.value === "" + ? null + : Number(event.target.value); + handleChange("maxStepsOverride", value); + }} + /> +
+
+
+
+ + +
+ { + if (!editable) { + return; + } + handleChange( + "errorCodeMapping", + checked + ? JSON.stringify(errorMappingExampleValue, null, 2) + : "null", + ); + }} + /> +
+ {inputs.errorCodeMapping !== "null" && ( +
+ { + if (!editable) { + return; + } + handleChange("errorCodeMapping", value); + }} + className="nowheel nopan" + fontSize={8} + /> +
+ )} +
+ +
+
+ + +
+
+ { + if (!editable) { + return; + } + handleChange("continueOnFailure", checked); + }} + /> +
+
+
+
+ + +
+
+ { + if (!editable) { + return; + } + handleChange("cacheActions", checked); + }} + /> +
+
+ +
+
+ + +
+
+ { + if (!editable) { + return; + } + handleChange("allowDownloads", checked); + }} + /> +
+
+
+
+ + +
+ { + if (!editable) { + return; + } + handleChange("downloadSuffix", event.target.value); + }} + /> +
+ +
+
+ + +
+ { + if (!editable) { + return; + } + handleChange("totpVerificationUrl", event.target.value); + }} + value={inputs.totpVerificationUrl ?? ""} + placeholder={commonFieldPlaceholders["totpVerificationUrl"]} + className="nopan text-xs" + /> +
+
+
+ + +
+ { + if (!editable) { + return; + } + handleChange("totpIdentifier", event.target.value); + }} + value={inputs.totpIdentifier ?? ""} + placeholder={commonFieldPlaceholders["totpIdentifier"]} + className="nopan text-xs" + /> +
+
+ + + +
+
+ ); +} + +export { NavigationNode }; diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/types.ts new file mode 100644 index 0000000000..8f19a7474d --- /dev/null +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/types.ts @@ -0,0 +1,39 @@ +import type { Node } from "@xyflow/react"; +import { NodeBaseData } from "../types"; + +export type NavigationNodeData = NodeBaseData & { + url: string; + navigationGoal: string; + errorCodeMapping: string; + maxRetries: number | null; + maxStepsOverride: number | null; + allowDownloads: boolean; + downloadSuffix: string | null; + parameterKeys: Array; + totpVerificationUrl: string | null; + totpIdentifier: string | null; + cacheActions: boolean; +}; + +export type NavigationNode = Node; + +export const navigationNodeDefaultData: NavigationNodeData = { + label: "", + url: "", + navigationGoal: "", + errorCodeMapping: "null", + maxRetries: null, + maxStepsOverride: null, + allowDownloads: false, + downloadSuffix: null, + editable: true, + parameterKeys: [], + totpVerificationUrl: null, + totpIdentifier: null, + continueOnFailure: false, + cacheActions: false, +} as const; + +export function isNavigationNode(node: Node): node is NavigationNode { + return node.type === "navigation"; +} diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts index 82222e963c..7b1525098a 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/index.ts @@ -23,6 +23,8 @@ import type { ValidationNode } from "./ValidationNode/types"; import { ValidationNode as ValidationNodeComponent } from "./ValidationNode/ValidationNode"; import type { ActionNode } from "./ActionNode/types"; import { ActionNode as ActionNodeComponent } from "./ActionNode/ActionNode"; +import { NavigationNode } from "./NavigationNode/types"; +import { NavigationNode as NavigationNodeComponent } from "./NavigationNode/NavigationNode"; export type UtilityNode = StartNode | NodeAdderNode; @@ -36,7 +38,8 @@ export type WorkflowBlockNode = | UploadNode | DownloadNode | ValidationNode - | ActionNode; + | ActionNode + | NavigationNode; export function isUtilityNode(node: AppNode): node is UtilityNode { return node.type === "nodeAdder" || node.type === "start"; @@ -61,4 +64,5 @@ export const nodeTypes = { start: memo(StartNodeComponent), validation: memo(ValidationNodeComponent), action: memo(ActionNodeComponent), -}; + navigation: memo(NavigationNodeComponent), +} as const; diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx index e4c873027d..5268f8be03 100644 --- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowNodeLibraryPanel.tsx @@ -14,6 +14,7 @@ import { WorkflowBlockNode } from "../nodes"; import { AddNodeProps } from "../FlowRenderer"; import { ClickIcon } from "@/components/icons/ClickIcon"; import { ScrollArea, ScrollAreaViewport } from "@/components/ui/scroll-area"; +import { RobotIcon } from "@/components/icons/RobotIcon"; const nodeLibraryItems: Array<{ nodeType: NonNullable; @@ -83,6 +84,12 @@ const nodeLibraryItems: Array<{ title: "Action Block", description: "Take a single action", }, + { + nodeType: "navigation", + icon: , + title: "Navigation Block", + description: "Navigate on the page", + }, ]; type Props = { diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts index ceb9454b3d..2a584983fa 100644 --- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts +++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts @@ -23,6 +23,7 @@ import { TextPromptBlockYAML, UploadToS3BlockYAML, ValidationBlockYAML, + NavigationBlockYAML, WorkflowCreateYAMLRequest, } from "../types/workflowYamlTypes"; import { @@ -52,6 +53,7 @@ import { NodeBaseData } from "./nodes/types"; import { uploadNodeDefaultData } from "./nodes/UploadNode/types"; import { validationNodeDefaultData } from "./nodes/ValidationNode/types"; import { actionNodeDefaultData } from "./nodes/ActionNode/types"; +import { navigationNodeDefaultData } from "./nodes/NavigationNode/types"; export const NEW_NODE_LABEL_PREFIX = "block_"; @@ -197,6 +199,27 @@ function convertToNode( }, }; } + case "navigation": { + return { + ...identifiers, + ...common, + type: "navigation", + data: { + ...commonData, + url: block.url ?? "", + navigationGoal: block.navigation_goal ?? "", + errorCodeMapping: JSON.stringify(block.error_code_mapping, null, 2), + allowDownloads: block.complete_on_download ?? false, + downloadSuffix: block.download_suffix ?? null, + maxRetries: block.max_retries ?? null, + parameterKeys: block.parameters.map((p) => p.key), + totpIdentifier: block.totp_identifier ?? null, + totpVerificationUrl: block.totp_verification_url ?? null, + cacheActions: block.cache_actions, + maxStepsOverride: block.max_steps_per_run ?? null, + }, + }; + } case "code": { return { ...identifiers, @@ -509,6 +532,17 @@ function createNode( }, }; } + case "navigation": { + return { + ...identifiers, + ...common, + type: "navigation", + data: { + ...navigationNodeDefaultData, + label, + }, + }; + } case "loop": { return { ...identifiers, @@ -662,6 +696,28 @@ function getWorkflowBlock(node: WorkflowBlockNode): BlockYAML { cache_actions: node.data.cacheActions, }; } + case "navigation": { + return { + ...base, + block_type: "navigation", + navigation_goal: node.data.navigationGoal, + error_code_mapping: JSONParseSafe(node.data.errorCodeMapping) as Record< + string, + string + > | null, + url: node.data.url, + ...(node.data.maxRetries !== null && { + max_retries: node.data.maxRetries, + }), + max_steps_per_run: node.data.maxStepsOverride, + complete_on_download: node.data.allowDownloads, + download_suffix: node.data.downloadSuffix, + parameter_keys: node.data.parameterKeys, + totp_identifier: node.data.totpIdentifier, + totp_verification_url: node.data.totpVerificationUrl, + cache_actions: node.data.cacheActions, + }; + } case "sendEmail": { return { ...base, @@ -1042,7 +1098,7 @@ function getAvailableOutputParameterKeys( return outputParameterKeys; } -function convertParameters( +function convertParametersToParameterYAML( parameters: Array>, ): Array { return parameters.map((parameter) => { @@ -1105,7 +1161,9 @@ function convertParameters( }); } -function convertBlocks(blocks: Array): Array { +function convertBlocksToBlockYAML( + blocks: Array, +): Array { return blocks.map((block) => { const base = { label: block.label, @@ -1160,12 +1218,30 @@ function convertBlocks(blocks: Array): Array { }; return blockYaml; } + case "navigation": { + const blockYaml: NavigationBlockYAML = { + ...base, + block_type: "navigation", + url: block.url, + navigation_goal: block.navigation_goal, + error_code_mapping: block.error_code_mapping, + max_retries: block.max_retries, + max_steps_per_run: block.max_steps_per_run, + complete_on_download: block.complete_on_download, + download_suffix: block.download_suffix, + parameter_keys: block.parameters.map((p) => p.key), + totp_identifier: block.totp_identifier, + totp_verification_url: block.totp_verification_url, + cache_actions: block.cache_actions, + }; + return blockYaml; + } case "for_loop": { const blockYaml: ForLoopBlockYAML = { ...base, block_type: "for_loop", loop_over_parameter_key: block.loop_over.key, - loop_blocks: convertBlocks(block.loop_blocks), + loop_blocks: convertBlocksToBlockYAML(block.loop_blocks), }; return blockYaml; } @@ -1244,8 +1320,8 @@ function convert(workflow: WorkflowApiResponse): WorkflowCreateYAMLRequest { webhook_callback_url: workflow.webhook_callback_url, totp_verification_url: workflow.totp_verification_url, workflow_definition: { - parameters: convertParameters(userParameters), - blocks: convertBlocks(workflow.workflow_definition.blocks), + parameters: convertParametersToParameterYAML(userParameters), + blocks: convertBlocksToBlockYAML(workflow.workflow_definition.blocks), }, is_saved_task: workflow.is_saved_task, }; diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts index fbdcf9352b..83b323ce93 100644 --- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts +++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts @@ -111,7 +111,9 @@ export const WorkflowBlockType = { SendEmail: "send_email", FileURLParser: "file_url_parser", Validation: "validation", -}; + Action: "action", + Navigation: "navigation", +} as const; export type WorkflowBlockType = (typeof WorkflowBlockType)[keyof typeof WorkflowBlockType]; @@ -198,9 +200,36 @@ export type ValidationBlock = WorkflowBlockBase & { parameters: Array; }; -export type ActionBlock = Omit & { +export type ActionBlock = WorkflowBlockBase & { block_type: "action"; + url: string | null; + title: string; + navigation_goal: string | null; + error_code_mapping: Record | null; + max_retries?: number; + max_steps_per_run?: number | null; parameters: Array; + complete_on_download?: boolean; + download_suffix?: string | null; + totp_verification_url?: string | null; + totp_identifier?: string | null; + cache_actions: boolean; +}; + +export type NavigationBlock = WorkflowBlockBase & { + block_type: "navigation"; + url: string | null; + title: string; + navigation_goal: string | null; + error_code_mapping: Record | null; + max_retries?: number; + max_steps_per_run?: number | null; + parameters: Array; + complete_on_download?: boolean; + download_suffix?: string | null; + totp_verification_url?: string | null; + totp_identifier?: string | null; + cache_actions: boolean; }; export type WorkflowBlock = @@ -213,7 +242,8 @@ export type WorkflowBlock = | SendEmailBlock | FileURLParserBlock | ValidationBlock - | ActionBlock; + | ActionBlock + | NavigationBlock; export type WorkflowDefinition = { parameters: Array; diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts index deb440b9e3..5ac3de8fef 100644 --- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts +++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts @@ -77,6 +77,7 @@ const BlockTypes = { FILE_URL_PARSER: "file_url_parser", VALIDATION: "validation", ACTION: "action", + NAVIGATION: "navigation", } as const; export type BlockType = (typeof BlockTypes)[keyof typeof BlockTypes]; @@ -91,7 +92,8 @@ export type BlockYAML = | FileUrlParserBlockYAML | ForLoopBlockYAML | ValidationBlockYAML - | ActionBlockYAML; + | ActionBlockYAML + | NavigationBlockYAML; export type BlockYAMLBase = { block_type: BlockType; @@ -139,6 +141,21 @@ export type ActionBlockYAML = BlockYAMLBase & { cache_actions: boolean; }; +export type NavigationBlockYAML = BlockYAMLBase & { + block_type: "navigation"; + url: string | null; + navigation_goal: string | null; + error_code_mapping: Record | null; + max_retries?: number; + max_steps_per_run?: number | null; + parameter_keys?: Array | null; + complete_on_download?: boolean; + download_suffix?: string | null; + totp_verification_url?: string | null; + totp_identifier?: string | null; + cache_actions: boolean; +}; + export type CodeBlockYAML = BlockYAMLBase & { block_type: "code"; code: string; From a5bca43159d0b57a062f99c8527ac4606c76016e Mon Sep 17 00:00:00 2001 From: Muhammed Salih Altun Date: Mon, 25 Nov 2024 19:35:29 +0300 Subject: [PATCH 3/3] Remove backend sync as unrelated --- skyvern/forge/agent.py | 6 +-- skyvern/forge/agent_functions.py | 63 ++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py index 6b646ab19a..893c776096 100644 --- a/skyvern/forge/agent.py +++ b/skyvern/forge/agent.py @@ -624,7 +624,6 @@ async def agent_step( task, step, browser_state, - organization, ) detailed_agent_step_output.scraped_page = scraped_page detailed_agent_step_output.extract_action_prompt = extract_action_prompt @@ -1098,7 +1097,6 @@ async def _scrape_with_type( step: Step, browser_state: BrowserState, scrape_type: ScrapeType, - organization: Organization | None = None, ) -> ScrapedPage: if scrape_type == ScrapeType.NORMAL: pass @@ -1121,7 +1119,7 @@ async def _scrape_with_type( return await scrape_website( browser_state, task.url, - app.AGENT_FUNCTION.cleanup_element_tree_factory(task=task, step=step, organization=organization), + app.AGENT_FUNCTION.cleanup_element_tree_factory(task=task, step=step), scrape_exclude=app.scrape_exclude, ) @@ -1130,7 +1128,6 @@ async def _build_and_record_step_prompt( task: Task, step: Step, browser_state: BrowserState, - organization: Organization | None = None, ) -> tuple[ScrapedPage, str]: # start the async tasks while running scrape_website self.async_operation_pool.run_operation(task.task_id, AgentPhase.scrape) @@ -1148,7 +1145,6 @@ async def _build_and_record_step_prompt( step=step, browser_state=browser_state, scrape_type=scrape_type, - organization=organization, ) break except FailedToTakeScreenshot as e: diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py index d8152089bb..e2692fe4d4 100644 --- a/skyvern/forge/agent_functions.py +++ b/skyvern/forge/agent_functions.py @@ -96,13 +96,19 @@ def _remove_skyvern_attributes(element: Dict) -> Dict: return element_copied -async def _convert_svg_to_string(task: Task, step: Step, organization: Organization | None, element: Dict) -> None: +async def _convert_svg_to_string( + element: Dict, + task: Task | None = None, + step: Step | None = None, +) -> None: if element.get("tagName") != "svg": return if element.get("isDropped", False): return + task_id = task.task_id if task else None + step_id = step.step_id if step else None element_id = element.get("id", "") svg_element = _remove_skyvern_attributes(element) svg_html = json_to_html(svg_element) @@ -117,8 +123,8 @@ async def _convert_svg_to_string(task: Task, step: Step, organization: Organizat except Exception: LOG.warning( "Failed to loaded SVG cache", - task_id=task.task_id, - step_id=step.step_id, + task_id=task_id, + step_id=step_id, exc_info=True, key=svg_key, ) @@ -131,8 +137,8 @@ async def _convert_svg_to_string(task: Task, step: Step, organization: Organizat LOG.warning( "SVG element is too large to convert, going to drop the svg element.", element_id=element_id, - task_id=task.task_id, - step_id=step.step_id, + task_id=task_id, + step_id=step_id, length=len(svg_html), ) del element["children"] @@ -154,8 +160,8 @@ async def _convert_svg_to_string(task: Task, step: Step, organization: Organizat except Exception: LOG.exception( "Failed to convert SVG to string shape by secondary llm. Will retry if haven't met the max try attempt after 3s.", - task_id=task.task_id, - step_id=step.step_id, + task_id=task_id, + step_id=step_id, element_id=element_id, retry=retry, ) @@ -170,10 +176,15 @@ async def _convert_svg_to_string(task: Task, step: Step, organization: Organizat async def _convert_css_shape_to_string( - task: Task, step: Step, organization: Organization | None, frame: Page | Frame, element: Dict + frame: Page | Frame, + element: Dict, + task: Task | None = None, + step: Step | None = None, ) -> None: element_id: str = element.get("id", "") + task_id = task.task_id if task else None + step_id = step.step_id if step else None shape_element = _remove_skyvern_attributes(element) svg_html = json_to_html(shape_element) hash_object = hashlib.sha256() @@ -187,8 +198,8 @@ async def _convert_css_shape_to_string( except Exception: LOG.warning( "Failed to loaded CSS shape cache", - task_id=task.task_id, - step_id=step.step_id, + task_id=task_id, + step_id=step_id, exc_info=True, key=shape_key, ) @@ -201,8 +212,8 @@ async def _convert_css_shape_to_string( if await locater.count() == 0: LOG.info( "No locater found to convert css shape", - task_id=task.task_id, - step_id=step.step_id, + task_id=task_id, + step_id=step_id, element_id=element_id, ) return None @@ -210,8 +221,8 @@ async def _convert_css_shape_to_string( if await locater.count() > 1: LOG.info( "multiple locaters found to convert css shape", - task_id=task.task_id, - step_id=step.step_id, + task_id=task_id, + step_id=step_id, element_id=element_id, ) return None @@ -235,8 +246,8 @@ async def _convert_css_shape_to_string( except Exception: LOG.exception( "Failed to convert css shape to string shape by secondary llm. Will retry if haven't met the max try attempt after 3s.", - task_id=task.task_id, - step_id=step.step_id, + task_id=task_id, + step_id=step_id, element_id=element_id, retry=retry, ) @@ -244,16 +255,16 @@ async def _convert_css_shape_to_string( else: LOG.info( "Max css shape convertion retry, going to abort the convertion.", - task_id=task.task_id, - step_id=step.step_id, + task_id=task_id, + step_id=step_id, element_id=element_id, ) return None except Exception: LOG.warning( "Failed to convert css shape to string shape by LLM", - task_id=task.task_id, - step_id=step.step_id, + task_id=task_id, + step_id=step_id, element_id=element_id, exc_info=True, ) @@ -316,9 +327,8 @@ async def generate_async_operations( def cleanup_element_tree_factory( self, - task: Task, - step: Step, - organization: Organization | None = None, + task: Task | None = None, + step: Step | None = None, ) -> CleanupElementTreeFunc: async def cleanup_element_tree_func(frame: Page | Frame, url: str, element_tree: list[dict]) -> list[dict]: """ @@ -335,15 +345,14 @@ async def cleanup_element_tree_func(frame: Page | Frame, url: str, element_tree: while queue: queue_ele = queue.pop(0) _remove_rect(queue_ele) - await _convert_svg_to_string(task, step, organization, queue_ele) + await _convert_svg_to_string(queue_ele, task, step) if _should_css_shape_convert(element=queue_ele): await _convert_css_shape_to_string( - task=task, - step=step, - organization=organization, frame=frame, element=queue_ele, + task=task, + step=step, ) # TODO: we can come back to test removing the unique_id