From f4cd32f16b7a8c04628b1af6206241a45a68edfc Mon Sep 17 00:00:00 2001 From: vyokky <7678676@qq.com> Date: Thu, 9 May 2024 15:02:10 +0800 Subject: [PATCH] bug fix --- ufo/agent/agent.py | 14 +- ufo/automator/ui_control/control_filter.py | 7 +- ufo/config/config_dev.yaml | 2 +- ufo/module/basic.py | 5 +- ufo/module/processors/processor.py | 12 +- ufo/prompter/agent_prompter.py | 16 +- .../lite/nonvisual/app_agent_example.yaml | 10 +- .../lite/nonvisual/host_agent_example.yaml | 12 +- .../lite/visual/app_agent_example.yaml | 10 +- .../lite/visual/host_agent_example.yaml | 12 +- .../examples/nonvisual/app_agent_example.yaml | 84 +++++----- .../nonvisual/host_agent_example.yaml | 146 ++++++++++++------ .../examples/visual/app_agent_example.yaml | 117 +++++++------- .../examples/visual/host_agent_example.yaml | 128 +++++++-------- ufo/prompts/share/base/app_agent.yaml | 8 +- ufo/prompts/share/base/host_agent.yaml | 8 +- ufo/prompts/share/lite/app_agent.yaml | 4 +- ufo/prompts/share/lite/host_agent.yaml | 8 +- 18 files changed, 324 insertions(+), 279 deletions(-) diff --git a/ufo/agent/agent.py b/ufo/agent/agent.py index c0474cf9..5c05589b 100644 --- a/ufo/agent/agent.py +++ b/ufo/agent/agent.py @@ -114,7 +114,7 @@ def message_constructor( request_history: str, action_history: str, control_info: str, - plan: str, + plan: List[str], request: str, include_last_screenshot: bool, ) -> list: @@ -127,7 +127,7 @@ def message_constructor( :param request_history: The request history. :param action_history: The action history. :param control_info: The control information. - :param plan: The plan. + :param plan: The plan list. :param request: The request. :param include_last_screenshot: The flag indicating whether to include the last screenshot. :return: The prompt message. @@ -188,7 +188,7 @@ def print_response(self, response_dict: Dict) -> None: ) utils.print_with_color("Status📊: {status}".format(status=status), "blue") utils.print_with_color( - "Next Plan📚: {plan}".format(plan=str(plan).replace("\\n", "\n")), "cyan" + "Next Plan📚: {plan}".format(plan="\n".join(plan)), "cyan" ) utils.print_with_color("Comment💬: {comment}".format(comment=comment), "green") @@ -447,7 +447,7 @@ def message_constructor( request_history: str, action_history: str, os_info: str, - plan: str, + plan: List[str], request: str, ) -> list: """ @@ -522,7 +522,7 @@ def print_response(self, response_dict: Dict) -> None: ) utils.print_with_color("Status📊: {status}".format(status=status), "blue") utils.print_with_color( - "Next Plan📚: {plan}".format(plan=str(plan).replace("\\n", "\n")), "cyan" + "Next Plan📚: {plan}".format(plan="\n".join(plan)), "cyan" ) utils.print_with_color("Comment💬: {comment}".format(comment=comment), "green") @@ -641,7 +641,7 @@ def message_constructor( request_history: str, action_history: str, control_info: str, - plan: str, + plan: List[str], request: str, include_last_screenshot: bool, ) -> list: @@ -688,7 +688,7 @@ def message_constructor( request_history: str, action_history: str, control_info: str, - plan: str, + plan: List[str], request: str, current_state: dict, state_diff: dict, diff --git a/ufo/automator/ui_control/control_filter.py b/ufo/automator/ui_control/control_filter.py index cdd288ff..613d79dc 100644 --- a/ufo/automator/ui_control/control_filter.py +++ b/ufo/automator/ui_control/control_filter.py @@ -56,7 +56,7 @@ def inplace_append_filtered_annotation_dict( return filtered_control_dict @staticmethod - def get_plans(plan, topk_plan): + def get_plans(plan: List[str], topk_plan: int) -> List[str]: """ Parses the given plan and returns a list of plans up to the specified topk_plan. @@ -67,8 +67,7 @@ def get_plans(plan, topk_plan): Returns: list: A list of plans up to the specified topk_plan. """ - plans = str(plan).split("\n")[:topk_plan] - return plans + return plan[:topk_plan] class BasicControlFilter: @@ -128,7 +127,7 @@ def control_filter(self, control_dicts, plans, **kwargs): pass @staticmethod - def plans_to_keywords(plans: list) -> list: + def plans_to_keywords(plans: List[str]) -> List[str]: """ Gets keywords from the plan. We only consider the words in the plan that are alphabetic or Chinese characters. diff --git a/ufo/config/config_dev.yaml b/ufo/config/config_dev.yaml index b151f0c1..449c07cf 100644 --- a/ufo/config/config_dev.yaml +++ b/ufo/config/config_dev.yaml @@ -47,7 +47,7 @@ DEMONSTRATION_PROMPT: "ufo/prompts/demonstration/demonstration_summary.yaml" DEMONSTRATION_SAVED_PATH: "vectordb/demonstration/" API_PROMPT: "ufo/prompts/share/base/api.yaml" # The prompt for the API -CLICK_API: "click" # The click API +CLICK_API: "click_input" # The click API INPUT_TEXT_API: "type_keys" # The input text API INPUT_TEXT_ENTER: False # whether to press enter after typing the text diff --git a/ufo/module/basic.py b/ufo/module/basic.py index 4fe575e5..42d5654d 100644 --- a/ufo/module/basic.py +++ b/ufo/module/basic.py @@ -31,6 +31,7 @@ ErrorState, MaxStepReachedState, NoneState, + Status, StatusToStateMapper, ) @@ -72,7 +73,7 @@ def __init__( self.app_agent = None # Status-related properties - self._status = "APP_SELECTION" + self._status = Status.APP_SELECTION # Application-related properties self.application = "" @@ -267,7 +268,7 @@ def __init__(self, task: str) -> None: self.app_agent = None # Status and state-related properties - self._status = "APP_SELECTION" + self._status = Status.APP_SELECTION self._state = StatusToStateMapper().get_appropriate_state(self._status) # Application-related properties diff --git a/ufo/module/processors/processor.py b/ufo/module/processors/processor.py index 771e1dbd..97f73787 100644 --- a/ufo/module/processors/processor.py +++ b/ufo/module/processors/processor.py @@ -131,7 +131,7 @@ def get_prompt_message(self) -> None: if agent_memory.length > 0: plan = agent_memory.get_latest_item().to_dict()["Plan"] else: - plan = "" + plan = [] # Construct the prompt message for the host agent. self._prompt_message = self.host_agent.message_constructor( @@ -438,7 +438,7 @@ def __init__( self._operation = None self._args = None self._image_url = [] - self.prev_plan = "" + self.prev_plan = [] self._control_reannotate = control_reannotate self.control_filter_factory = ControlFilterFactory() self.filtered_annotation_dict = None @@ -790,7 +790,7 @@ def _safe_guard_judgement(self, action: str, control_text: str) -> bool: return False # Handle the PENDING_AND_FINISH case - elif Status.FINISH in self._plan: + elif len(self._plan) > 0 and Status.FINISH in self._plan[0]: self._status = Status.FINISH return True @@ -810,9 +810,9 @@ def get_prev_plan(self) -> str: agent_memory = self.app_agent.memory if agent_memory.length > 0: - prev_plan = agent_memory.get_latest_item().to_dict()["Plan"].strip() + prev_plan = agent_memory.get_latest_item().to_dict()["Plan"] else: - prev_plan = "" + prev_plan = [] return prev_plan @@ -861,7 +861,7 @@ def get_filtered_annotation_dict( control_filter_type = configs["CONTROL_FILTER_TYPE"] topk_plan = configs["CONTROL_FILTER_TOP_K_PLAN"] - if len(control_filter_type) == 0 or self.prev_plan == "": + if len(control_filter_type) == 0 or self.prev_plan == []: return annotation_dict control_filter_type_lower = [ diff --git a/ufo/prompter/agent_prompter.py b/ufo/prompter/agent_prompter.py index eefddf19..8c2e994a 100644 --- a/ufo/prompter/agent_prompter.py +++ b/ufo/prompter/agent_prompter.py @@ -62,7 +62,7 @@ def user_prompt_construction( request_history: List[str], action_history: List[str], control_item: List[str], - prev_plan: str, + prev_plan: List[str], user_request: str, retrieved_docs: str = "", ) -> str: @@ -78,7 +78,7 @@ def user_prompt_construction( action_history=json.dumps(action_history), request_history=json.dumps(request_history), control_item=json.dumps(control_item), - prev_plan=prev_plan, + prev_plan=json.dumps(prev_plan), user_request=user_request, retrieved_docs=retrieved_docs, ) @@ -250,7 +250,7 @@ def user_prompt_construction( request_history: List[str], action_history: List[str], control_item: List[str], - prev_plan: str, + prev_plan: List[str], user_request: str, retrieved_docs: str = "", ) -> str: @@ -267,7 +267,7 @@ def user_prompt_construction( action_history=json.dumps(action_history), request_history=json.dumps(request_history), control_item=json.dumps(control_item), - prev_plan=prev_plan, + prev_plan=json.dumps(prev_plan), user_request=user_request, retrieved_docs=retrieved_docs, ) @@ -280,7 +280,7 @@ def user_content_construction( request_history: List[str], action_history: List[str], control_item: List[str], - prev_plan: str, + prev_plan: List[str], user_request: str, retrieved_docs: str = "", include_last_screenshot: bool = True, @@ -484,7 +484,7 @@ def user_prompt_construction( request_history: List[str], action_history: List[str], control_item: List[str], - prev_plan: str, + prev_plan: List[str], user_request: str, retrieved_docs: str = "", current_state: dict = {}, @@ -505,7 +505,7 @@ def user_prompt_construction( action_history=json.dumps(action_history), request_history=json.dumps(request_history), control_item=json.dumps(control_item), - prev_plan=prev_plan, + prev_plan=json.dumps(prev_plan), user_request=user_request, retrieved_docs=retrieved_docs, current_state=json.dumps(current_state), @@ -520,7 +520,7 @@ def user_content_construction( request_history: List[str], action_history: List[str], control_item: List[str], - prev_plan: str, + prev_plan: List[str], user_request: str, retrieved_docs: str = "", current_state: dict = {}, diff --git a/ufo/prompts/examples/lite/nonvisual/app_agent_example.yaml b/ufo/prompts/examples/lite/nonvisual/app_agent_example.yaml index 10ddc063..3d654a1e 100644 --- a/ufo/prompts/examples/lite/nonvisual/app_agent_example.yaml +++ b/ufo/prompts/examples/lite/nonvisual/app_agent_example.yaml @@ -18,11 +18,11 @@ example1: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Input the email address of the receiver. - (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. - (3) Input the content of the email. I need to input 'Dear Jack,\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\nBest regards,\nZac'. - (4) Click the Send button to send the email. + Plan: + - (1) Input the email address of the receiver. + - (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. + - (3) Input the content of the email. I need to input 'Dear Jack,\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\nBest regards,\nZac'. + - (4) Click the Send button to send the email. Comment: |- After I click the New Email button, the New Email window will be opened and available for composing the email. Tips: |- diff --git a/ufo/prompts/examples/lite/nonvisual/host_agent_example.yaml b/ufo/prompts/examples/lite/nonvisual/host_agent_example.yaml index 03947f0f..13c6dd63 100644 --- a/ufo/prompts/examples/lite/nonvisual/host_agent_example.yaml +++ b/ufo/prompts/examples/lite/nonvisual/host_agent_example.yaml @@ -15,12 +15,12 @@ example1: Mail - Outlook - Zac Status: |- CONTINUE - Plan: |- - (1) Open the windows of outlook application. - (2) Input the email address of the receiver. - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. - (4) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. - (5) Click the Send button to send the email. This action is sensitive and need to be confirmed by the user. + Plan: + - (1) Open the windows of outlook application. + - (2) Input the email address of the receiver. + - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. + - (4) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. + - (5) Click the Send button to send the email. This action is sensitive and need to be confirmed by the user. Comment: |- It is time to open the outlook application! diff --git a/ufo/prompts/examples/lite/visual/app_agent_example.yaml b/ufo/prompts/examples/lite/visual/app_agent_example.yaml index 50de1675..1924eccc 100644 --- a/ufo/prompts/examples/lite/visual/app_agent_example.yaml +++ b/ufo/prompts/examples/lite/visual/app_agent_example.yaml @@ -18,11 +18,11 @@ example1: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Input the email address of the receiver. - (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. - (3) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. - (4) Click the Send button to send the email. + Plan: + - (1) Input the email address of the receiver. + - (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. + - (3) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. + - (4) Click the Send button to send the email. Comment: |- After I click the New Email button, the New Email window will be opened and available for composing the email. Tips: |- diff --git a/ufo/prompts/examples/lite/visual/host_agent_example.yaml b/ufo/prompts/examples/lite/visual/host_agent_example.yaml index fcd53ba8..1b3ab0d5 100644 --- a/ufo/prompts/examples/lite/visual/host_agent_example.yaml +++ b/ufo/prompts/examples/lite/visual/host_agent_example.yaml @@ -14,11 +14,11 @@ example1: Mail - Outlook - Zac Status: |- CONTINUE - Plan: |- - (1) Open the windows of outlook application. - (2) Input the email address of the receiver. - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. - (4) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. - (5) Click the Send button to send the email. This action is sensitive and need to be confirmed by the user. + Plan: + - (1) Open the windows of outlook application. + - (2) Input the email address of the receiver. + - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. + - (4) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. + - (5) Click the Send button to send the email. This action is sensitive and need to be confirmed by the user. Comment: |- It is time to open the outlook application! diff --git a/ufo/prompts/examples/nonvisual/app_agent_example.yaml b/ufo/prompts/examples/nonvisual/app_agent_example.yaml index 7e8b4614..b5debe08 100644 --- a/ufo/prompts/examples/nonvisual/app_agent_example.yaml +++ b/ufo/prompts/examples/nonvisual/app_agent_example.yaml @@ -18,11 +18,11 @@ example1: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Input the email address of the receiver. - (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. - (3) Input the content of the email. I need to input 'Dear Jack,\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\nBest regards,\nZac'. - (4) Click the Send button to send the email. + Plan: + - (1) Input the email address of the receiver. + - (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. + - (3) Input the content of the email. I need to input 'Dear Jack,\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\nBest regards,\nZac'. + - (4) Click the Send button to send the email. Comment: |- After I click the New Email button, the New Email window will be opened and available for composing the email. Tips: |- @@ -48,12 +48,12 @@ example2: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Find the New Email button in the Main Page and click it to open a New Email window. If the New Email button is still not available in the control item list, I may need to look for take action on other control items to navigate to the New Email button. - (2) Input the email address of the receiver. - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. - (4) Input the content of the email. I need to input 'Dear Jack,\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\nBest regards,\nZac'. - (5) Click the Send button to send the email. + Plan: + - (1) Find the New Email button in the Main Page and click it to open a New Email window. If the New Email button is still not available in the control item list, I may need to look for take action on other control items to navigate to the New Email button. + - (2) Input the email address of the receiver. + - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. + - (4) Input the content of the email. I need to input 'Dear Jack,\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\nBest regards,\nZac'. + - (5) Click the Send button to send the email. Comment: |- I am looking for the New Email button, and will try to find it in the Main Page. Tips: |- @@ -79,8 +79,8 @@ example3: {"text": "Hello Tom. It's 3 PM. Are you available to join the meeting now?"} Status: |- CONTINUE - Plan: |- - (1) Click the Send button to send the message. This is a sensitive action that need to be confirmed by the user before the execution. + Plan: + - (1) Click the Send button to send the message. This is a sensitive action that need to be confirmed by the user before the execution. Comment: |- Inputting the message is not a sensitive action and do not need to be confirmed. Tips: |- @@ -111,8 +111,8 @@ example4: [Sender's Name] Status: |- FINISH - Plan: |- - + Plan: + - Comment: |- I revised the previous plan base on the control item list since I think the New Email window has already opened and the title of email has already been inputted. I cannot input the email address since it is not provided in the user request. Since the user did not ask me to send the email, the task is finished after I draft the content of the email. Tips: |- @@ -138,8 +138,8 @@ example5: {"button": "left", "double": false} Status: |- FINISH - Plan: |- - + Plan: + - Comment: |- I searched for the word 'UFO' in the document successfully. The user should view the search result. Tips: |- @@ -165,13 +165,13 @@ example6: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Click the 'Info' ListItem, to access the 'Info' page. - (2) Click the 'Check for Issues' MenuItem, to open a menu for checking. - (3) Click the 'Inspect Document', to open a document inspector. - (4) Click the 'Inspect' button, to inspect the ppt for notes. - (5) I should now find and click the 'Remove All Presentation Notes'. However, this button is usually at the bottom of the menu, and therefore I should apply wheel_mouse_input(wheel_dist=-20) to a ScrollBar to reach the menu bottom to make this button visible. - (6) Once 'Remove All Presentation Notes' is visible, I should click it to remove all notes in the presentation.pptx file. This action is sensitive to the system and need to be confirmed by the user. + Plan: + - (1) Click the 'Info' ListItem, to access the 'Info' page. + - (2) Click the 'Check for Issues' MenuItem, to open a menu for checking. + - (3) Click the 'Inspect Document', to open a document inspector. + - (4) Click the 'Inspect' button, to inspect the ppt for notes. + - (5) I should now find and click the 'Remove All Presentation Notes'. However, this button is usually at the bottom of the menu, and therefore I should apply wheel_mouse_input(wheel_dist=-20) to a ScrollBar to reach the menu bottom to make this button visible. + - (6) Once 'Remove All Presentation Notes' is visible, I should click it to remove all notes in the presentation.pptx file. This action is sensitive to the system and need to be confirmed by the user. Comment: |- I plan to use the 'Remove All Presentation Notes' function. This is the fastest way to remove all notes in the presentation.pptx file. Tips: |- @@ -197,10 +197,10 @@ example7: {"text": "Imdiffusion GitHub"} Status: |- CONTINUE - Plan: |- - (1) After input 'Imdiffusion GitHub', click Google Search to search for the Imdiffusion repo on github. - (2) Once the searched results are visible, click the Imdiffusion repo Hyperlink in the searched results to open the repo page. - (3) Observing and summarize the number of stars the Imdiffusion repo page, and reply to the user request. + Plan: + - (1) After input 'Imdiffusion GitHub', click Google Search to search for the Imdiffusion repo on github. + - (2) Once the searched results are visible, click the Imdiffusion repo Hyperlink in the searched results to open the repo page. + - (3) Observing and summarize the number of stars the Imdiffusion repo page, and reply to the user request. Comment: |- I plan to use Google search for the Imdiffusion repo on github and summarize the number of stars the Imdiffusion repo page visually. Tips: |- @@ -226,13 +226,13 @@ example8: {"text": "Get party dinner (5 people) preparation done before 5PM today."} Status: |- CONTINUE - Plan: |- - (1) Click the ListItem of the task that was just created to set more details. - (2) Add a step 1 for the task by inputing 'Menu Planning and Ingredient Acquisition' in the Add step edit control. - (3) Add a step 2 for the task by inputing 'Meal Preparation Schedule' in the Add step edit control following the step one. - (4) Add a step 3 for the task by inputing 'Setting Up the Dining Area' in the Add step edit control following the step two. - (5) Add a note to this task in the 'Add note' edit blocking at the bottom by inputing 'Considering preferences, dietary restrictions and food allergiesof the guests.'. - (6) Close the details view by clicking the 'Dismiss detail view' botton. + Plan: + - (1) Click the ListItem of the task that was just created to set more details. + - (2) Add a step 1 for the task by inputing 'Menu Planning and Ingredient Acquisition' in the Add step edit control. + - (3) Add a step 2 for the task by inputing 'Meal Preparation Schedule' in the Add step edit control following the step one. + - (4) Add a step 3 for the task by inputing 'Setting Up the Dining Area' in the Add step edit control following the step two. + - (5) Add a note to this task in the 'Add note' edit blocking at the bottom by inputing 'Considering preferences, dietary restrictions and food allergiesof the guests.'. + - (6) Close the details view by clicking the 'Dismiss detail view' botton. Comment: |- After creating the task, I need to set more details for the reminder, including adding steps and notes. Tips: |- @@ -259,10 +259,10 @@ example9: {"button": "left", "double": true} Status: |- CONTINUE - Plan: |- - (1) Click the 'Create presentation from file' button to open a file picker. - (2) Select and click the meeting_notes.docx file in the file picker. - (3) Click the 'Send' buttom to send the request to Copilot. This completes the task. + Plan: + - (1) Click the 'Create presentation from file' button to open a file picker. + - (2) Select and click the meeting_notes.docx file in the file picker. + - (3) Click the 'Send' buttom to send the request to Copilot. This completes the task. Comment: |- I need use the Copilot Add-in to create a slide from the meeting_notes.docx. Tips: |- @@ -288,9 +288,9 @@ example10: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Click the Comment button to open the comment text box. - (2) Input '@Zac, please revise the presentation1.pptx.' in the comment text box. + Plan: + - (1) Click the Comment button to open the comment text box. + - (2) Input '@Zac, please revise the presentation1.pptx.' in the comment text box. Comment: |- I need to leave a comment in the presentation1.pptx to remind Zac to revise the presentation1.pptx. Tips: |- diff --git a/ufo/prompts/examples/nonvisual/host_agent_example.yaml b/ufo/prompts/examples/nonvisual/host_agent_example.yaml index 93aa9dfb..eba2e5de 100644 --- a/ufo/prompts/examples/nonvisual/host_agent_example.yaml +++ b/ufo/prompts/examples/nonvisual/host_agent_example.yaml @@ -15,12 +15,12 @@ example1: Mail - Outlook - Zac Status: |- CONTINUE - Plan: |- - (1) Open the windows of outlook application. - (2) Input the email address of the receiver. - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. - (4) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. - (5) Click the Send button to send the email. This action is sensitive and need to be confirmed by the user. + Plan: + - (1) Open the windows of outlook application. + - (2) Input the email address of the receiver. + - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. + - (4) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. + - (5) Click the Send button to send the email. This action is sensitive and need to be confirmed by the user. Comment: |- It is time to open the outlook application! @@ -40,11 +40,11 @@ example2: Mike Lee | Microsoft Teams Status: |- CONTINUE - Plan: |- - (1) Open the windows of Teams application. - (2) Click the Chat button to open the chat window with Tom. - (3) Input the message. I need to input 'Hi Tom, can you join the meeting at 3pm?' - (4) Click the Send button to send the message. This action is sensitive and need to be confirmed by the user. + Plan: + - (1) Open the windows of Teams application. + - (2) Click the Chat button to open the chat window with Tom. + - (3) Input the message. I need to input 'Hi Tom, can you join the meeting at 3pm?' + - (4) Click the Send button to send the message. This action is sensitive and need to be confirmed by the user. Comment: |- It is time to open the Teams application. @@ -65,10 +65,10 @@ example3: Mail - Outlook - Zac Status: |- CONTINUE - Plan: |- - (1) Open the Mail - Outlook - Zac to complete the next task. - (2) Base on previous email draft, revise the email body to be more detail using SetText API. - (3) Click the Send button to send the email. This action is sensitive to the system and need to be confirmed by the user. + Plan: + - (1) Open the Mail - Outlook - Zac to complete the next task. + - (2) Base on previous email draft, revise the email body to be more detail using SetText API. + - (3) Click the Send button to send the email. This action is sensitive to the system and need to be confirmed by the user. Comment: |- The previous request is to draft the email and fill in the email address of the receiver, which as been completed. Now it is time to revise the email to be longer and send it. @@ -89,15 +89,15 @@ example4: presentation - PowerPoint Status: |- CONTINUE - Plan: |- - (1) Open and set focus the presentation - PowerPoint. - (2) Click the 'File' button at the upper left corner, to open a menu. - (3) Click the 'Info' ListItem, to access the 'Info' page. - (4) Click the 'Check for Issues' MenuItem, to open a menu for checking. - (5) Click the 'Inspect Document', to open a document inspector. - (6) Click the 'Inspect' button, to inspect the ppt for notes. - (7) I should now find and click the 'Remove All Presentation Notes'. However, this button is usually at the bottom of the menu and therefore I should apply wheel_mouse_input(wheel_dist=-20) to a ScrollBar to reach the menu bottom to make this button visible. - (8) Once 'Remove All Presentation Notes' is visible, I should click it to remove all notes in the presentation.pptx file. This action is sensitive to the system and need to be confirmed by the user. + Plan: + - (1) Open and set focus the presentation - PowerPoint. + - (2) Click the 'File' button at the upper left corner, to open a menu. + - (3) Click the 'Info' ListItem, to access the 'Info' page. + - (4) Click the 'Check for Issues' MenuItem, to open a menu for checking. + - (5) Click the 'Inspect Document', to open a document inspector. + - (6) Click the 'Inspect' button, to inspect the ppt for notes. + - (7) I should now find and click the 'Remove All Presentation Notes'. However, this button is usually at the bottom of the menu and therefore I should apply wheel_mouse_input(wheel_dist=-20) to a ScrollBar to reach the menu bottom to make this button visible. + - (8) Once 'Remove All Presentation Notes' is visible, I should click it to remove all notes in the presentation.pptx file. This action is sensitive to the system and need to be confirmed by the user. Comment: |- I plan to use the 'Remove All Presentation Notes' function. This is the fastest way to remove all notes in the presentation.pptx file. @@ -117,12 +117,12 @@ example5: Google - Microsoft​ Edge Status: |- CONTINUE - Plan: |- - (1) Open and set focus the Google - Microsoft​ Edge. - (2) Set text of 'Imdiffusion GitHub' in the search box of Google. The control type of the search box is usually in a type of ComboBox. - (3) Click Google Search to search for the Imdiffusion repo on github. - (4) Once the searched results are visible, click the Imdiffusion repo Hyperlink in the search results to open the repo page. - (5) Observing and summarize the number of stars the Imdiffusion repo page, and reply to the user request. + Plan: + - (1) Open and set focus the Google - Microsoft​ Edge. + - (2) Set text of 'Imdiffusion GitHub' in the search box of Google. The control type of the search box is usually in a type of ComboBox. + - (3) Click Google Search to search for the Imdiffusion repo on github. + - (4) Once the searched results are visible, click the Imdiffusion repo Hyperlink in the search results to open the repo page. + - (5) Observing and summarize the number of stars the Imdiffusion repo page, and reply to the user request. Comment: |- I plan to Google search for the Imdiffusion repo on github and summarize the number of stars the Imdiffusion repo page visually. @@ -141,15 +141,15 @@ example6: Microsoft To Do Status: |- CONTINUE - Plan: |- - (1) Open the Microsoft To Do application. - (2) Add a task of by inputing 'Get party dinner (5 people) preparation done before 5PM today.' to the edit block. - (3) Click the ListItem of the task that was just created to set more details. - (4) Add a step 1 for the task by inputing 'Menu Planning and Ingredient Acquisition' in the Add step edit control. - (5) Add a step 2 for the task by inputing 'Meal Preparation Schedule' in the Add step edit control following the step one. - (6) Add a step 3 for the task by inputing 'Setting Up the Dining Area' in the Add step edit control following the step two. - (7) Add a note to this task in the 'Add note' edit blocking at the bottom by inputing 'Considering preferences, dietary restrictions and food allergiesof the guests.'. - (8) Close the details view by clicking the 'Dismiss detail view' botton. + Plan: + - (1) Open the Microsoft To Do application. + - (2) Add a task of by inputing 'Get party dinner (5 people) preparation done before 5PM today.' to the edit block. + - (3) Click the ListItem of the task that was just created to set more details. + - (4) Add a step 1 for the task by inputing 'Menu Planning and Ingredient Acquisition' in the Add step edit control. + - (5) Add a step 2 for the task by inputing 'Meal Preparation Schedule' in the Add step edit control following the step one. + - (6) Add a step 3 for the task by inputing 'Setting Up the Dining Area' in the Add step edit control following the step two. + - (7) Add a note to this task in the 'Add note' edit blocking at the bottom by inputing 'Considering preferences, dietary restrictions and food allergiesof the guests.'. + - (8) Close the details view by clicking the 'Dismiss detail view' botton. Comment: |- I plan to use the Microsoft To Do application to set a reminder for the user, and add details and notes to the reminder. @@ -169,12 +169,12 @@ example7: presentation1 - PowerPoint Status: |- CONTINUE - Plan: |- - (1) Open the presentation1 - PowerPoint. - (2) Double click the Copilot Add-in to open the Copilot that can help to create a slide from the meeting_notes.docx. The Copilot Add-in can directly create a slide from the meeting_notes.docx. - (3) Click the 'Create presentation from file' button to open a file picker. - (4) Select and click the meeting_notes.docx file in the file picker. - (5) Click the 'Send' buttom to send the request to Copilot. This completes the task. + Plan: + - (1) Open the presentation1 - PowerPoint. + - (2) Double click the Copilot Add-in to open the Copilot that can help to create a slide from the meeting_notes.docx. The Copilot Add-in can directly create a slide from the meeting_notes.docx. + - (3) Click the 'Create presentation from file' button to open a file picker. + - (4) Select and click the meeting_notes.docx file in the file picker. + - (5) Click the 'Send' buttom to send the request to Copilot. This completes the task. Comment: |- I plan to open the powerpoint application and use the Copilot Add-in to create a slide from the meeting_notes.docx. @@ -194,12 +194,58 @@ example8: presentation1 - PowerPoint Status: |- CONTINUE - Plan: |- - (1) Open the presentation1 - PowerPoint. - (2) Click the Insert Button to open the Insert menu. - (3) Click the Comment button to open the comment text box. - (4) Input '@Zac, please revise the presentation1.pptx.' in the comment text box. + Plan: + - (1) Open the presentation1 - PowerPoint. + - (2) Click the Insert Button to open the Insert menu. + - (3) Click the Comment button to open the comment text box. + - (4) Input '@Zac, please revise the presentation1.pptx.' in the comment text box. Comment: |- I plan to leave a comment in the presentation1.pptx to remind Zac to revise the presentation1.pptx. + +example_openapp1: + Request: + open a ppt file on my desktop named test.pptx and modify the title to Apple is the best tech Company + Response: + Observation: |- + I observe that the PowerPoint application is not available in the control item list, so I need to open the PowerPoint application and the test.pptx file. + Thought: |- + I need to open test.pptx through function OpenAPP directly, as I can not observe it in the screenshot. After that, I will proceed to modify the title of the ppt file., + ControlLabel: |- + 2 + ControlText: |- + PowerPoint + Status: |- + CONTINUE + Plan: + - "(1) Call the function OpenAPP with the arguments {'APP': 'powerpnt', 'file_path': 'Desktop\test.pptx'} to open the test.pptx file." + - (2) When the file is open, delete text in the title and input 'Apple is the best tech Company' in the title. + Comment: |- + Since the PowerPoint application is not visible in the screenshot, I will use the function OpenAPP to open the test.pptx file directly from the desktop. Then modify the title of the ppt file after its opened. + AppsToOpen: {'APP': 'powerpnt', 'file_path': 'Desktop\test.pptx'} + + +example_openapp2: + Request: |- + open file explorer APP for me and find label.txt file. + Response: + Observation: |- + I observe that the file explorer is not available in the control item list. So I need to open the file explorer via python code, and then use search box to find the label.txt. + Thought: |- + I need to open file explorer through function OpenAPP directly, as I can not observe it in the screenshot. + ControlLabel: |- + 3 + ControlText: + Explorer + Status: |- + CONTINUE + Plan: + - "(1) Call the function OpenAPP with the arguments {'APP': 'powerpnt', 'file_path': ''} to open the file explorer APP." + - (2) Once the file explorer is open, I need to use the search box to find the label.txt file., + Comment: |- + Since the file explorer application is not visible in the screenshot, I will use the function OpenAPP to open the file explorer file directly. + AppsToOpen: |- + {'APP': 'powerpnt', 'file_path': ''} + + diff --git a/ufo/prompts/examples/visual/app_agent_example.yaml b/ufo/prompts/examples/visual/app_agent_example.yaml index 291222d4..4e5944f2 100644 --- a/ufo/prompts/examples/visual/app_agent_example.yaml +++ b/ufo/prompts/examples/visual/app_agent_example.yaml @@ -18,11 +18,11 @@ example1: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Input the email address of the receiver. - (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. - (3) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. - (4) Click the Send button to send the email. + Plan: + - (1) Input the email address of the receiver. + - (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. + - (3) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. + - (4) Click the Send button to send the email. Comment: |- After I click the New Email button, the New Email window will be opened and available for composing the email. Tips: |- @@ -48,23 +48,22 @@ example2: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Find the New Email button in the Main Page and click it to open a New Email window. If the New Email button is still not visible in the screenshot, I may need to look for take action on other control items to navigate to the New Email button. - (2) Input the email address of the receiver. - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. - (4) Input the content of the email. I need to input + Plan: + - (1) Find the New Email button in the Main Page and click it to open a New Email window. If the New Email button is still not visible in the screenshot, I may need to look for take action on other control items to navigate to the New Email button. + - (2) Input the email address of the receiver. + - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. + - (4) Input the content of the email. I need to input - 'Dear Jack, - I hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board. - Your commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole. - Your code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success. - - Please know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out. - Once again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community. - Best regards, - Zac'. - - (5) Click the Send button to send the email. + 'Dear Jack, + I hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board. + Your commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole. + Your code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success. + + Please know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out. + Once again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community. + Best regards, + Zac'. + - (5) Click the Send button to send the email. Comment: |- I am looking for the New Email button, and will try to find it in the Main Page. Tips: |- @@ -91,8 +90,8 @@ example3: {"text": "Hello Tom. It's 3 PM. Are you available to join the meeting now?"} Status: |- CONTINUE - Plan: |- - (1) Click the Send button to send the message. This is a sensitive action that need to be confirmed by the user before the execution. + Plan: + - (1) Click the Send button to send the message. This is a sensitive action that need to be confirmed by the user before the execution. Comment: |- Inputting the message is not a sensitive action and do not need to be confirmed. Tips: |- @@ -124,8 +123,8 @@ example4: [Sender's Name] Status: |- FINISH - Plan: |- - + Plan: + - Comment: |- I revised the previous plan base on the screenshot since I observe that New Email window has already opened and the title of email has already been inputted. I cannot input the email address since it is not provided in the user request. Since the user did not ask me to send the email, the task is finished after I draft the content of the email. Tips: |- @@ -151,8 +150,8 @@ example5: {"button": "left", "double": false} Status: |- FINISH - Plan: |- - + Plan: + - Comment: |- I searched for the word 'UFO' in the document successfully. The user should view the search result. Tips: |- @@ -176,13 +175,13 @@ example6: Args: {} Status: |- APP_SELECTION - Plan: |- - (1) Switch to the image of framework.png to complete the next task, the current status need to set to 'APP_SELECTION'. - (2) Describe in detail the workflow of the framework in the image of framework.png base on the screenshot of the image. - (3) Switch to the Teams application. - (4) Open the chat window with Tom. - (5) Input the text of the Document control named 'framework.docx' and the description of the workflow of the framework in the image of framework.png in the Edit control named 'Type a new message'. These information can be extracted from the results from my history steps and I will use them to compose the message. - (6) Click the Send button to send the message. This action is sensitive to the system and need to be confirmed by the user. I need to confirm the action before clicking the Send button. If the user confirms the action, the task is finished. + Plan: + - (1) Switch to the image of framework.png to complete the next task, the current status need to set to 'APP_SELECTION'. + - (2) Describe in detail the workflow of the framework in the image of framework.png base on the screenshot of the image. + - (3) Switch to the Teams application. + - (4) Open the chat window with Tom. + - (5) Input the text of the Document control named 'framework.docx' and the description of the workflow of the framework in the image of framework.png in the Edit control named 'Type a new message'. These information can be extracted from the results from my history steps and I will use them to compose the message. + - (6) Click the Send button to send the message. This action is sensitive to the system and need to be confirmed by the user. I need to confirm the action before clicking the Send button. If the user confirms the action, the task is finished. Comment: |- The partial tasks on 'framework.docx' is completed once I take the current action. I need to set Status immediately to 'APP_SELECTION' to switch to the image of framework.png to complete the next task. Tips: |- @@ -208,13 +207,13 @@ example7: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Click the 'Info' ListItem, to access the 'Info' page. - (2) Click the 'Check for Issues' MenuItem, to open a menu for checking. - (3) Click the 'Inspect Document', to open a document inspector. - (4) Click the 'Inspect' button, to inspect the ppt for notes. - (5) I should now find and click the 'Remove All Presentation Notes'. However, this button is usually at the bottom of the menu, and therefore I should apply wheel_mouse_input(wheel_dist=-20) to a ScrollBar to reach the menu bottom to make this button visible. - (6) Once 'Remove All Presentation Notes' is visible, I should click it to remove all notes in the presentation.pptx file. This action is sensitive to the system and need to be confirmed by the user. + Plan: + - (1) Click the 'Info' ListItem, to access the 'Info' page. + - (2) Click the 'Check for Issues' MenuItem, to open a menu for checking. + - (3) Click the 'Inspect Document', to open a document inspector. + - (4) Click the 'Inspect' button, to inspect the ppt for notes. + - (5) I should now find and click the 'Remove All Presentation Notes'. However, this button is usually at the bottom of the menu, and therefore I should apply wheel_mouse_input(wheel_dist=-20) to a ScrollBar to reach the menu bottom to make this button visible. + - (6) Once 'Remove All Presentation Notes' is visible, I should click it to remove all notes in the presentation.pptx file. This action is sensitive to the system and need to be confirmed by the user. Comment: |- I plan to use the 'Remove All Presentation Notes' function. This is the fastest way to remove all notes in the presentation.pptx file. Tips: |- @@ -240,10 +239,10 @@ example8: {"text": "Imdiffusion GitHub"} Status: |- CONTINUE - Plan: |- - (1) After input 'Imdiffusion GitHub', click Google Search to search for the Imdiffusion repo on github. - (2) Once the searched results are visible, click the Imdiffusion repo Hyperlink in the searched results to open the repo page. - (3) Observing and summarize the number of stars the Imdiffusion repo page, and reply to the user request. + Plan: + - (1) After input 'Imdiffusion GitHub', click Google Search to search for the Imdiffusion repo on github. + - (2) Once the searched results are visible, click the Imdiffusion repo Hyperlink in the searched results to open the repo page. + - (3) Observing and summarize the number of stars the Imdiffusion repo page, and reply to the user request. Comment: |- I plan to use Google search for the Imdiffusion repo on github and summarize the number of stars the Imdiffusion repo page visually. Tips: |- @@ -270,13 +269,13 @@ example9: {"text": "Get party dinner (5 people) preparation done before 5PM today."} Status: |- CONTINUE - Plan: |- - (1) Click the ListItem of the task that was just created to set more details. - (2) Add a step 1 for the task by inputing 'Menu Planning and Ingredient Acquisition' in the Add step edit control. - (3) Add a step 2 for the task by inputing 'Meal Preparation Schedule' in the Add step edit control following the step one. - (4) Add a step 3 for the task by inputing 'Setting Up the Dining Area' in the Add step edit control following the step two. - (5) Add a note to this task in the 'Add note' edit blocking at the bottom by inputing 'Considering preferences, dietary restrictions and food allergiesof the guests.'. - (6) Close the details view by clicking the 'Dismiss detail view' botton. + Plan: + - (1) Click the ListItem of the task that was just created to set more details. + - (2) Add a step 1 for the task by inputing 'Menu Planning and Ingredient Acquisition' in the Add step edit control. + - (3) Add a step 2 for the task by inputing 'Meal Preparation Schedule' in the Add step edit control following the step one. + - (4) Add a step 3 for the task by inputing 'Setting Up the Dining Area' in the Add step edit control following the step two. + - (5) Add a note to this task in the 'Add note' edit blocking at the bottom by inputing 'Considering preferences, dietary restrictions and food allergiesof the guests.'. + - (6) Close the details view by clicking the 'Dismiss detail view' botton. Comment: |- After creating the task, I need to set more details for the reminder, including adding steps and notes. Tips: |- @@ -303,10 +302,10 @@ example10: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Click the 'Create presentation from file' button to open a file picker. - (2) When the 'meeting_notes' is visible in the file picker, select and click the "meeting_notes" ListItem to create a slide. - (3) Click the 'Send' buttom to send the request to Copilot. This completes the task. + Plan: + - (1) Click the 'Create presentation from file' button to open a file picker. + - (2) When the 'meeting_notes' is visible in the file picker, select and click the "meeting_notes" ListItem to create a slide. + - (3) Click the 'Send' buttom to send the request to Copilot. This completes the task. Comment: |- I need to use the Copilot Add-in to create a slide from the meeting_notes.docx. Tips: |- @@ -331,9 +330,9 @@ example11: {"button": "left", "double": false} Status: |- CONTINUE - Plan: |- - (1) Click the "Save As Adobe PDF" button to open the save as dialog. - (2) Click the "Save" button to save the presentation1.pptx into pdf format. This completes the task. + Plan: + - (1) Click the "Save As Adobe PDF" button to open the save as dialog. + - (2) Click the "Save" button to save the presentation1.pptx into pdf format. This completes the task. Comment: |- I need to use the "Save As Adobe PDF" button to save the presentation1.pptx into pdf format. Tips: |- diff --git a/ufo/prompts/examples/visual/host_agent_example.yaml b/ufo/prompts/examples/visual/host_agent_example.yaml index bdb0ad71..1281ca4b 100644 --- a/ufo/prompts/examples/visual/host_agent_example.yaml +++ b/ufo/prompts/examples/visual/host_agent_example.yaml @@ -14,12 +14,12 @@ example1: Mail - Outlook - Zac Status: |- CONTINUE - Plan: |- - (1) Open the windows of outlook application. - (2) Input the email address of the receiver. - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. - (4) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. - (5) Click the Send button to send the email. This action is sensitive and need to be confirmed by the user. + Plan: + - (1) Open the windows of outlook application. + - (2) Input the email address of the receiver. + - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'. + - (4) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'. + - (5) Click the Send button to send the email. This action is sensitive and need to be confirmed by the user. Comment: |- It is time to open the outlook application! AppsToOpen: |- @@ -40,11 +40,11 @@ example2: Mike Lee | Microsoft Teams Status: |- CONTINUE - Plan: |- - (1) Open the windows of Teams application. - (2) Click the Chat button to open the chat window with Tom. - (3) Input the message. I need to input 'Hi Tom, can you join the meeting at 3pm?' - (4) Click the Send button to send the message. This action is sensitive and need to be confirmed by the user. + Plan: + - (1) Open the windows of Teams application. + - (2) Click the Chat button to open the chat window with Tom. + - (3) Input the message. I need to input 'Hi Tom, can you join the meeting at 3pm?' + - (4) Click the Send button to send the message. This action is sensitive and need to be confirmed by the user. Comment: |- It is time to open the Teams application. AppsToOpen: |- @@ -66,12 +66,12 @@ example3: framework.png | image Status: |- CONTINUE - Plan: |- - (1) Open the image of framework.png to complete the next task. - (2) Describe in detail the workflow of the framework in the image of framework.png base on the screenshot of the image. - (3) Switch to the Teams application.\ - (4) Open the chat window with Tom. - (5) Input the text of the Document control named 'framework.docx' and the description of the workflow of the framework in the image of framework.png in the Edit control named 'Type a new message'. These information can be extracted from my history actions.\\n (6) Click the Send button to send the message. This action is sensitive to the system and need to be confirmed by the user. I need to confirm the action before clicking the Send button. If the user confirms the action, the task is finished. + Plan: + - (1) Open the image of framework.png to complete the next task. + - (2) Describe in detail the workflow of the framework in the image of framework.png base on the screenshot of the image. + - (3) Switch to the Teams application.\ + - (4) Open the chat window with Tom. + - (5) Input the text of the Document control named 'framework.docx' and the description of the workflow of the framework in the image of framework.png in the Edit control named 'Type a new message'. These information can be extracted from my history actions.\\n (6) Click the Send button to send the message. This action is sensitive to the system and need to be confirmed by the user. I need to confirm the action before clicking the Send button. If the user confirms the action, the task is finished. Comment: |- After I get the text of the Document control named 'framework.docx', I need to switch to the image of framework.png to complete the next task, and summarize them to sent the message to Tom on Teams. AppsToOpen: |- @@ -92,10 +92,10 @@ example4: Mail - Outlook - Zac Status: |- CONTINUE - Plan: |- - (1) Open the Mail - Outlook - Zac to complete the next task. - (2) Base on previous email draft, revise the email body to be more detail using SetText API. - (3) Click the Send button to send the email. This action is sensitive to the system and need to be confirmed by the user. + Plan: + - (1) Open the Mail - Outlook - Zac to complete the next task. + - (2) Base on previous email draft, revise the email body to be more detail using SetText API. + - (3) Click the Send button to send the email. This action is sensitive to the system and need to be confirmed by the user. Comment: |- The previous request is to draft the email and fill in the email address of the receiver, which as been completed. Now it is time to revise the email to be longer and send it. AppsToOpen: |- @@ -115,15 +115,15 @@ example5: presentation - PowerPoint Status: |- CONTINUE - Plan: |- - (1) Open and set focus the presentation - PowerPoint. - (2) Click the 'File' button at the upper left corner, to open a menu. - (3) Click the 'Info' ListItem, to access the 'Info' page. - (4) Click the 'Check for Issues' MenuItem, to open a menu for checking. - (5) Click the 'Inspect Document', to open a document inspector. - (6) Click the 'Inspect' button, to inspect the ppt for notes. - (7) I should now find and click the 'Remove All Presentation Notes'. However, this button is usually at the bottom of the menu and therefore I should apply wheel_mouse_input(wheel_dist=-20) to a ScrollBar to reach the menu bottom to make this button visible. - (8) Once 'Remove All Presentation Notes' is visible, I should click it to remove all notes in the presentation.pptx file. This action is sensitive to the system and need to be confirmed by the user. + Plan: + - (1) Open and set focus the presentation - PowerPoint. + - (2) Click the 'File' button at the upper left corner, to open a menu. + - (3) Click the 'Info' ListItem, to access the 'Info' page. + - (4) Click the 'Check for Issues' MenuItem, to open a menu for checking. + - (5) Click the 'Inspect Document', to open a document inspector. + - (6) Click the 'Inspect' button, to inspect the ppt for notes. + - (7) I should now find and click the 'Remove All Presentation Notes'. However, this button is usually at the bottom of the menu and therefore I should apply wheel_mouse_input(wheel_dist=-20) to a ScrollBar to reach the menu bottom to make this button visible. + - (8) Once 'Remove All Presentation Notes' is visible, I should click it to remove all notes in the presentation.pptx file. This action is sensitive to the system and need to be confirmed by the user. Comment: |- I plan to use the 'Remove All Presentation Notes' function. This is the fastest way to remove all notes in the presentation.pptx file. AppsToOpen: |- @@ -145,12 +145,12 @@ example6: Google - Microsoft​ Edge Status: |- CONTINUE - Plan: |- - (1) Open and set focus the Google - Microsoft​ Edge. - (2) Set text of 'Imdiffusion GitHub' in the search box of Google. The control type of the search box is usually in a type of ComboBox. - (3) Click Google Search to search for the Imdiffusion repo on github. - (4) Once the searched results are visible, click the Imdiffusion repo Hyperlink in the search results to open the repo page. - (5) Observing and summarize the number of stars the Imdiffusion repo page, and reply to the user request. + Plan: + - (1) Open and set focus the Google - Microsoft​ Edge. + - (2) Set text of 'Imdiffusion GitHub' in the search box of Google. The control type of the search box is usually in a type of ComboBox. + - (3) Click Google Search to search for the Imdiffusion repo on github. + - (4) Once the searched results are visible, click the Imdiffusion repo Hyperlink in the search results to open the repo page. + - (5) Observing and summarize the number of stars the Imdiffusion repo page, and reply to the user request. Comment: |- I plan to Google search for the Imdiffusion repo on github and summarize the number of stars the Imdiffusion repo page visually. AppsToOpen: |- @@ -171,15 +171,15 @@ example7: Microsoft To Do Status: |- CONTINUE - Plan: |- - (1) Open the Microsoft To Do application. - (2) Add a task of by inputing 'Get party dinner (5 people) preparation done before 5PM today.' to the edit block. - (3) Click the ListItem of the task that was just created to set more details. - (4) Add a step 1 for the task by inputing 'Menu Planning and Ingredient Acquisition' in the Add step edit control. - (5) Add a step 2 for the task by inputing 'Meal Preparation Schedule' in the Add step edit control following the step one. - (6) Add a step 3 for the task by inputing 'Setting Up the Dining Area' in the Add step edit control following the step two. - (7) Add a note to this task in the 'Add note' edit blocking at the bottom by inputing 'Considering preferences, dietary restrictions and food allergiesof the guests.'. - (8) Close the details view by clicking the 'Dismiss detail view' botton. + Plan: + - (1) Open the Microsoft To Do application. + - (2) Add a task of by inputing 'Get party dinner (5 people) preparation done before 5PM today.' to the edit block. + - (3) Click the ListItem of the task that was just created to set more details. + - (4) Add a step 1 for the task by inputing 'Menu Planning and Ingredient Acquisition' in the Add step edit control. + - (5) Add a step 2 for the task by inputing 'Meal Preparation Schedule' in the Add step edit control following the step one. + - (6) Add a step 3 for the task by inputing 'Setting Up the Dining Area' in the Add step edit control following the step two. + - (7) Add a note to this task in the 'Add note' edit blocking at the bottom by inputing 'Considering preferences, dietary restrictions and food allergiesof the guests.'. + - (8) Close the details view by clicking the 'Dismiss detail view' botton. Comment: |- I plan to use the Microsoft To Do application to set a reminder for the user, and add details and notes to the reminder. AppsToOpen: |- @@ -199,12 +199,12 @@ example8: presentation1 - PowerPoint Status: |- CONTINUE - Plan: |- - (1) Open the presentation1 - PowerPoint. - (2) Double click the Copilot Add-in to open the Copilot that can help to create a slide from the meeting_notes.docx. The Copilot Add-in can directly create a slide from the meeting_notes.docx. - (3) Click the 'Create presentation from file' button to open a file picker. - (4) Select and click the meeting_notes.docx file in the file picker. - (5) Click the 'Send' buttom to send the request to Copilot. This completes the task. + Plan: + - (1) Open the presentation1 - PowerPoint. + - (2) Double click the Copilot Add-in to open the Copilot that can help to create a slide from the meeting_notes.docx. The Copilot Add-in can directly create a slide from the meeting_notes.docx. + - (3) Click the 'Create presentation from file' button to open a file picker. + - (4) Select and click the meeting_notes.docx file in the file picker. + - (5) Click the 'Send' buttom to send the request to Copilot. This completes the task. Comment: |- I plan to open the powerpoint application and use the Copilot Add-in to create a slide from the meeting_notes.docx. @@ -232,11 +232,11 @@ example9: presentation1 - PowerPoint Status: |- CONTINUE - Plan: |- - (1) Open the presentation1 - PowerPoint. - (2) Click the Insert Button to open the Insert menu. - (3) Click the Comment button to open the comment text box. - (4) Input '@Zac, please revise the presentation1.pptx.' in the comment text box. + Plan: + - (1) Open the presentation1 - PowerPoint. + - (2) Click the Insert Button to open the Insert menu. + - (3) Click the Comment button to open the comment text box. + - (4) Input '@Zac, please revise the presentation1.pptx.' in the comment text box. Comment: |- I plan to leave a comment in the presentation1.pptx to remind Zac to revise the presentation1.pptx. AppsToOpen: |- @@ -248,7 +248,7 @@ example_openapp1: open a ppt file on my desktop named test.pptx and modify the title to Apple is the best tech Company Response: Observation: |- - I observe that the PowerPoint application is not visible in the screenshot, so I need to open the PowerPoint application and the test.pptx file., + I observe that the PowerPoint application is not visible in the screenshot, nor available in the list of applications. So I need to open the PowerPoint application and the test.pptx file., Thought: |- I need to open test.pptx through function OpenAPP directly, as I can not observe it in the screenshot. After that, I will proceed to modify the title of the ppt file., ControlLabel: |- @@ -257,9 +257,9 @@ example_openapp1: PowerPoint Status: |- CONTINUE - Plan: |- - (1) Call the function OpenAPP with the arguments {'APP': 'powerpnt', 'file_path': 'Desktop\test.pptx'} to open the test.pptx file. - (2) When the file is open, delete text in the title and input 'Apple is the best tech Company' in the title. + Plan: + - "Call the function OpenAPP with the arguments {'APP': 'powerpnt', 'file_path': 'Desktop\test.pptx'} to open the test.pptx file." + - "When the file is open, delete text in the title and input 'Apple is the best tech Company' in the title." Comment: |- Since the PowerPoint application is not visible in the screenshot, I will use the function OpenAPP to open the test.pptx file directly from the desktop. Then modify the title of the ppt file after its opened. AppsToOpen: {'APP': 'powerpnt', 'file_path': 'Desktop\test.pptx'} @@ -270,7 +270,7 @@ example_openapp2: open file explorer APP for me and find label.txt file. Response: Observation: |- - I observe that the file explorer is not visible in the screenshot, so I need to open the file explorer via python code, and then use search box to find the label.txt. + I observe that the file explorer is not visible in the screenshot, nor available in the list of applications. So I need to open the file explorer application and find the label.txt file. Thought: |- I need to open file explorer through function OpenAPP directly, as I can not observe it in the screenshot. ControlLabel: |- @@ -279,9 +279,9 @@ example_openapp2: Explorer Status: |- CONTINUE - Plan: |- - (1) Call the function OpenAPP with the arguments {'APP': 'powerpnt', 'file_path': ''} to open the file explorer APP. - (2) Once the file explorer is open, I need to use the search box to find the label.txt file., + Plan: + - "(1) Call the function OpenAPP with the arguments {'APP': 'powerpnt', 'file_path': ''} to open the file explorer APP." + - (2) Once the file explorer is open, I need to use the search box to find the label.txt file., Comment: |- Since the file explorer application is not visible in the screenshot, I will use the function OpenAPP to open the file explorer file directly. AppsToOpen: |- diff --git a/ufo/prompts/share/base/app_agent.yaml b/ufo/prompts/share/base/app_agent.yaml index 0da8dc5d..e340eda0 100644 --- a/ufo/prompts/share/base/app_agent.yaml +++ b/ufo/prompts/share/base/app_agent.yaml @@ -10,7 +10,7 @@ system: |- - You are provided the function return from your previous action for reference to decide the next step. You may use the return of your previous action to complete the user request. - You are provided the steps history, including historical actions, thoughts, and results of your previous steps for reference to decide the next step. Use them to help you think about the next step. - You are required to select the control item and take one-step action on it to complete the user request for one step. The one-step action means calling a function with arguments for only once. - - You are required to decide whether the task status, and detail a plan of following actions to accomplish the current user request. Do not include any additional actions beyond the completion of the current user request. + - You are required to decide whether the task status, and detail a list of plan of following actions to accomplish the current user request. Do not include any additional actions beyond the completion of the current user request. ## On screenshots - You are provided two versions of screenshots of the current application in a single image, one with annotation (right) and one without annotation (left). @@ -85,7 +85,7 @@ system: |- "Function": "Args": "Status": - "Plan": for reference, and you can reflect on it and revise if necessary. If you believe the task is finished and no further actions are required after the current action, output "".> + "Plan": for reference, and you can reflect on it and revise if necessary. If you believe the task is finished and no further actions are required after the current action, output "".> "Comment": }} - If the user request includes asking questions, and you can answer the question without taking calling API on the application at current step, you should answer the question in the "Comment" field in the response, and set the "Status" as "FINISH". @@ -129,7 +129,7 @@ system_nonvisual: |- - You are provided the function return from your previous action for reference to decide the next step. You may use the return of your previous action to complete the user request. - You are provided the steps history, including historical actions, thoughts, and results of your previous steps for reference to decide the next step. Use them to help you think about the next step. - You are required to select the control item and take one-step action on it to complete the user request for one step. The one-step action means calling a function with arguments for only once. - - You are required to decide whether the task status, and detail a plan of following actions to accomplish the current user request. Do not include any additional actions beyond the completion of the current user request. + - You are required to decide whether the task status, and detail a list of plan of following actions to accomplish the current user request. Do not include any additional actions beyond the completion of the current user request. ## Control item - The control item is the element on the page that you can interact with, we limit the actionable control item to the following: @@ -188,7 +188,7 @@ system_nonvisual: |- "Function": "Args": "Status": - "Plan": for reference, and you can reflect on it and revise if necessary. If you believe the task is finished and no further actions are required after the current action, output "".> + "Plan": for reference, and you can reflect on it and revise if necessary. If you believe the task is finished and no further actions are required after the current action, output "".> "Comment": }} - If the user request includes asking questions, and you can answer the question without taking calling API on the application at current step, you should answer the question in the "Comment" field in the response, and set the "Status" as "FINISH". diff --git a/ufo/prompts/share/base/host_agent.yaml b/ufo/prompts/share/base/host_agent.yaml index 06629566..2e665e64 100644 --- a/ufo/prompts/share/base/host_agent.yaml +++ b/ufo/prompts/share/base/host_agent.yaml @@ -25,7 +25,7 @@ system: |- - Some of the applications may not visible in the screenshot, but they are available in the list of . You can try to select these applications if required. - When a user presents a request, your task is to [1] Identify and select the appropriate application or control item. - [2] Detail a plan of following actions to accomplish the given task. + [2] Detail a list of plan of following actions to accomplish the given task. [3] If the target application is visible, decide the current status of the task base on the screenshot. Draft your plan based on the current status of the task, and do not include the steps that have been completed on the application, or beyond the user request. [4] Determine whether the status of the task is finished or not. - When making you plan, please refer to the history of actions, thoughts, and results of your previous steps, and previous user requests. Make sure your plan is complete ONLY for the current user request. You must not include redundant steps beyond the completion of the current user request. @@ -58,7 +58,7 @@ system: |- "ControlLabel": "ControlText": "Status": - "Plan": .> + "Plan": .> "Comment": {open_app_comment} }} @@ -93,7 +93,7 @@ system_nonvisual: |- - Some of the applications may not available from the control item list, but they are available in the list of . You can try to select these applications if required. - When a user presents a request, your task is to [1] Identify and select the appropriate application or control item. - [2] Detail a plan of following actions to accomplish the given task. + [2] Detail a list of plan of following actions to accomplish the given task. [3] If the target application is available, decide the current status of the task base on the control item list. Draft your plan based on the current status of the task, and do not include the steps that have been completed on the application, or beyond the user request. [4] Determine whether the status of the task is finished or not. - When making you plan, please refer to the history of actions, thoughts, and results of your previous steps, and previous user requests. Make sure your plan is complete ONLY for the current user request. You must not include redundant steps beyond the completion of the current user request. @@ -126,7 +126,7 @@ system_nonvisual: |- "ControlLabel": "ControlText": "Status": - "Plan": .> + "Plan": .> "Comment": {open_app_comment} }} diff --git a/ufo/prompts/share/lite/app_agent.yaml b/ufo/prompts/share/lite/app_agent.yaml index 5c8488f8..c4e1f1cc 100644 --- a/ufo/prompts/share/lite/app_agent.yaml +++ b/ufo/prompts/share/lite/app_agent.yaml @@ -52,7 +52,7 @@ system: |- "Function": "Args": "Status": - "Plan": for reference, and you can reflect on it and revise if necessary. If you believe the task is finished and no further actions are required after the current action, output "".> + "Plan": for reference, and you can reflect on it and revise if necessary. If you believe the task is finished and no further actions are required after the current action, output "".> "Comment": }} - If the user request includes asking questions, and you can answer the question without taking calling API on the application at current step, you should answer the question in the "Comment" field in the response, and set the "Status" as "FINISH". @@ -129,7 +129,7 @@ system_nonvisual: |- "Function": "Args": "Status": - "Plan": for reference, and you can reflect on it and revise if necessary. If you believe the task is finished and no further actions are required after the current action, output "".> + "Plan": for reference, and you can reflect on it and revise if necessary. If you believe the task is finished and no further actions are required after the current action, output "".> "Comment": }} - If the user request includes asking questions, and you can answer the question without taking calling API on the application at current step, you should answer the question in the "Comment" field in the response, and set the "Status" as "FINISH". - If the required control item is not avaialble from the control item list, and not available in the control item list, you may need to take action on other control items to navigate to the required control item. diff --git a/ufo/prompts/share/lite/host_agent.yaml b/ufo/prompts/share/lite/host_agent.yaml index f5b8af7b..f3cca426 100644 --- a/ufo/prompts/share/lite/host_agent.yaml +++ b/ufo/prompts/share/lite/host_agent.yaml @@ -11,7 +11,7 @@ system: |- - Some of the applications may not visible in the screenshot, you can try if required. - When a user presents a request, your task is to [1] Identify and select the appropriate application or control item. - [2] Detail a plan of following actions to accomplish the given task. + [2] Detail a list of plan of following actions to accomplish the given task. [3] If the target application is visible, decide the current status of the task base on the screenshot. [4] Determine whether the status of the task is finished or not. - When making you plan, please refer to the history of actions, thoughts, and results of your previous steps, and previous user requests. Make sure your plan is complete ONLY for the current user request. You must not include redundant steps beyond the completion of the current user request. @@ -29,7 +29,7 @@ system: |- "ControlLabel": "ControlText": "Status": - "Plan": .> + "Plan": .> "Comment": }} - If the user request is just asking question and do not need actions, you should answer the user request on the "Comment" field, and set the "Status" as "FINISH". @@ -58,7 +58,7 @@ system_nonvisual: |- - Some of the applications may not available from the control item list, but they are available in the list of . You can try to select these applications if required. - When a user presents a request, your task is to [1] Identify and select the appropriate application or control item. - [2] Detail a plan of following actions to accomplish the given task. + [2] Detail a list of plan of following actions to accomplish the given task. [3] If the target application is available, decide the current status of the task base on the control item list. Draft your plan based on the current status of the task, and do not include the steps that have been completed on the application, or beyond the user request. [4] Determine whether the status of the task is finished or not. - When making you plan, please refer to the history of actions, thoughts, and results of your previous steps, and previous user requests. Make sure your plan is complete ONLY for the current user request. You must not include redundant steps beyond the completion of the current user request. @@ -81,7 +81,7 @@ system_nonvisual: |- "ControlLabel": "ControlText": "Status": - "Plan": .> + "Plan": .> "Comment": }} - If the user request is just asking question and do not need to take action on the application, you should answer the user request on the "Comment" field, and set the "Status" as "FINISH".