fix qwen bug

README.md

@@ -89,7 +89,8 @@ pip install -r requirements.txt
 ```
 
 ### ⚙️ Step 2: Configure the LLMs
-Before running UFO, you need to provide your LLM configurations **individully for HostAgent and AppAgent**. You can create your own config file `ufo/config/config.yaml`, by copying the `ufo/config/config.yaml.template` and editing config for **APP_AGENT** and **ACTION_AGENT** as follows: 
+Before running UFO, you need to provide your LLM configurations **individually for HostAgent and AppAgent**. You can create your own config file `ufo/config/config.yaml`, by copying the `ufo/config/config.yaml.template` and editing config for **APP_AGENT** and **ACTION_AGENT** as follows: 
+
 
 #### OpenAI
 ```bash

ufo/agent/agent.py

@@ -522,7 +522,7 @@ def print_response(self, response_dict: Dict) -> None:
         )
         utils.print_with_color("Status📊: {status}".format(status=status), "blue")
         utils.print_with_color(
-            "Next Plan📚: {plan}".format(plan="\n".join(plan)), "cyan"
+            "Next Plan📚: {plan}".format(plan=str(plan) if isinstance(plan[0], dict) else "\n".join(plan)), "cyan"
         )
         utils.print_with_color("Comment💬: {comment}".format(comment=comment), "green")
 

ufo/config/config_prices.yaml

@@ -1,6 +1,6 @@
 # Source: https://openai.com/pricing
 # Prices in $ per 1000 tokens
-# Last updated: 2024-03-27
+# Last updated: 2024-05-13
 PRICES: { 
     "openai/gpt-4-0613": {"input": 0.03, "output": 0.06},
     "openai/gpt-3.5-turbo-0613": {"input": 0.0015, "output": 0.002},
@@ -9,6 +9,9 @@ PRICES: {
     "openai/gpt-4-1106-vision-preview": {"input": 0.01, "output": 0.03},
     "openai/gpt-4": {"input": 0.03, "output": 0.06},
     "openai/gpt-4-32k": {"input": 0.06, "output": 0.12},
+    "openai/gpt-4-turbo": {"input":0.01,"output": 0.03},
+    "openai/gpt-4o": {"input": 0.005,"output": 0.015},
+    "openai/gpt-4o-2024-05-13": {"input": 0.005, "output": 0.015},
     "openai/gpt-3.5-turbo-0125": {"input": 0.0005, "output": 0.0015},
     "openai/gpt-3.5-turbo-1106": {"input": 0.001, "output": 0.002},
     "openai/gpt-3.5-turbo-instruct": {"input": 0.0015, "output": 0.002},
@@ -28,6 +31,9 @@ PRICES: {
     "azure/gpt-4-1106-preview": {"input": 0.01, "output": 0.03},
     "azure/gpt-4-0125-preview": {"input": 0.01, "output": 0.03},
     "azure/gpt-4-visual-preview": {"input": 0.01, "output": 0.03},
+    "azure/gpt-4-turbo-20240409": {"input":0.01,"output": 0.03},
+    "azure/gpt-4o": {"input": 0.005,"output": 0.015},
+    "azure/gpt-4o-20240513": {"input": 0.005, "output": 0.015},
     "qwen/qwen-vl-plus": {"input": 0.008, "output": 0.008},
     "qwen/qwen-vl-max": {"input": 0.02, "output": 0.02},
 }

ufo/llm/base.py

@@ -30,6 +30,8 @@ def get_service(name):
                 module = import_module(".openai", package="ufo.llm")
             else:
                 module = import_module("." + name.lower(), package="ufo.llm")
+        else:
+            raise ValueError(f"Service {name} not found.")
         return getattr(module, service_name)
 
     def get_cost_estimator(

ufo/llm/qwen.py

@@ -21,6 +21,8 @@ def __init__(self, config, agent_type: str):
         self.config = config
         self.max_retry = self.config["MAX_RETRY"]
         self.timeout = self.config["TIMEOUT"]
+        self.api_type = self.config_llm["API_TYPE"].lower()
+        self.prices = self.config["PRICES"]
         dashscope.api_key = self.config_llm["API_KEY"]
         self.tmp_dir = None
 
@@ -91,11 +93,9 @@ def chat_completion(
                         else:
                             shutil.rmtree(self.tmp_dir, ignore_errors=True)
                             responses.append(
-                                self.parse_qwen_response(
-                                    response.output.choices[0].message.content[0][
+                                response.output.choices[0].message.content[0][
                                         "text"
                                     ]
-                                )
                             )
                             cost += _cost
                             break
@@ -164,32 +164,4 @@ def save_image_from_base64(base64_str, path, filename):
                         img_data, temp_dir, filename
                     )
                     _ = content.pop("image_url")
-        return _messages
-
-    def parse_qwen_response(self, content):
-        """
-        Parses the qwen response and returns a JSON string.
-        Args:
-            content (str): The content to be parsed.
-        Returns:
-            str: A JSON string representing the parsed content.
-        """
-        dict_str = [_.strip() for _ in content.split("\n")][1:-1]
-        _dict_str = {}
-
-        def remove_quotes(s):
-            if (s.startswith('"') and s.endswith('"')) or (
-                s.startswith("'") and s.endswith("'")
-            ):
-                return s[1:-1]
-            elif (s.startswith('"')) or (s.startswith("'")):
-                return s[1]
-            elif (s.endswith('"')) or (s.endswith("'")):
-                return s[:-1]
-            return s
-
-        for _str in dict_str:
-            key, value = _str.split(":")
-            _dict_str[remove_quotes(key)] = value.strip()
-
-        return json.dumps(_dict_str)
+        return _messages

ufo/module/session.py

@@ -221,7 +221,7 @@ def round_hostagent_execution(self) -> None:
         self._step += 1
 
         self.app_window = current_round.get_application_window()
-        self.application = self.app_window.window_text()
+        self.application = self.app_window.window_text() if self.app_window else None
         self.app_agent = self.host_agent.get_active_appagent()
 
     def round_appagent_execution(self) -> None:

ufo/prompts/share/lite/host_agent.yaml

@@ -1,4 +1,9 @@
 version: 0.1
+open_app_guideline: |-
+  - For OpenAPP operation, some Windows apps can be opened directly by calling the function OpenAPP with the arguments , here is some examples, you should put them as argument of function OpenAPP. Here are examplesL powerpoint: "powerpnt", word: "winword", outlook: "outlook", settings: "ms-settings:", file explorer: "explorer", teams: "msteams", notepad: "notepad", Microsoft To Do: "ms-todo:"
+
+open_app_comment: |-
+  - "AppsToOpen": <Default value of it is null, if the user request contains to open a specific application, this field should be a dictionary, contains 2 filed: "APP" and "filepath", this field is set as the arguments of the function OpenAPP.>
 
 system: |-
   - Your name is UFO, a virtual assistant that can help users to complete their current requests by interacting with the UI of the Windows OS and describe the content in the screenshot.