Merge pull request #97 from microsoft/pre-release

Bug fixed & requirement updated

requirements.txt

@@ -14,6 +14,7 @@ lxml==5.1.0
 psutil==5.9.8
 beautifulsoup4==4.12.3
 sentence-transformers==2.5.1
+pandas==1.4.3
 ##For Qwen
 #dashscope==1.15.0
 ##For removing stopwords

ufo/automator/app_apis/excel/excelclient.py

@@ -1,7 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-from typing import Any, Dict, Type
+from typing import Any, Dict, List, Type
 
 import pandas as pd
 
@@ -53,6 +53,23 @@ def table2markdown(self, sheet_name: str) -> str:
 
         return df.to_markdown(index=False)
 
+    def insert_excel_table(
+        self, sheet_name: str, table: List[List[Any]], start_row: int, start_col: int
+    ):
+        """
+        Insert a table into the sheet.
+        :param sheet_name: The sheet name.
+        :param table: The list of lists of values to be inserted.
+        :param start_row: The start row.
+        :param start_col: The start column.
+        """
+        sheet = self.com_object.Sheets(sheet_name)
+        for i, row in enumerate(table):
+            for j, value in enumerate(row):
+                sheet.Cells(start_row + i, start_col + j).Value = value
+
+        return table
+
     @staticmethod
     def format_value(value: Any) -> str:
         """
@@ -92,3 +109,29 @@ def name(cls) -> str:
         The name of the command.
         """
         return "table2markdown"
+
+
+@ExcelWinCOMReceiver.register
+class InsertExcelTable(WinCOMCommand):
+    """
+    The command to insert a table.
+    """
+
+    def execute(self):
+        """
+        Execute the command to insert a table.
+        :return: The inserted table.
+        """
+        return self.receiver.insert_excel_table(
+            sheet_name=self.params.get("sheet_name", 1),
+            table=self.params.get("table"),
+            start_row=self.params.get("start_row", 1),
+            start_column=self.params.get("start_col", 1),
+        )
+
+    @classmethod
+    def name(cls) -> str:
+        """
+        The name of the command.
+        """
+        return "insert_excel_table"

ufo/prompts/apps/excel/api.yaml

@@ -11,4 +11,21 @@ table2markdown:
     [4] Available control item: Any control item in the Excel app.
     [5] Return: the markdown format string of the table content of the sheet.
 
+insert_excel_table:
+  summary: |-
+    "insert_excel_table" is to insert a table to the Excel sheet. The table is a list of list of strings or numbers.
+  class_name: |-
+    InsertExcelTable
+  usage: |-
+    [1] API call: insert_excel_table(table: List[List[Any]], sheet_name: str, start_row: int, start_col: int)
+    [2] Args:
+      - table: The table content to insert. The table is a list of list of strings or numbers.
+      - sheet_name: The name of the sheet to insert the table.
+      - start_row: The start row to insert the table, starting from 1.
+      - start_col: The start column to insert the table, starting from 1.
+    [3] Example: insert_excel_table(table=[['Name', 'Age', 'Gender'], ['Alice', 30, 'Female'], ['Bob', 25, 'Male'], ['Charlie', 35, 'Male']], sheet_name="Sheet1", start_row=1, start_col=1)
+    [4] Available control item: Any control item in the Excel app.
+    [5] Return: The table content is inserted to the Excel sheet.
+
+
   

ufo/prompts/share/base/api.yaml

@@ -4,7 +4,7 @@ click_input:
   class_name: |-
     ClickInputCommand
   usage: |-
-    [1] API call: click_input(button=, double)
+    [1] API call: click_input(button: str, double: bool)
     [2] Args:
       - button: 'The mouse button to click. One of ''left'', ''right'', ''middle'' or ''x'' (Default: ''left'')'
       - double: 'Whether to perform a double click or not (Default: False)'
@@ -15,11 +15,11 @@ click_input:
 
 set_edit_text:
   summary: |-
-    "set_edit_text" is to add new text to the control item. If there is already text in the control item, the new text will be appended to the end of the existing text.
+    "set_edit_text" is to add new text to the control item. If there is already text in the control item, the new text will append to the end of the existing text.
   class_name: |-
     SetEditTextCommand
   usage: |-
-    [1] API call: set_edit_text(text="")
+    [1] API call: set_edit_text(text: str="The text to input.")
     [2] Args:
       - text: The text input to the Edit control item. You must also use Double Backslash escape character to escape the single quote in the string argument.
     [3] Example: set_edit_text(text="Hello World. \\n I enjoy the reading of the book 'The Lord of the Rings'. It's a great book.")
@@ -42,26 +42,27 @@ annotation:
 
 summary:
   summary: |-
-    "summary" is to summarize your observation of the current application window base on the clean screenshot, or base on available control items. 
+    "summary" is to summarize your observation of the current application window base on the clean screenshot, or base on available control items. You must use your vision to summarize the image with required information using the argument "text". Do not add information that is not in the image.
   class_name: |-
     SummaryCommand
   usage: |-
-    [1] API call: summary(text="")
-    [2] Args: None
+    [1] API call: summary(text: str="Your description of the image.")
+    [2] Args: 
+      - text: The text description of the image with required information. 
     [3] Example: summary(text="The image shows a workflow of a AI agent framework. \\n The framework has three components: the 'data collection', the 'data processing' and the 'data analysis'.")
     [4] Available control item: All control items.
     [5] Return: the summary of the image.
 
 texts:
   summary: |-
-    "texts" is to get the text of the control item. It typical apply to Edit and Document control item when user request is to get the text of the control item.
+    "texts" is to get the text of the control item. It typical apply to Edit and Document control item when user request is to get the text of the control item. This only works for Edit and Document control items. If you want to get the text of other control items, you can use the "summary" API to describe the required information based on the screenshot by yourself.
   class_name: |-
     GetTextsCommand
   usage: |-
     [1] API call: texts()
     [2] Args: None
     [3] Example: texts()
-    [4] All control items.
+    [4] Available control item: Edit and Document control items.
     [5] Return: the text content of the control item.
 
 wheel_mouse_input:
@@ -70,7 +71,7 @@ wheel_mouse_input:
   class_name: |-
     WheelMouseInputCommand
   usage: |-
-    [1] API call: wheel_mouse_input()
+    [1] API call: wheel_mouse_input(wheel_dist: int)
     [2] Args: 
         - wheel_dist: The distance to scroll. Positive values indicate upward scrolling, negative values indicate downward scrolling.
     [3] Example: wheel_mouse_input(wheel_dist=-20)

ufo/prompts/share/lite/api.yaml

@@ -4,7 +4,7 @@ click_input:
   class_name: |-
     ClickInputCommand
   usage: |-
-    [1] API call: click_input(button=, double)
+    [1] API call: click_input(button: str, double: bool)
     [2] Args:
       - button: 'The mouse button to click. One of ''left'', ''right'', ''middle'' or ''x'' (Default: ''left'')'
       - double: 'Whether to perform a double click or not (Default: False)'
@@ -15,11 +15,11 @@ click_input:
 
 set_edit_text:
   summary: |-
-    "set_edit_text" is to add new text to the control item. If there is already text in the control item, the new text will be appended to the end of the existing text.
+    "set_edit_text" is to add new text to the control item. If there is already text in the control item, the new text will append to the end of the existing text.
   class_name: |-
     SetEditTextCommand
   usage: |-
-    [1] API call: set_edit_text(text="")
+    [1] API call: set_edit_text(text: str="The text to input.")
     [2] Args:
       - text: The text input to the Edit control item. You must also use Double Backslash escape character to escape the single quote in the string argument.
     [3] Example: set_edit_text(text="Hello World. \\n I enjoy the reading of the book 'The Lord of the Rings'. It's a great book.")
@@ -42,26 +42,27 @@ annotation:
 
 summary:
   summary: |-
-    "summary" is to summarize your observation of the current application window base on the clean screenshot, or base on available control items. 
+    "summary" is to summarize your observation of the current application window base on the clean screenshot, or base on available control items. You must use your vision to summarize the image with required information using the argument "text". Do not add information that is not in the image.
   class_name: |-
     SummaryCommand
   usage: |-
-    [1] API call: summary(text="")
-    [2] Args: None
+    [1] API call: summary(text: str="Your description of the image.")
+    [2] Args: 
+      - text: The text description of the image with required information. 
     [3] Example: summary(text="The image shows a workflow of a AI agent framework. \\n The framework has three components: the 'data collection', the 'data processing' and the 'data analysis'.")
     [4] Available control item: All control items.
     [5] Return: the summary of the image.
 
 texts:
   summary: |-
-    "texts" is to get the text of the control item. It typical apply to Edit and Document control item when user request is to get the text of the control item.
+    "texts" is to get the text of the control item. It typical apply to Edit and Document control item when user request is to get the text of the control item. This only works for Edit and Document control items. If you want to get the text of other control items, you can use the "summary" API to describe the required information based on the screenshot by yourself.
   class_name: |-
     GetTextsCommand
   usage: |-
     [1] API call: texts()
     [2] Args: None
     [3] Example: texts()
-    [4] All control items.
+    [4] Available control item: Edit and Document control items.
     [5] Return: the text content of the control item.
 
 wheel_mouse_input:
@@ -70,7 +71,7 @@ wheel_mouse_input:
   class_name: |-
     WheelMouseInputCommand
   usage: |-
-    [1] API call: wheel_mouse_input()
+    [1] API call: wheel_mouse_input(wheel_dist: int)
     [2] Args: 
         - wheel_dist: The distance to scroll. Positive values indicate upward scrolling, negative values indicate downward scrolling.
     [3] Example: wheel_mouse_input(wheel_dist=-20)