Merge pull request #705 from Joannall/master

Add drop tmp table, Refined kg search

Author: Oceania2018

src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs

@@ -90,6 +90,7 @@ public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
         {
             var numTables = workbook.NumberOfSheets;
             var commandList = new List<SqlContextOut>();
+            var state = _services.GetRequiredService<IConversationStateService>();
 
             for (int sheetIdx = 0; sheetIdx < numTables; sheetIdx++)
             {
@@ -109,15 +110,17 @@ public IEnumerable<SqlContextOut> WriteExcelDataToDB(IWorkbook workbook)
                     commandList.Add(commandResult);
                     continue;
                 }
-                var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
 
                 string table = $"{_database}.{_tableName}";
+                state.SetState("tmp_table", table);
+
+                var (isInsertSuccess, insertMessage) = SqlInsertDataFn(sheet);
                 string exampleData = GetInsertExample(table);
 
                 commandResult = new SqlContextOut
                 {
                     isSuccessful = isInsertSuccess,
-                    Message = $"{insertMessage}\r\nExample Data: {exampleData}",
+                    Message = $"{insertMessage}\r\nExample Data: {exampleData}. \r\n The remaining data contains different values. ",
                     FileName = _currentFileName
                 };
                 commandList.Add(commandResult);

src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/templates/knowledge.generation.plain.liquid

@@ -3,6 +3,9 @@ You are a knowledge extractor for knowledge base. Extract the answer in "SQL Ans
 * Skip the question/answer for tmp table.
 * Don't include tmp table in the answer.
 * Include all the explanation comments as additional knowledge.
+* Don't replace the parameter in the question.
+* Replace the specific id in answer based on the question.
+* Don't need multiple set of query solutions for the same question. Only keep one.
 
 =====
 User Questions:

src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs

@@ -25,11 +25,14 @@ public async Task<bool> Execute(RoleDialogModel message)
 
         state.SetState("max_tokens", "4096");
         var task = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
+        var searchQuestions = new List<string>(task.Questions);
+        searchQuestions.AddRange(task.NormQuestions);
+        searchQuestions = searchQuestions.Distinct().ToList();
 
         // Get knowledge from vectordb
         var hooks = _services.GetServices<IKnowledgeHook>();
         var knowledges = new List<string>();
-        foreach (var question in task.Questions)
+        foreach (var question in searchQuestions)
         {
             foreach (var hook in hooks)
             {

src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/PrimaryRequirementRequest.cs

@@ -7,4 +7,7 @@ public class PrimaryRequirementRequest
 
     [JsonPropertyName("questions")]
     public string[] Questions { get; set; } = [];
+
+    [JsonPropertyName("norm_questions")]
+    public string[] NormQuestions { get; set; } = [];
 }

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/functions/plan_primary_stage.json

@@ -10,10 +10,35 @@
       },
       "questions": {
         "type": "array",
-        "description": "Rephrase user requirements in details and in multiple ways, don't miss any information especially for those line items, values and numbers.",
+        "description": "Break down user requirements in details and in multiple ways, don't miss any entity type/value",
         "items": {
           "type": "string",
-          "description": "Question converted from requirement in different ways to search in the knowledge base, be short and you can refer to the global knowledge.One question should contain only one main topic."
+          "description": "Question converted from requirement in different ways to search in the knowledge base, be short and you can refer to the global knowledge.One question should contain only one main topic that with one entity type."
+        }
+      },
+      "norm_questions": {
+        "type": "array",
+        "description": "normalize the generated questions, remove specific entity value.",
+        "items": {
+          "type": "string",
+          "description": "Normalized question"
+        }
+      },
+      "entities": {
+        "type": "array",
+        "description": "entities with type and value",
+        "items": {
+          "type": "object",
+          "properties": {
+            "type": {
+              "type": "string",
+              "description": "entity type"
+            },
+            "value": {
+              "type": "string",
+              "description": "entity value"
+            }
+          }
         }
       }
     },

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid

@@ -8,7 +8,8 @@ Thinking process:
     - If there is extra knowledge or relationship needed between steps, set the need_breakdown_task to true for both steps.
     - If the solution mentioned "related solutions" is needed, set the need_breakdown_task to true.
     - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_breakdown_task to true.
-    - If you need to lookup the dictionary to verify or get the enum/term/dictionary value, set the need_lookup_dictionary to true.
+    - If you need to lookup the dictionary to verify or get the enum/term/dictionary value(exclude example data from attachment), set the need_lookup_dictionary to true.
+    - Don't set need_lookup_dictionary to true for attachment data.
     - Seperate the dictionary lookup and need additional information/knowledge into different subtask.
 3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@';
 4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}]

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs

@@ -41,6 +41,12 @@ public async Task<bool> Execute(RoleDialogModel message)
                 _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.")
             };
 
+            if (refinedArgs.SqlStatements.Length == 1 && refinedArgs.SqlStatements[0].StartsWith("DROP TABLE"))
+            {
+                message.Content = "Drop table successfully";
+                return true;
+            }
+
             if (results.Count() == 0)
             {
                 message.Content = "No record found";

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/verify_dictionary_term.json

@@ -1,6 +1,6 @@
 {
   "name": "verify_dictionary_term",
-  "description": "Get id from dictionary table by keyword. Call this function only if need_lookup_dictionary is true and is_insert is false",
+  "description": "Get id from dictionary table by keyword. Call this function only if need_lookup_dictionary is true and is_insert is false. You can only query one table at a time.",
   "parameters": {
     "type": "object",
     "properties": {
@@ -18,7 +18,7 @@
       },
       "tables": {
         "type": "array",
-        "description": "all related tables",
+        "description": "all related dictionary tables",
         "items": {
           "type": "string",
           "description": "table name"