Fix tool eval scripts and update persisted tool hierarchy json data (#1035)

* Fix tool eval scripts and update persisted tool hierarchy json data

* Update namespace prompts data

* Update tool evaluation results

* Fix ToolListCommandTest

Author: JasonYeMSFT

core/Azure.Mcp.Core/src/Areas/Tools/Commands/ToolsListCommand.cs

@@ -1,9 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Runtime.InteropServices;
 using Azure.Mcp.Core.Areas.Tools.Options;
 using Azure.Mcp.Core.Commands;
 using Azure.Mcp.Core.Models.Option;
+using Microsoft.AspNetCore.Mvc.ModelBinding;
 using Microsoft.Extensions.Logging;
 
 namespace Azure.Mcp.Core.Areas.Tools.Commands;
@@ -77,23 +79,15 @@ public override async Task<CommandResponse> ExecuteAsync(CommandContext context,
                     .ToList();
 
                 // Add the commands to be surfaced directly to the list.
+                // For commands in the surfaced list, each command is exposed as a separate tool in the namespace mode.
                 foreach (var name in surfaced)
                 {
                     var subgroup = rootGroup.SubGroup.FirstOrDefault(g => string.Equals(g.Name, name, StringComparison.OrdinalIgnoreCase));
                     if (subgroup is not null)
                     {
-                        var commands = CommandFactory.GetVisibleCommands(subgroup.Commands).Select(kvp =>
-                            {
-                                var command = kvp.Value.GetCommand();
-                                return new CommandInfo
-                                {
-                                    Name = command.Name,
-                                    Description = command.Description ?? string.Empty,
-                                    Command = $"{subgroup.Name} {command.Name}"
-                                    // Omit Options and Subcommands for surfaced commands as well.
-                                };
-                            });
-                        namespaceCommands.AddRange(commands);
+                        List<CommandInfo> foundCommands = [];
+                        searchCommandInCommandGroup("", subgroup, foundCommands);
+                        namespaceCommands.AddRange(foundCommands);
                     }
                 }
 
@@ -138,4 +132,24 @@ private static CommandInfo CreateCommand(string tokenizedName, IBaseCommand comm
             Metadata = command.Metadata
         };
     }
+
+    private void searchCommandInCommandGroup(string commandPrefix, CommandGroup searchedGroup, List<CommandInfo> foundCommands)
+    {
+        var commands = CommandFactory.GetVisibleCommands(searchedGroup.Commands).Select(kvp =>
+        {
+            var command = kvp.Value.GetCommand();
+            return new CommandInfo
+            {
+                Name = $"{commandPrefix.Replace(" ", "_")}{searchedGroup.Name}_{command.Name}",
+                Description = command.Description ?? string.Empty,
+                Command = $"{(!string.IsNullOrEmpty(commandPrefix) ? commandPrefix : "")}{searchedGroup.Name} {command.Name}"
+                // Omit Options and Subcommands for surfaced commands as well.
+            };
+        });
+        foundCommands.AddRange(commands);
+        foreach (CommandGroup nextLevelSubGroup in searchedGroup.SubGroup)
+        {
+            searchCommandInCommandGroup($"{commandPrefix}{searchedGroup.Name} ", nextLevelSubGroup, foundCommands);
+        }
+    }
 }

core/Azure.Mcp.Core/tests/Azure.Mcp.Core.UnitTests/Areas/Tools/UnitTests/ToolsListCommandTests.cs

@@ -353,7 +353,8 @@ public async Task ExecuteAsync_WithNamespaceSwitch_ReturnsNamespacesOnly()
             else
             {
                 // Surfaced extension command: Command is "{namespace} {commandName}", Name is just "{commandName}"
-                Assert.EndsWith(ns.Name, ns.Command);
+                // When Azure MCP presents the commands as tools, the spaces in the commands are replaced by underscore
+                Assert.EndsWith(ns.Name, ns.Command.Replace(" ", "_"));
             }
 
             Assert.Equal(ns.Name, ns.Name.Trim());

eng/tools/ToolDescriptionEvaluator/Generate-GroupedPromptsJson.ps1

@@ -5,14 +5,14 @@ Generates prompts JSON files for consolidated tools, namespace tools, or both.
 .DESCRIPTION
 Modes:
     Consolidated - Produces consolidated-prompts.json mapping each consolidated tool name to aggregated prompts.
-    Namespace    - Produces namespace-prompts.json mapping each namespace to aggregated prompts from all descendant subcommands. Keys are prefixed with 'azmcp_' for consistency with per-command prompt keys.
+    Namespace    - Produces namespace-prompts.json mapping each namespace to aggregated prompts from all descendant subcommands.
     Both         - Produces both outputs in a single run.
 
 Reads:
     - consolidated-tools.json (contains consolidated_tools array; each tool has an mappedToolList list)
     - namespace-tools.json (contains top-level namespaces with recursive subcommands)
     - tools.json (optional for future enrichment / validation)
-    - prompts.json (maps individual command keys to prompt examples; e.g. command "azmcp acr registry list" => key "azmcp_acr_registry_list")
+    - prompts.json (maps individual command keys to prompt examples; e.g. command "acr registry list" => key "acr_registry_list")
 
 For every entity (consolidated tool name OR namespace name) the script:
     1. Resolves its underlying command strings
@@ -56,7 +56,7 @@ Idempotent. Safe to re-run. Designed to be executed from repo root or script dir
 param(
     [Parameter(Mandatory)][ValidateSet('Consolidated','Namespace','Both')][string]$Mode,
     [string]$ConsolidatedToolsPath = "../../../core/Azure.Mcp.Core/src/Areas/Server/Resources/consolidated-tools.json",
-    [string]$NamespaceToolsPath = "./namespace-tools.json",
+    [string]$NamespaceToolsPath = "./namespace-tools.json", # This file contains the output of the `azmcp tools list --namespaces` command.
     [string]$PromptsPath = "./prompts.json",
     [string]$ToolsPath = "./tools.json",
     [string]$OutputPath,
@@ -126,12 +126,12 @@ function Resolve-PromptsForCommands {
 
 <#
     NOTE: Namespace tools JSON is now a FLAT list where each element is either:
-      1. A top-level namespace entry:   { name, command: "azmcp <ns>" }
-      2. A surfaced leaf command entry: { name, command: "azmcp extension <leaf>" }
+      1. A top-level namespace entry:   { name, command: "<ns>" }
+      2. A surfaced leaf command entry: { name, command: "extension <leaf>" }
 
     Previously the structure contained recursive subcommands; the old recursive walker is retained
     only for backwards compatibility scenarios (not currently invoked). The new logic derives the
-    leaf command set for a namespace by scanning all prompt keys with the prefix 'azmcp_<ns>_'.
+    leaf command set for a namespace by scanning all prompt keys with the prefix '<ns>_'.
     Surfaced leaf commands simply aggregate their own prompts.
 #>
 function Get-NamespaceCommandStrings {
@@ -144,11 +144,11 @@ function Get-NamespaceCommandStrings {
     if ($null -eq $Node -or -not $Node.command) { return $acc }
     $cmd = ($Node.command -replace '\s+', ' ').Trim()
 
-    # Pattern: azmcp <namespace>
-    if ($cmd -match '^(azmcp)\s+([^\s]+)$') {
-        $ns = $Matches[2]
-        # Collect all prompt keys beginning with azmcp_<ns>_ (leaf commands)
-        $prefix = "azmcp_${ns}_"
+    # Pattern: <namespace>
+    if ($cmd -match '^([^\s]+)$') {
+        $ns = $Matches[1]
+        # Collect all prompt keys beginning with <ns>_ (leaf commands)
+        $prefix = "${ns}_"
         $matchingLeafKeys = $AllPromptKeys | Where-Object { $_.StartsWith($prefix, [System.StringComparison]::OrdinalIgnoreCase) }
         foreach ($k in $matchingLeafKeys) {
             # Convert prompt key back to command string: underscores -> spaces
@@ -157,7 +157,7 @@ function Get-NamespaceCommandStrings {
         return ($acc | Sort-Object -Unique)
     }
 
-    # Surfaced leaf (e.g., azmcp extension azqr) – aggregate only itself
+    # Surfaced leaf (e.g., extension azqr) – aggregate only itself
     $acc += $cmd
     return $acc
 }
@@ -233,12 +233,11 @@ function Invoke-NamespaceGeneration {
 
         $commandStrings = @(Get-NamespaceCommandStrings -Node $ns -AllPromptKeys $AllPromptKeys)
 
-        # Determine aggregation key: namespace entries get 'azmcp_<ns>', surfaced leaf commands keep their converted key
-        $isNamespace = $ns.command -match '^(azmcp)\s+([^\s]+)$'
+        # Determine aggregation key: namespace entries get '<ns>', surfaced leaf commands keep their converted key
+
+        $isNamespace = $ns.command -match '^([^\s]+)$'
         if ($isNamespace) {
-            $nsName = $Matches[2]
-            switch ($nsName) { 'azqr' { $nsName = 'extension_azqr'; break } }
-            if (-not $nsName.StartsWith('azmcp_')) { $nsName = 'azmcp_' + $nsName }
+            $nsName = $Matches[1]
             $resolved = Resolve-PromptsForCommands -CommandStrings $commandStrings -PromptsJson $PromptsJson -AllPromptKeys $AllPromptKeys -Warnings ([ref]$warnings) -VerboseWarnings:$VerboseWarnings
             $outputMap[$nsName] = @($resolved)
         }

eng/tools/ToolDescriptionEvaluator/Generate-PromptsJsonFromMarkdown.ps1

@@ -21,7 +21,7 @@ with the same base file name as the source markdown file (e.g. prompts.md -> pro
 Generates a prompts.json file based on the specified markdown in the current directory.
 
 .EXAMPLE
-./Generate-PromptsJsonFromMarkdown.ps1 -PromptsSource ./prompts.md -OutputPath ./eng/tools/ToolDescriptionEvaluator/myprompts.json
+./Generate-PromptsJsonFromMarkdown.ps1 -PromptsSource ./prompts.md -OutputPath ./myprompts.json
 Generates a JSON file with the specified name and path. If no filename is provided, the same name as the source markdown
 is used with a .json extension.
 
@@ -87,7 +87,6 @@ function Get-ToolPrompts([string]$Path) {
         if ($parts.Count -lt 2) { return }
         $tool = $parts[0]
         $prompt = $parts[1]
-        if (-not $tool.StartsWith('azmcp_')) { return }
         if (-not $prompt) { return }
         if (-not $result.ContainsKey($tool)) { $result[$tool] = @() }
         $result[$tool] += (Convert-Special-Characters $prompt)

eng/tools/ToolDescriptionEvaluator/consolidated-prompts.json

@@ -23,15 +23,16 @@
     "What is the status of function app <function_app_name>?"
   ],
   "add_azure_app_service_database": [
-    "Add a CosmosDB database to app service <app_name>",
-    "Add a database connection to my app service <app_name> in resource group <resource_group>",
-    "Add a MySQL database to app service <app_name>",
-    "Add a PostgreSQL database to app service <app_name>",
-    "Add database <database_name> on server <database_server> to app service <app_name>",
-    "Add database <database_name> with retry policy to app service <app_name>",
-    "Configure a SQL Server database for app service <app_name>",
-    "Configure tenant <tenant> for database <database_name> in app service <app_name>",
-    "Set connection string for database <database_name> in app service <app_name>"
+    "Add database connection <connection_string> for database <database_name> on server <database_server> to app service <app_name> in resource group <resource_group>",
+    "Add database connection <connection_string> to my app service <app_name> for database <database_name> in resource group <resource_group>",
+    "Add database connection string for <database_name> to app service <app_name> using connection string <connection_string> in resource group <resource_group>",
+    "Add MySQL database <database_name> to app service <app_name> using connection <connection_string> in resource group <resource_group>",
+    "Add PostgreSQL database <database_name> to app service <app_name> using connection <connection_string> in resource group <resource_group>",
+    "Configure database <database_name> for app service <app_name> with the connection string <connection_string> in resource group <resource_group>",
+    "Configure SQL Server database <database_name> for app service <app_name> with connection string <connection_string> in resource group <resource_group>",
+    "Connect CosmosDB database <database_name> using connection string <connection_string> to app service <app_name> in resource group <resource_group>",
+    "Connect database <database_name> to my app service <app_name> using connection string <connection_string> in resource group <resource_group>",
+    "Set up database <database_name> for app service <app_name> with connection string <connection_string> under resource group <resource_group>"
   ],
   "get_azure_databases_details": [
     "Display the properties of SQL server <server_name>",
@@ -197,12 +198,12 @@
     "Scan my Azure subscription for compliance recommendations"
   ],
   "generate_azure_cli_commands": [
-    "Get Azure CLI command to create a Storage account with name <storage_account_name>",
-    "Show me how to use Azure CLI to list all virtual machines in my subscription",
+    "Create a Storage account with name <storage_account_name> using Azure CLI",
+    "List all virtual machines in my subscription using Azure CLI",
     "Show me the details of the storage account <account_name> with Azure CLI commands"
   ],
   "install_azure_cli_extensions": [
-    "Get Azure CLI installation instructions",
+    "<Ask the MCP host to uninstall az cli on your machine and run test prompts for extension_cli_generate>",
     "How to install azd",
     "What is Azure Functions Core tools and how to install it"
   ],
@@ -277,6 +278,7 @@
     "Upload certificate file <file_path> to key vault <key_vault_account_name>"
   ],
   "get_azure_best_practices": [
+    "configure azure mcp in coding agent for my repo",
     "Fetch the Azure Terraform best practices",
     "Get the latest Azure best practices",
     "Get the latest Azure code generation best practices",
@@ -291,7 +293,7 @@
   ],
   "design_azure_architecture": [
     "Generate the azure architecture diagram for this application",
-    "Help me create a cloud service that will serve as ATM for users",
+    "Help me design an Azure cloud service that will serve as an ATM for users",
     "How can I design a cloud service in Azure that will store and present videos for users?",
     "I want to design a cloud app for ordering groceries",
     "Please help me design an architecture for a large-scale file upload, storage, and retrieval service"
@@ -315,11 +317,11 @@
     "Get the details of knowledge base <agent-name> in the Azure AI Search service <service-name>",
     "Get the details of knowledge source <source-name> in the Azure AI Search service <service-name>",
     "Get the schema configuration for knowledge index <index-name>",
-    "List all agents in my AI Foundry project",
+    "List all agents in my Azure AI Foundry resource",
     "List all AI Foundry model deployments",
     "List all AI Foundry models",
     "List all AI Foundry resources in my subscription",
-    "List all available OpenAI models in my Azure resource",
+    "List all available OpenAI models in my Azure AI Foundry resource",
     "List all Cognitive Search services in my subscription",
     "List all indexes in the Cognitive Search service <service-name>",
     "List all knowledge bases in the Azure AI Search service <service-name>",
@@ -331,7 +333,7 @@
     "Show me all AI Foundry model deployments",
     "Show me my Cognitive Search services",
     "Show me the AI Foundry resources in resource group <resource_group_name>",
-    "Show me the available agents in my AI Foundry project",
+    "Show me the available agents in my Azure AI Foundry resource",
     "Show me the available AI Foundry models",
     "Show me the Cognitive Search services in my subscription",
     "Show me the details of the index <index-name> in Cognitive Search service <service-name>",
@@ -343,8 +345,8 @@
     "Show me the knowledge source <source-name> in search service <service-name>",
     "Show me the knowledge sources in the Azure AI Search service <service-name>",
     "Show me the knowledge sources in the search service <service-name>",
-    "Show me the OpenAI model deployments",
-    "Show me the schema for knowledge index <index-name> in my AI Foundry project"
+    "Show me the OpenAI model deployments in my Azure AI Foundry resource",
+    "Show me the schema for knowledge index <index-name> in my Azure AI Foundry resource"
   ],
   "retrieve_azure_ai_knowledge_base_content": [
     "Ask knowledge base <agent-name> in search service <service-name> to retrieve information about <query>",
@@ -356,16 +358,16 @@
     "What does knowledge base <agent-name> in search service <service-name> know about <query>"
   ],
   "use_azure_openai_models": [
-    "Create a chat completion with the message \"Hello, how are you today?\"",
-    "Create a completion with the prompt \"What is Azure?\"",
-    "Create vector embeddings for my text using Azure OpenAI",
-    "Generate embeddings for the text \"Azure OpenAI Service\""
+    "Create a chat completion with the message \"Hello, how are you today?\" using my Azure AI Foundry resource",
+    "Create a completion with the prompt \"What is Azure?\" using my Azure AI Foundry resource",
+    "Create vector embeddings for my text using my Azure AI Foundry resource",
+    "Generate embeddings for the text \"Azure OpenAI Service\" using my Azure AI Foundry resource"
   ],
   "connect_azure_ai_foundry_agents": [
-    "Query an agent in my AI foundry project"
+    "Query an agent in my Azure AI foundry resource"
   ],
   "query_and_evaluate_azure_ai_foundry_agents": [
-    "Query and evaluate an agent in my AI Foundry project for task_adherence"
+    "Query and evaluate an agent in my Azure AI Foundry resource for task_adherence"
   ],
   "evaluate_azure_ai_foundry_agents": [
     "Evaluate the full query and response I got from my agent for task_adherence"
@@ -421,8 +423,10 @@
     "Show me the available regions for these resource types <resource_types>"
   ],
   "get_azure_messaging_service_details": [
+    "Get the details of my consumer group <consumer_group_name> in my event hub <event_hub_name>, namespace <namespace_name>, and resource group <resource_group_name>",
     "Get the details of my event hub <event_hub_name> in my namespace <namespace_name> and resource group <resource_group_name>",
     "Get the details of my namespace <namespace_name> in my resource group <resource_group_name>",
+    "List all consumer groups in my event hub <event_hub_name> in namespace <namespace_name>",
     "List all Event Grid subscriptions in subscription <subscription>",
     "List all Event Grid topics in my subscription",
     "List all Event Grid topics in resource group <resource_group_name> in subscription <subscription>",
@@ -441,10 +445,13 @@
     "Show me the Event Grid topics in my subscription"
   ],
   "edit_azure_data_analytics_resources": [
+    "Create a new consumer group <consumer_group_name> in my event hub <event_hub_name>, namespace <namespace_name>, and resource group <resource_group_name>",
     "Create a new event hub <event_hub_name> in my namespace <namespace_name> and resource group <resource_group_name>",
     "Create an new namespace <namespace_name> in my resource group <resource_group_name>",
+    "Delete my consumer group <consumer_group_name> in my event hub <event_hub_name>, namespace <namespace_name>, and resource group <resource_group_name>",
     "Delete my event hub <event_hub_name> in my namespace <namespace_name> and resource group <resource_group_name>",
     "Delete my namespace <namespace_name> in my resource group <resource_group_name>",
+    "Update my consumer group <consumer_group_name> in my event hub <event_hub_name>, namespace <namespace_name>, and resource group <resource_group_name>",
     "Update my event hub <event_hub_name> in my namespace <namespace_name> and resource group <resource_group_name>",
     "Update my namespace <namespace_name> in my resource group <resource_group_name>"
   ],