From 88f1c4c8f02206e0580f3d4b7e13fab6d315e7bf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Feb 2026 10:33:49 +0000 Subject: [PATCH 1/3] Initial plan From 2e7954068076942bb0e06bfd2f55e572b2da2aae Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Feb 2026 10:42:05 +0000 Subject: [PATCH 2/3] Add MCP tool usage statistics to audit command Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/audit.go | 8 +- pkg/cli/audit_report.go | 49 ++++++++++- pkg/cli/audit_report_render.go | 81 +++++++++++++++++ pkg/cli/gateway_logs.go | 154 +++++++++++++++++++++++++++++++++ 4 files changed, 290 insertions(+), 2 deletions(-) diff --git a/pkg/cli/audit.go b/pkg/cli/audit.go index f3bee0b87d..b2733c671e 100644 --- a/pkg/cli/audit.go +++ b/pkg/cli/audit.go @@ -310,6 +310,12 @@ func AuditWorkflowRun(ctx context.Context, runID int64, owner, repo, hostname st fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to analyze redacted domains: %v", err))) } + // Extract MCP tool usage data from gateway logs + mcpToolUsage, err := extractMCPToolUsageData(runOutputDir, verbose) + if err != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to extract MCP tool usage: %v", err))) + } + // List all artifacts artifacts, err := listArtifacts(runOutputDir) if err != nil && verbose { @@ -329,7 +335,7 @@ func AuditWorkflowRun(ctx context.Context, runID int64, owner, repo, hostname st } // Build structured audit data - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, mcpToolUsage) // Render output based on format preference if jsonOutput { diff --git a/pkg/cli/audit_report.go b/pkg/cli/audit_report.go index f9ab697eef..7bbc69878b 100644 --- a/pkg/cli/audit_report.go +++ b/pkg/cli/audit_report.go @@ -33,6 +33,7 @@ type AuditData struct { Errors []ErrorInfo `json:"errors,omitempty"` Warnings []ErrorInfo `json:"warnings,omitempty"` ToolUsage []ToolUsageInfo `json:"tool_usage,omitempty"` + MCPToolUsage *MCPToolUsageData `json:"mcp_tool_usage,omitempty"` } // Finding represents a key insight discovered during audit @@ -126,6 +127,51 @@ type ToolUsageInfo struct { MaxDuration string `json:"max_duration,omitempty" console:"header:Max Duration,omitempty"` } +// MCPToolUsageData contains detailed MCP tool usage statistics and individual call records +type MCPToolUsageData struct { + Summary []MCPToolSummary `json:"summary"` // Aggregated statistics per tool + ToolCalls []MCPToolCall `json:"tool_calls"` // Individual tool call records + Servers []MCPServerStats `json:"servers,omitempty"` // Server-level statistics +} + +// MCPToolSummary contains aggregated statistics for a single MCP tool +type MCPToolSummary struct { + ServerName string `json:"server_name" console:"header:Server"` + ToolName string `json:"tool_name" console:"header:Tool"` + CallCount int `json:"call_count" console:"header:Calls"` + TotalInputSize int `json:"total_input_size" console:"header:Total Input,format:number"` + TotalOutputSize int `json:"total_output_size" console:"header:Total Output,format:number"` + MaxInputSize int `json:"max_input_size" console:"header:Max Input,format:number"` + MaxOutputSize int `json:"max_output_size" console:"header:Max Output,format:number"` + AvgDuration string `json:"avg_duration,omitempty" console:"header:Avg Duration,omitempty"` + MaxDuration string `json:"max_duration,omitempty" console:"header:Max Duration,omitempty"` + ErrorCount int `json:"error_count,omitempty" console:"header:Errors,omitempty"` +} + +// MCPToolCall represents a single MCP tool call with full details +type MCPToolCall struct { + Timestamp string `json:"timestamp"` + ServerName string `json:"server_name"` + ToolName string `json:"tool_name"` + Method string `json:"method,omitempty"` + InputSize int `json:"input_size"` + OutputSize int `json:"output_size"` + Duration string `json:"duration,omitempty"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} + +// MCPServerStats contains server-level statistics +type MCPServerStats struct { + ServerName string `json:"server_name" console:"header:Server"` + RequestCount int `json:"request_count" console:"header:Requests"` + ToolCallCount int `json:"tool_call_count" console:"header:Tool Calls"` + TotalInputSize int `json:"total_input_size" console:"header:Total Input,format:number"` + TotalOutputSize int `json:"total_output_size" console:"header:Total Output,format:number"` + AvgDuration string `json:"avg_duration,omitempty" console:"header:Avg Duration,omitempty"` + ErrorCount int `json:"error_count,omitempty" console:"header:Errors,omitempty"` +} + // OverviewDisplay is a display-optimized version of OverviewData for console rendering type OverviewDisplay struct { RunID int64 `console:"header:Run ID"` @@ -139,7 +185,7 @@ type OverviewDisplay struct { } // buildAuditData creates structured audit data from workflow run information -func buildAuditData(processedRun ProcessedRun, metrics LogMetrics) AuditData { +func buildAuditData(processedRun ProcessedRun, metrics LogMetrics, mcpToolUsage *MCPToolUsageData) AuditData { run := processedRun.Run auditReportLog.Printf("Building audit data for run ID %d", run.DatabaseID) @@ -276,6 +322,7 @@ func buildAuditData(processedRun ProcessedRun, metrics LogMetrics) AuditData { Errors: errors, Warnings: warnings, ToolUsage: toolUsage, + MCPToolUsage: mcpToolUsage, } } diff --git a/pkg/cli/audit_report_render.go b/pkg/cli/audit_report_render.go index fbec0c68b7..56125c5d19 100644 --- a/pkg/cli/audit_report_render.go +++ b/pkg/cli/audit_report_render.go @@ -129,6 +129,13 @@ func renderConsole(data AuditData, logsPath string) { renderToolUsageTable(data.ToolUsage) } + // MCP Tool Usage Section - detailed MCP statistics + if data.MCPToolUsage != nil && len(data.MCPToolUsage.Summary) > 0 { + fmt.Fprintln(os.Stderr, console.FormatSectionHeader("MCP Tool Usage")) + fmt.Fprintln(os.Stderr) + renderMCPToolUsageTable(data.MCPToolUsage) + } + // Errors and Warnings Section if len(data.Errors) > 0 || len(data.Warnings) > 0 { fmt.Fprintln(os.Stderr, console.FormatSectionHeader("Errors and Warnings")) @@ -259,6 +266,80 @@ func renderToolUsageTable(toolUsage []ToolUsageInfo) { fmt.Fprint(os.Stderr, console.RenderTable(config)) } +// renderMCPToolUsageTable renders MCP tool usage with detailed statistics +func renderMCPToolUsageTable(mcpData *MCPToolUsageData) { + auditReportLog.Printf("Rendering MCP tool usage table with %d tools", len(mcpData.Summary)) + + // Render server-level statistics first + if len(mcpData.Servers) > 0 { + fmt.Fprintln(os.Stderr, " Server Statistics:") + fmt.Fprintln(os.Stderr) + + serverConfig := console.TableConfig{ + Headers: []string{"Server", "Requests", "Tool Calls", "Total Input", "Total Output", "Avg Duration", "Errors"}, + Rows: make([][]string, 0, len(mcpData.Servers)), + } + + for _, server := range mcpData.Servers { + inputStr := console.FormatFileSize(int64(server.TotalInputSize)) + outputStr := console.FormatFileSize(int64(server.TotalOutputSize)) + durationStr := server.AvgDuration + if durationStr == "" { + durationStr = "N/A" + } + errorStr := fmt.Sprintf("%d", server.ErrorCount) + if server.ErrorCount == 0 { + errorStr = "-" + } + + row := []string{ + stringutil.Truncate(server.ServerName, 25), + fmt.Sprintf("%d", server.RequestCount), + fmt.Sprintf("%d", server.ToolCallCount), + inputStr, + outputStr, + durationStr, + errorStr, + } + serverConfig.Rows = append(serverConfig.Rows, row) + } + + fmt.Fprint(os.Stderr, console.RenderTable(serverConfig)) + fmt.Fprintln(os.Stderr) + } + + // Render tool-level statistics + if len(mcpData.Summary) > 0 { + fmt.Fprintln(os.Stderr, " Tool Statistics:") + fmt.Fprintln(os.Stderr) + + toolConfig := console.TableConfig{ + Headers: []string{"Server", "Tool", "Calls", "Total In", "Total Out", "Max In", "Max Out"}, + Rows: make([][]string, 0, len(mcpData.Summary)), + } + + for _, tool := range mcpData.Summary { + totalInStr := console.FormatFileSize(int64(tool.TotalInputSize)) + totalOutStr := console.FormatFileSize(int64(tool.TotalOutputSize)) + maxInStr := console.FormatFileSize(int64(tool.MaxInputSize)) + maxOutStr := console.FormatFileSize(int64(tool.MaxOutputSize)) + + row := []string{ + stringutil.Truncate(tool.ServerName, 20), + stringutil.Truncate(tool.ToolName, 30), + fmt.Sprintf("%d", tool.CallCount), + totalInStr, + totalOutStr, + maxInStr, + maxOutStr, + } + toolConfig.Rows = append(toolConfig.Rows, row) + } + + fmt.Fprint(os.Stderr, console.RenderTable(toolConfig)) + } +} + // renderFirewallAnalysis renders firewall analysis with summary and domain breakdown func renderFirewallAnalysis(analysis *FirewallAnalysis) { // Summary statistics diff --git a/pkg/cli/gateway_logs.go b/pkg/cli/gateway_logs.go index a32e3f3c3c..8b3362c54e 100644 --- a/pkg/cli/gateway_logs.go +++ b/pkg/cli/gateway_logs.go @@ -22,6 +22,7 @@ import ( "github.com/githubnext/gh-aw/pkg/console" "github.com/githubnext/gh-aw/pkg/logger" + "github.com/githubnext/gh-aw/pkg/timeutil" ) var gatewayLogsLog = logger.New("cli:gateway_logs") @@ -375,6 +376,159 @@ func truncateString(s string, maxLen int) string { return s[:maxLen-3] + "..." } +// extractMCPToolUsageData creates detailed MCP tool usage data from gateway metrics +func extractMCPToolUsageData(logDir string, verbose bool) (*MCPToolUsageData, error) { + // Parse gateway logs + gatewayMetrics, err := parseGatewayLogs(logDir, verbose) + if err != nil { + // Return nil if gateway.jsonl doesn't exist (not an error for workflows without MCP) + if strings.Contains(err.Error(), "not found") { + return nil, nil + } + return nil, fmt.Errorf("failed to parse gateway logs: %w", err) + } + + if gatewayMetrics == nil || len(gatewayMetrics.Servers) == 0 { + return nil, nil + } + + mcpData := &MCPToolUsageData{ + Summary: []MCPToolSummary{}, + ToolCalls: []MCPToolCall{}, + Servers: []MCPServerStats{}, + } + + // Read gateway.jsonl again to get individual tool call records + gatewayLogPath := filepath.Join(logDir, "gateway.jsonl") + file, err := os.Open(gatewayLogPath) + if err != nil { + return nil, fmt.Errorf("failed to open gateway.jsonl: %w", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + var entry GatewayLogEntry + if err := json.Unmarshal([]byte(line), &entry); err != nil { + continue // Skip malformed lines + } + + // Only process tool call events + if entry.Event == "tool_call" || entry.Event == "rpc_call" || entry.Event == "request" { + toolName := entry.ToolName + if toolName == "" { + toolName = entry.Method + } + + // Skip entries without tool information + if entry.ServerName == "" || toolName == "" { + continue + } + + // Create individual tool call record + toolCall := MCPToolCall{ + Timestamp: entry.Timestamp, + ServerName: entry.ServerName, + ToolName: toolName, + Method: entry.Method, + InputSize: entry.InputSize, + OutputSize: entry.OutputSize, + Status: entry.Status, + Error: entry.Error, + } + + if entry.Duration > 0 { + toolCall.Duration = timeutil.FormatDuration(time.Duration(entry.Duration * float64(time.Millisecond))) + } + + mcpData.ToolCalls = append(mcpData.ToolCalls, toolCall) + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading gateway.jsonl: %w", err) + } + + // Build summary statistics from aggregated metrics + for serverName, serverMetrics := range gatewayMetrics.Servers { + // Server-level stats + serverStats := MCPServerStats{ + ServerName: serverName, + RequestCount: serverMetrics.RequestCount, + ToolCallCount: serverMetrics.ToolCallCount, + TotalInputSize: 0, + TotalOutputSize: 0, + ErrorCount: serverMetrics.ErrorCount, + } + + if serverMetrics.RequestCount > 0 { + avgDur := serverMetrics.TotalDuration / float64(serverMetrics.RequestCount) + serverStats.AvgDuration = timeutil.FormatDuration(time.Duration(avgDur * float64(time.Millisecond))) + } + + // Tool-level stats + for toolName, toolMetrics := range serverMetrics.Tools { + summary := MCPToolSummary{ + ServerName: serverName, + ToolName: toolName, + CallCount: toolMetrics.CallCount, + TotalInputSize: toolMetrics.TotalInputSize, + TotalOutputSize: toolMetrics.TotalOutputSize, + MaxInputSize: 0, // Will be calculated below + MaxOutputSize: 0, // Will be calculated below + ErrorCount: toolMetrics.ErrorCount, + } + + if toolMetrics.AvgDuration > 0 { + summary.AvgDuration = timeutil.FormatDuration(time.Duration(toolMetrics.AvgDuration * float64(time.Millisecond))) + } + if toolMetrics.MaxDuration > 0 { + summary.MaxDuration = timeutil.FormatDuration(time.Duration(toolMetrics.MaxDuration * float64(time.Millisecond))) + } + + // Calculate max input/output sizes from individual tool calls + for _, tc := range mcpData.ToolCalls { + if tc.ServerName == serverName && tc.ToolName == toolName { + if tc.InputSize > summary.MaxInputSize { + summary.MaxInputSize = tc.InputSize + } + if tc.OutputSize > summary.MaxOutputSize { + summary.MaxOutputSize = tc.OutputSize + } + } + } + + mcpData.Summary = append(mcpData.Summary, summary) + + // Update server totals + serverStats.TotalInputSize += toolMetrics.TotalInputSize + serverStats.TotalOutputSize += toolMetrics.TotalOutputSize + } + + mcpData.Servers = append(mcpData.Servers, serverStats) + } + + // Sort summaries by server name, then tool name + sort.Slice(mcpData.Summary, func(i, j int) bool { + if mcpData.Summary[i].ServerName != mcpData.Summary[j].ServerName { + return mcpData.Summary[i].ServerName < mcpData.Summary[j].ServerName + } + return mcpData.Summary[i].ToolName < mcpData.Summary[j].ToolName + }) + + // Sort servers by name + sort.Slice(mcpData.Servers, func(i, j int) bool { + return mcpData.Servers[i].ServerName < mcpData.Servers[j].ServerName + }) + + return mcpData, nil +} + // displayAggregatedGatewayMetrics aggregates and displays gateway metrics across all processed runs func displayAggregatedGatewayMetrics(processedRuns []ProcessedRun, outputDir string, verbose bool) { // Aggregate gateway metrics from all runs From b746202fcc221143a648e709d83970688e2c2eb8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Feb 2026 10:45:57 +0000 Subject: [PATCH 3/3] Add comprehensive tests for MCP tool usage statistics Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/audit_agent_example_test.go | 4 +- pkg/cli/audit_agent_output_test.go | 2 +- pkg/cli/audit_input_size_test.go | 2 +- pkg/cli/audit_mcp_tool_usage_test.go | 209 +++++++++++++++++++++++++++ pkg/cli/audit_report.go | 38 ++--- pkg/cli/audit_report_helpers_test.go | 4 +- pkg/cli/audit_report_render.go | 4 +- pkg/cli/audit_report_test.go | 6 +- pkg/cli/audit_test.go | 4 +- scratchpad/serena-tools-data.json | 11 +- 10 files changed, 243 insertions(+), 41 deletions(-) create mode 100644 pkg/cli/audit_mcp_tool_usage_test.go diff --git a/pkg/cli/audit_agent_example_test.go b/pkg/cli/audit_agent_example_test.go index 0e9426edcd..f95d470248 100644 --- a/pkg/cli/audit_agent_example_test.go +++ b/pkg/cli/audit_agent_example_test.go @@ -109,7 +109,7 @@ func TestAgentFriendlyOutputExample(t *testing.T) { } // Build audit data - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) // Test JSON output t.Run("JSON Output", func(t *testing.T) { @@ -301,7 +301,7 @@ func TestAgentFriendlyOutputFailureScenario(t *testing.T) { } // Build audit data - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) // Test failure analysis t.Run("Failure Analysis", func(t *testing.T) { diff --git a/pkg/cli/audit_agent_output_test.go b/pkg/cli/audit_agent_output_test.go index a687d6572c..7a083739f0 100644 --- a/pkg/cli/audit_agent_output_test.go +++ b/pkg/cli/audit_agent_output_test.go @@ -485,7 +485,7 @@ func TestAuditDataJSONStructure(t *testing.T) { } // Build audit data - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) // Marshal to JSON jsonBytes, err := json.MarshalIndent(auditData, "", " ") diff --git a/pkg/cli/audit_input_size_test.go b/pkg/cli/audit_input_size_test.go index db5cd7eb46..0b42bfbacf 100644 --- a/pkg/cli/audit_input_size_test.go +++ b/pkg/cli/audit_input_size_test.go @@ -130,7 +130,7 @@ func TestAuditDataJSONIncludesInputSizes(t *testing.T) { } // Build audit data - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) // Verify tool usage data includes input sizes if len(auditData.ToolUsage) == 0 { diff --git a/pkg/cli/audit_mcp_tool_usage_test.go b/pkg/cli/audit_mcp_tool_usage_test.go new file mode 100644 index 0000000000..d755263043 --- /dev/null +++ b/pkg/cli/audit_mcp_tool_usage_test.go @@ -0,0 +1,209 @@ +//go:build !integration + +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExtractMCPToolUsageData(t *testing.T) { + tests := []struct { + name string + logContent string + wantServers int + wantTools int + wantToolCalls int + wantErr bool + checkServerStats bool + expectedCallCount int + }{ + { + name: "valid gateway log with tool calls", + logContent: `{"timestamp":"2024-01-12T10:00:00Z","level":"info","type":"request","event":"tool_call","server_name":"github","tool_name":"search_issues","method":"search_issues","duration":150.5,"input_size":1024,"output_size":5120,"status":"success"} +{"timestamp":"2024-01-12T10:00:01Z","level":"info","type":"request","event":"tool_call","server_name":"github","tool_name":"search_issues","method":"search_issues","duration":200.3,"input_size":512,"output_size":6144,"status":"success"} +{"timestamp":"2024-01-12T10:00:02Z","level":"info","type":"request","event":"tool_call","server_name":"github","tool_name":"get_repository","method":"get_repository","duration":100.0,"input_size":256,"output_size":2048,"status":"success"} +`, + wantServers: 1, + wantTools: 2, + wantToolCalls: 3, + wantErr: false, + checkServerStats: true, + expectedCallCount: 3, + }, + { + name: "multiple servers", + logContent: `{"timestamp":"2024-01-12T10:00:00Z","level":"info","type":"request","event":"tool_call","server_name":"github","tool_name":"search_issues","duration":150.5,"input_size":1024,"output_size":5120,"status":"success"} +{"timestamp":"2024-01-12T10:00:01Z","level":"info","type":"request","event":"tool_call","server_name":"playwright","tool_name":"navigate","duration":250.0,"input_size":512,"output_size":1024,"status":"success"} +`, + wantServers: 2, + wantTools: 2, + wantToolCalls: 2, + wantErr: false, + }, + { + name: "tool call with errors", + logContent: `{"timestamp":"2024-01-12T10:00:00Z","level":"info","type":"request","event":"tool_call","server_name":"github","tool_name":"search_issues","duration":50.0,"input_size":100,"output_size":0,"status":"error","error":"connection timeout"} +{"timestamp":"2024-01-12T10:00:01Z","level":"info","type":"request","event":"tool_call","server_name":"github","tool_name":"search_issues","duration":100.0,"input_size":200,"output_size":1000,"status":"success"} +`, + wantServers: 1, + wantTools: 1, + wantToolCalls: 2, + wantErr: false, + }, + { + name: "no gateway.jsonl file", + logContent: "", + wantServers: 0, + wantTools: 0, + wantToolCalls: 0, + wantErr: false, // Should return nil, not an error + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + + // Only create gateway.jsonl if there's content + if tt.logContent != "" { + gatewayLogPath := filepath.Join(tmpDir, "gateway.jsonl") + err := os.WriteFile(gatewayLogPath, []byte(tt.logContent), 0644) + require.NoError(t, err, "Failed to write test gateway.jsonl") + } + + // Extract MCP tool usage data + mcpData, err := extractMCPToolUsageData(tmpDir, false) + + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + + // If no content, expect nil data + if tt.logContent == "" { + assert.Nil(t, mcpData, "Expected nil data when gateway.jsonl doesn't exist") + return + } + + require.NotNil(t, mcpData, "Expected non-nil MCP data") + + // Verify server count + assert.Len(t, mcpData.Servers, tt.wantServers, "Server count mismatch") + + // Verify tool summary count + assert.Len(t, mcpData.Summary, tt.wantTools, "Tool summary count mismatch") + + // Verify individual tool calls count + assert.Len(t, mcpData.ToolCalls, tt.wantToolCalls, "Tool calls count mismatch") + + // Additional checks for server statistics + if tt.checkServerStats && len(mcpData.Servers) > 0 { + server := mcpData.Servers[0] + assert.Equal(t, "github", server.ServerName, "Server name mismatch") + assert.Equal(t, tt.expectedCallCount, server.ToolCallCount, "Tool call count mismatch") + assert.Positive(t, server.TotalInputSize, "Total input size should be positive") + assert.Positive(t, server.TotalOutputSize, "Total output size should be positive") + } + }) + } +} + +func TestMCPToolSummaryCalculations(t *testing.T) { + tmpDir := t.TempDir() + + // Create a log with multiple calls to the same tool with varying sizes + logContent := `{"timestamp":"2024-01-12T10:00:00Z","level":"info","type":"request","event":"tool_call","server_name":"github","tool_name":"search_issues","duration":100.0,"input_size":500,"output_size":2000,"status":"success"} +{"timestamp":"2024-01-12T10:00:01Z","level":"info","type":"request","event":"tool_call","server_name":"github","tool_name":"search_issues","duration":150.0,"input_size":1500,"output_size":8000,"status":"success"} +{"timestamp":"2024-01-12T10:00:02Z","level":"info","type":"request","event":"tool_call","server_name":"github","tool_name":"search_issues","duration":120.0,"input_size":800,"output_size":5000,"status":"success"} +` + + gatewayLogPath := filepath.Join(tmpDir, "gateway.jsonl") + err := os.WriteFile(gatewayLogPath, []byte(logContent), 0644) + require.NoError(t, err) + + mcpData, err := extractMCPToolUsageData(tmpDir, false) + require.NoError(t, err) + require.NotNil(t, mcpData) + + // Verify we have exactly one tool summary + require.Len(t, mcpData.Summary, 1, "Should have exactly one tool summary") + tool := mcpData.Summary[0] + + // Verify aggregated statistics + assert.Equal(t, "github", tool.ServerName) + assert.Equal(t, "search_issues", tool.ToolName) + assert.Equal(t, 3, tool.CallCount, "Should have 3 calls") + assert.Equal(t, 2800, tool.TotalInputSize, "Total input: 500+1500+800 = 2800") + assert.Equal(t, 15000, tool.TotalOutputSize, "Total output: 2000+8000+5000 = 15000") + assert.Equal(t, 1500, tool.MaxInputSize, "Max input should be 1500") + assert.Equal(t, 8000, tool.MaxOutputSize, "Max output should be 8000") + + // Verify we have 3 individual tool call records + require.Len(t, mcpData.ToolCalls, 3, "Should have 3 tool call records") + + // Verify individual tool call data + for i, tc := range mcpData.ToolCalls { + assert.Equal(t, "github", tc.ServerName, "Tool call %d: server name mismatch", i) + assert.Equal(t, "search_issues", tc.ToolName, "Tool call %d: tool name mismatch", i) + assert.Equal(t, "success", tc.Status, "Tool call %d: status mismatch", i) + assert.NotEmpty(t, tc.Timestamp, "Tool call %d: timestamp should not be empty", i) + assert.NotEmpty(t, tc.Duration, "Tool call %d: duration should not be empty", i) + } +} + +func TestBuildAuditDataWithMCPToolUsage(t *testing.T) { + tmpDir := t.TempDir() + + // Create a simple gateway log + logContent := `{"timestamp":"2024-01-12T10:00:00Z","level":"info","type":"request","event":"tool_call","server_name":"github","tool_name":"search_issues","duration":100.0,"input_size":1024,"output_size":5120,"status":"success"} +` + gatewayLogPath := filepath.Join(tmpDir, "gateway.jsonl") + err := os.WriteFile(gatewayLogPath, []byte(logContent), 0644) + require.NoError(t, err) + + // Extract MCP data + mcpData, err := extractMCPToolUsageData(tmpDir, false) + require.NoError(t, err) + require.NotNil(t, mcpData) + + // Create a ProcessedRun with minimal data + processedRun := ProcessedRun{ + Run: WorkflowRun{ + DatabaseID: 12345, + WorkflowName: "Test Workflow", + Status: "completed", + Conclusion: "success", + }, + } + + // Create LogMetrics with minimal data + metrics := LogMetrics{ + TokenUsage: 1000, + EstimatedCost: 0.01, + Turns: 5, + } + + // Build audit data + auditData := buildAuditData(processedRun, metrics, mcpData) + + // Verify MCP tool usage is included + require.NotNil(t, auditData.MCPToolUsage, "MCP tool usage should be included in audit data") + assert.Len(t, auditData.MCPToolUsage.Summary, 1, "Should have one tool summary") + assert.Len(t, auditData.MCPToolUsage.ToolCalls, 1, "Should have one tool call") + assert.Len(t, auditData.MCPToolUsage.Servers, 1, "Should have one server") + + // Verify the summary data + tool := auditData.MCPToolUsage.Summary[0] + assert.Equal(t, "github", tool.ServerName) + assert.Equal(t, "search_issues", tool.ToolName) + assert.Equal(t, 1, tool.CallCount) + assert.Equal(t, 1024, tool.TotalInputSize) + assert.Equal(t, 5120, tool.TotalOutputSize) +} diff --git a/pkg/cli/audit_report.go b/pkg/cli/audit_report.go index 7bbc69878b..b1ed27260e 100644 --- a/pkg/cli/audit_report.go +++ b/pkg/cli/audit_report.go @@ -136,29 +136,29 @@ type MCPToolUsageData struct { // MCPToolSummary contains aggregated statistics for a single MCP tool type MCPToolSummary struct { - ServerName string `json:"server_name" console:"header:Server"` - ToolName string `json:"tool_name" console:"header:Tool"` - CallCount int `json:"call_count" console:"header:Calls"` - TotalInputSize int `json:"total_input_size" console:"header:Total Input,format:number"` - TotalOutputSize int `json:"total_output_size" console:"header:Total Output,format:number"` - MaxInputSize int `json:"max_input_size" console:"header:Max Input,format:number"` - MaxOutputSize int `json:"max_output_size" console:"header:Max Output,format:number"` - AvgDuration string `json:"avg_duration,omitempty" console:"header:Avg Duration,omitempty"` - MaxDuration string `json:"max_duration,omitempty" console:"header:Max Duration,omitempty"` - ErrorCount int `json:"error_count,omitempty" console:"header:Errors,omitempty"` + ServerName string `json:"server_name" console:"header:Server"` + ToolName string `json:"tool_name" console:"header:Tool"` + CallCount int `json:"call_count" console:"header:Calls"` + TotalInputSize int `json:"total_input_size" console:"header:Total Input,format:number"` + TotalOutputSize int `json:"total_output_size" console:"header:Total Output,format:number"` + MaxInputSize int `json:"max_input_size" console:"header:Max Input,format:number"` + MaxOutputSize int `json:"max_output_size" console:"header:Max Output,format:number"` + AvgDuration string `json:"avg_duration,omitempty" console:"header:Avg Duration,omitempty"` + MaxDuration string `json:"max_duration,omitempty" console:"header:Max Duration,omitempty"` + ErrorCount int `json:"error_count,omitempty" console:"header:Errors,omitempty"` } // MCPToolCall represents a single MCP tool call with full details type MCPToolCall struct { - Timestamp string `json:"timestamp"` - ServerName string `json:"server_name"` - ToolName string `json:"tool_name"` - Method string `json:"method,omitempty"` - InputSize int `json:"input_size"` - OutputSize int `json:"output_size"` - Duration string `json:"duration,omitempty"` - Status string `json:"status"` - Error string `json:"error,omitempty"` + Timestamp string `json:"timestamp"` + ServerName string `json:"server_name"` + ToolName string `json:"tool_name"` + Method string `json:"method,omitempty"` + InputSize int `json:"input_size"` + OutputSize int `json:"output_size"` + Duration string `json:"duration,omitempty"` + Status string `json:"status"` + Error string `json:"error,omitempty"` } // MCPServerStats contains server-level statistics diff --git a/pkg/cli/audit_report_helpers_test.go b/pkg/cli/audit_report_helpers_test.go index cf995fefcd..3a4eb8018d 100644 --- a/pkg/cli/audit_report_helpers_test.go +++ b/pkg/cli/audit_report_helpers_test.go @@ -320,7 +320,7 @@ func TestConsoleOutputIncludesFileInfo(t *testing.T) { } // Build audit data - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) auditData.DownloadedFiles = downloadedFiles // Verify downloaded files are in audit data @@ -455,7 +455,7 @@ func TestAuditReportFileListingIntegration(t *testing.T) { } // Build audit data with the extracted files - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) // The buildAuditData should have extracted files automatically if len(auditData.DownloadedFiles) == 0 { diff --git a/pkg/cli/audit_report_render.go b/pkg/cli/audit_report_render.go index 56125c5d19..c8dfd625bd 100644 --- a/pkg/cli/audit_report_render.go +++ b/pkg/cli/audit_report_render.go @@ -269,12 +269,12 @@ func renderToolUsageTable(toolUsage []ToolUsageInfo) { // renderMCPToolUsageTable renders MCP tool usage with detailed statistics func renderMCPToolUsageTable(mcpData *MCPToolUsageData) { auditReportLog.Printf("Rendering MCP tool usage table with %d tools", len(mcpData.Summary)) - + // Render server-level statistics first if len(mcpData.Servers) > 0 { fmt.Fprintln(os.Stderr, " Server Statistics:") fmt.Fprintln(os.Stderr) - + serverConfig := console.TableConfig{ Headers: []string{"Server", "Requests", "Tool Calls", "Total Input", "Total Output", "Avg Duration", "Errors"}, Rows: make([][]string, 0, len(mcpData.Servers)), diff --git a/pkg/cli/audit_report_test.go b/pkg/cli/audit_report_test.go index 48f8f2d40c..9775c1ebac 100644 --- a/pkg/cli/audit_report_test.go +++ b/pkg/cli/audit_report_test.go @@ -905,7 +905,7 @@ func TestBuildAuditDataComplete(t *testing.T) { } // Build audit data - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) // Verify overview t.Run("Overview", func(t *testing.T) { @@ -1018,7 +1018,7 @@ func TestBuildAuditDataMinimal(t *testing.T) { metrics := workflow.LogMetrics{} - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) // Should still produce valid data assert.Equal(t, int64(1), auditData.Overview.RunID, @@ -1129,7 +1129,7 @@ func TestToolUsageAggregation(t *testing.T) { }, } - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) // Tool usage should be aggregated // The exact aggregation depends on workflow.PrettifyToolName behavior diff --git a/pkg/cli/audit_test.go b/pkg/cli/audit_test.go index 465f4bb071..e0c622fa31 100644 --- a/pkg/cli/audit_test.go +++ b/pkg/cli/audit_test.go @@ -636,7 +636,7 @@ func TestBuildAuditData(t *testing.T) { } // Build audit data - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) // Verify overview if auditData.Overview.RunID != 123456 { @@ -984,7 +984,7 @@ func TestBuildAuditDataWithFirewall(t *testing.T) { } // Build audit data - auditData := buildAuditData(processedRun, metrics) + auditData := buildAuditData(processedRun, metrics, nil) // Verify firewall analysis is included if auditData.FirewallAnalysis == nil { diff --git a/scratchpad/serena-tools-data.json b/scratchpad/serena-tools-data.json index 277c91768a..3bdfa01b89 100644 --- a/scratchpad/serena-tools-data.json +++ b/scratchpad/serena-tools-data.json @@ -132,14 +132,7 @@ } ], "serena_tools_detail": { - "used_tools": [ - "mcp__serena__check_onboarding_performed", - "mcp__serena__find_symbol", - "mcp__serena__get_current_config", - "mcp__serena__initial_instructions", - "mcp__serena__list_memories", - "mcp__serena__search_for_pattern" - ], + "used_tools": ["mcp__serena__check_onboarding_performed", "mcp__serena__find_symbol", "mcp__serena__get_current_config", "mcp__serena__initial_instructions", "mcp__serena__list_memories", "mcp__serena__search_for_pattern"], "unused_registered_tools": [ "serena___activate_project", "serena___delete_memory", @@ -448,4 +441,4 @@ } ] } -} \ No newline at end of file +}