diff --git a/Cargo.toml b/Cargo.toml index 1a745518..f1740c6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,7 +104,7 @@ repository.workspace = true [features] default = [] -full = ["a2a", "discord", "gateway", "index", "slack", "tui"] +full = ["a2a", "discord", "gateway", "index", "mock", "slack", "tui"] a2a = ["dep:zeph-a2a", "zeph-a2a?/server"] candle = ["zeph-llm/candle", "zeph-core/candle"] metal = ["zeph-llm/metal", "zeph-core/metal"] diff --git a/crates/zeph-core/src/agent/mod.rs b/crates/zeph-core/src/agent/mod.rs index f2556ed3..87c80175 100644 --- a/crates/zeph-core/src/agent/mod.rs +++ b/crates/zeph-core/src/agent/mod.rs @@ -539,6 +539,24 @@ impl Agent { manager.shutdown_all_shared().await; } + if let Some(ref tx) = self.metrics_tx { + let m = tx.borrow(); + if m.filter_applications > 0 { + #[allow(clippy::cast_precision_loss)] + let pct = if m.filter_raw_tokens > 0 { + m.filter_saved_tokens as f64 / m.filter_raw_tokens as f64 * 100.0 + } else { + 0.0 + }; + tracing::info!( + raw_tokens = m.filter_raw_tokens, + saved_tokens = m.filter_saved_tokens, + applications = m.filter_applications, + "tool output filtering saved ~{} tokens ({pct:.0}%)", + m.filter_saved_tokens, + ); + } + } tracing::info!("agent shutdown complete"); } @@ -1139,6 +1157,7 @@ pub(super) mod agent_tests { tool_name: "bash".to_string(), summary: "tool executed successfully".to_string(), blocks_executed: 1, + filter_stats: None, }))]); let agent_channel = MockChannel::new(vec!["execute tool".to_string()]); @@ -1336,6 +1355,7 @@ pub(super) mod agent_tests { tool_name: "bash".to_string(), summary: "[error] command failed [exit code 1]".to_string(), blocks_executed: 1, + filter_stats: None, })), Ok(None), ]); @@ -1355,6 +1375,7 @@ pub(super) mod agent_tests { tool_name: "bash".to_string(), summary: " ".to_string(), blocks_executed: 1, + filter_stats: None, }))]); let mut agent = Agent::new(provider, channel, registry, None, 5, executor); @@ -1446,6 +1467,7 @@ pub(super) mod agent_tests { tool_name: "bash".to_string(), summary: "step 1 complete".to_string(), blocks_executed: 1, + filter_stats: None, })), Ok(None), ]); @@ -1473,6 +1495,7 @@ pub(super) mod agent_tests { tool_name: "bash".to_string(), summary: "continuing".to_string(), blocks_executed: 1, + filter_stats: None, }))); } let executor = MockToolExecutor::new(outputs); diff --git a/crates/zeph-core/src/agent/streaming.rs b/crates/zeph-core/src/agent/streaming.rs index 78a0cb53..d1fd0adb 100644 --- a/crates/zeph-core/src/agent/streaming.rs +++ b/crates/zeph-core/src/agent/streaming.rs @@ -251,6 +251,7 @@ impl Agent { } /// Returns `true` if the tool loop should continue. + #[allow(clippy::too_many_lines)] pub(crate) async fn handle_tool_result( &mut self, response: &str, @@ -258,6 +259,15 @@ impl Agent { ) -> Result { match result { Ok(Some(output)) => { + if let Some(ref fs) = output.filter_stats { + let saved = fs.estimated_tokens_saved() as u64; + let raw = (fs.raw_chars / 4) as u64; + self.update_metrics(|m| { + m.filter_raw_tokens += raw; + m.filter_saved_tokens += saved; + m.filter_applications += 1; + }); + } if output.summary.trim().is_empty() { tracing::warn!("tool execution returned empty output"); self.record_skill_outcomes("success", None).await; diff --git a/crates/zeph-core/src/metrics.rs b/crates/zeph-core/src/metrics.rs index e4771f74..212b234d 100644 --- a/crates/zeph-core/src/metrics.rs +++ b/crates/zeph-core/src/metrics.rs @@ -26,6 +26,9 @@ pub struct MetricsSnapshot { pub cache_read_tokens: u64, pub cache_creation_tokens: u64, pub cost_spent_cents: f64, + pub filter_raw_tokens: u64, + pub filter_saved_tokens: u64, + pub filter_applications: u64, } pub struct MetricsCollector { @@ -89,6 +92,25 @@ mod tests { assert_eq!(cloned.provider_name, "ollama"); } + #[test] + fn filter_metrics_tracking() { + let (collector, rx) = MetricsCollector::new(); + collector.update(|m| { + m.filter_raw_tokens += 250; + m.filter_saved_tokens += 200; + m.filter_applications += 1; + }); + collector.update(|m| { + m.filter_raw_tokens += 100; + m.filter_saved_tokens += 80; + m.filter_applications += 1; + }); + let s = rx.borrow(); + assert_eq!(s.filter_raw_tokens, 350); + assert_eq!(s.filter_saved_tokens, 280); + assert_eq!(s.filter_applications, 2); + } + #[test] fn summaries_count_tracks_summarizations() { let (collector, rx) = MetricsCollector::new(); diff --git a/crates/zeph-mcp/src/executor.rs b/crates/zeph-mcp/src/executor.rs index fd7de6a3..90e373a3 100644 --- a/crates/zeph-mcp/src/executor.rs +++ b/crates/zeph-mcp/src/executor.rs @@ -67,6 +67,7 @@ impl ToolExecutor for McpToolExecutor { tool_name: "mcp".to_owned(), summary: outputs.join("\n\n"), blocks_executed, + filter_stats: None, })) } } diff --git a/crates/zeph-tools/src/composite.rs b/crates/zeph-tools/src/composite.rs index 3b8394a7..e216d9fa 100644 --- a/crates/zeph-tools/src/composite.rs +++ b/crates/zeph-tools/src/composite.rs @@ -59,6 +59,7 @@ mod tests { tool_name: "test".to_owned(), summary: "matched".to_owned(), blocks_executed: 1, + filter_stats: None, })) } } @@ -89,6 +90,7 @@ mod tests { tool_name: "test".to_owned(), summary: "second".to_owned(), blocks_executed: 1, + filter_stats: None, })) } } @@ -164,6 +166,7 @@ mod tests { tool_name: call.tool_id.clone(), summary: "file_handler".to_owned(), blocks_executed: 1, + filter_stats: None, })) } else { Ok(None) @@ -186,6 +189,7 @@ mod tests { tool_name: "bash".to_owned(), summary: "shell_handler".to_owned(), blocks_executed: 1, + filter_stats: None, })) } else { Ok(None) diff --git a/crates/zeph-tools/src/executor.rs b/crates/zeph-tools/src/executor.rs index ad511913..11a46683 100644 --- a/crates/zeph-tools/src/executor.rs +++ b/crates/zeph-tools/src/executor.rs @@ -8,12 +8,36 @@ pub struct ToolCall { pub params: HashMap, } +/// Cumulative filter statistics for a single tool execution. +#[derive(Debug, Clone, Default)] +pub struct FilterStats { + pub raw_chars: usize, + pub filtered_chars: usize, +} + +impl FilterStats { + #[must_use] + #[allow(clippy::cast_precision_loss)] + pub fn savings_pct(&self) -> f64 { + if self.raw_chars == 0 { + return 0.0; + } + (1.0 - self.filtered_chars as f64 / self.raw_chars as f64) * 100.0 + } + + #[must_use] + pub fn estimated_tokens_saved(&self) -> usize { + self.raw_chars.saturating_sub(self.filtered_chars) / 4 + } +} + /// Structured result from tool execution. #[derive(Debug, Clone)] pub struct ToolOutput { pub tool_name: String, pub summary: String, pub blocks_executed: u32, + pub filter_stats: Option, } impl fmt::Display for ToolOutput { @@ -150,6 +174,7 @@ mod tests { tool_name: "bash".to_owned(), summary: "$ echo hello\nhello".to_owned(), blocks_executed: 1, + filter_stats: None, }; assert_eq!(output.to_string(), "$ echo hello\nhello"); } @@ -241,4 +266,28 @@ mod tests { let result = exec.execute_tool_call(&call).await.unwrap(); assert!(result.is_none()); } + + #[test] + fn filter_stats_savings_pct() { + let fs = FilterStats { + raw_chars: 1000, + filtered_chars: 200, + }; + assert!((fs.savings_pct() - 80.0).abs() < 0.01); + } + + #[test] + fn filter_stats_savings_pct_zero() { + let fs = FilterStats::default(); + assert!((fs.savings_pct()).abs() < 0.01); + } + + #[test] + fn filter_stats_estimated_tokens_saved() { + let fs = FilterStats { + raw_chars: 1000, + filtered_chars: 200, + }; + assert_eq!(fs.estimated_tokens_saved(), 200); // (1000 - 200) / 4 + } } diff --git a/crates/zeph-tools/src/file.rs b/crates/zeph-tools/src/file.rs index 92639611..43d3b21b 100644 --- a/crates/zeph-tools/src/file.rs +++ b/crates/zeph-tools/src/file.rs @@ -143,6 +143,7 @@ impl FileExecutor { tool_name: "read".to_owned(), summary: selected.join("\n"), blocks_executed: 1, + filter_stats: None, })) } @@ -163,6 +164,7 @@ impl FileExecutor { tool_name: "write".to_owned(), summary: format!("Wrote {} bytes to {path_str}", content.len()), blocks_executed: 1, + filter_stats: None, })) } @@ -190,6 +192,7 @@ impl FileExecutor { tool_name: "edit".to_owned(), summary: format!("Edited {path_str}"), blocks_executed: 1, + filter_stats: None, })) } @@ -221,6 +224,7 @@ impl FileExecutor { matches.join("\n") }, blocks_executed: 1, + filter_stats: None, })) } @@ -262,6 +266,7 @@ impl FileExecutor { results.join("\n") }, blocks_executed: 1, + filter_stats: None, })) } } diff --git a/crates/zeph-tools/src/lib.rs b/crates/zeph-tools/src/lib.rs index 5af8ccfd..86968ebc 100644 --- a/crates/zeph-tools/src/lib.rs +++ b/crates/zeph-tools/src/lib.rs @@ -19,8 +19,8 @@ pub use audit::{AuditEntry, AuditLogger, AuditResult}; pub use composite::CompositeExecutor; pub use config::{AuditConfig, ScrapeConfig, ShellConfig, ToolsConfig}; pub use executor::{ - MAX_TOOL_OUTPUT_CHARS, ToolCall, ToolError, ToolEvent, ToolEventTx, ToolExecutor, ToolOutput, - truncate_tool_output, + FilterStats, MAX_TOOL_OUTPUT_CHARS, ToolCall, ToolError, ToolEvent, ToolEventTx, ToolExecutor, + ToolOutput, truncate_tool_output, }; pub use file::FileExecutor; pub use filter::{FilterConfig, FilterResult, OutputFilter, OutputFilterRegistry, sanitize_output}; diff --git a/crates/zeph-tools/src/scrape.rs b/crates/zeph-tools/src/scrape.rs index 06310e8c..e24dff63 100644 --- a/crates/zeph-tools/src/scrape.rs +++ b/crates/zeph-tools/src/scrape.rs @@ -105,6 +105,7 @@ impl ToolExecutor for WebScrapeExecutor { tool_name: "web-scrape".to_owned(), summary: outputs.join("\n\n"), blocks_executed, + filter_stats: None, })) } @@ -132,6 +133,7 @@ impl ToolExecutor for WebScrapeExecutor { tool_name: "web-scrape".to_owned(), summary: result, blocks_executed: 1, + filter_stats: None, })) } } diff --git a/crates/zeph-tools/src/shell.rs b/crates/zeph-tools/src/shell.rs index f158766c..5392cf2a 100644 --- a/crates/zeph-tools/src/shell.rs +++ b/crates/zeph-tools/src/shell.rs @@ -7,7 +7,9 @@ use schemars::JsonSchema; use crate::audit::{AuditEntry, AuditLogger, AuditResult}; use crate::config::ShellConfig; -use crate::executor::{ToolCall, ToolError, ToolEvent, ToolEventTx, ToolExecutor, ToolOutput}; +use crate::executor::{ + FilterStats, ToolCall, ToolError, ToolEvent, ToolEventTx, ToolExecutor, ToolOutput, +}; use crate::filter::{OutputFilterRegistry, sanitize_output}; use crate::permissions::{PermissionAction, PermissionPolicy}; @@ -130,6 +132,7 @@ impl ShellExecutor { } let mut outputs = Vec::with_capacity(blocks.len()); + let mut cumulative_filter_stats: Option = None; #[allow(clippy::cast_possible_truncation)] let blocks_executed = blocks.len() as u32; @@ -224,6 +227,10 @@ impl ShellExecutor { savings_pct = fr.savings_pct(), "output filter applied" ); + let stats = + cumulative_filter_stats.get_or_insert_with(FilterStats::default); + stats.raw_chars += fr.raw_chars; + stats.filtered_chars += fr.filtered_chars; fr.output } None => sanitized, @@ -238,6 +245,7 @@ impl ShellExecutor { tool_name: "bash".to_owned(), summary: outputs.join("\n\n"), blocks_executed, + filter_stats: cumulative_filter_stats, })) } diff --git a/crates/zeph-tools/src/trust_gate.rs b/crates/zeph-tools/src/trust_gate.rs index 017c0b73..bb0f834a 100644 --- a/crates/zeph-tools/src/trust_gate.rs +++ b/crates/zeph-tools/src/trust_gate.rs @@ -118,6 +118,7 @@ mod tests { tool_name: call.tool_id.clone(), summary: "ok".into(), blocks_executed: 1, + filter_stats: None, })) } } diff --git a/crates/zeph-tui/src/widgets/resources.rs b/crates/zeph-tui/src/widgets/resources.rs index 162092e7..1be0b0cc 100644 --- a/crates/zeph-tui/src/widgets/resources.rs +++ b/crates/zeph-tui/src/widgets/resources.rs @@ -27,6 +27,18 @@ pub fn render(metrics: &MetricsSnapshot, frame: &mut Frame, area: Rect) { metrics.cache_read_tokens ))); } + if metrics.filter_applications > 0 { + #[allow(clippy::cast_precision_loss)] + let pct = if metrics.filter_raw_tokens > 0 { + metrics.filter_saved_tokens as f64 / metrics.filter_raw_tokens as f64 * 100.0 + } else { + 0.0 + }; + res_lines.push(Line::from(format!( + " Filter saved: {} tok ({pct:.0}%)", + metrics.filter_saved_tokens, + ))); + } let resources = Paragraph::new(res_lines).block( Block::default() .borders(Borders::ALL) diff --git a/src/main.rs b/src/main.rs index 024cf0f8..7b6dee67 100644 --- a/src/main.rs +++ b/src/main.rs @@ -781,6 +781,7 @@ mod tests { use std::path::Path; use zeph_core::channel::Channel; use zeph_core::config::ProviderKind; + #[cfg(feature = "a2a")] use zeph_llm::ollama::OllamaProvider; #[tokio::test] diff --git a/tests/integration.rs b/tests/integration.rs index 53616c6f..ce9cd058 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -165,6 +165,7 @@ impl ToolExecutor for OutputToolExecutor { tool_name: "bash".to_string(), summary: self.output.clone(), blocks_executed: 1, + filter_stats: None, })) } } @@ -177,6 +178,7 @@ impl ToolExecutor for EmptyOutputToolExecutor { tool_name: "bash".to_string(), summary: String::new(), blocks_executed: 1, + filter_stats: None, })) } } @@ -189,6 +191,7 @@ impl ToolExecutor for ErrorOutputToolExecutor { tool_name: "bash".to_string(), summary: "[error] command failed".into(), blocks_executed: 1, + filter_stats: None, })) } } @@ -217,6 +220,7 @@ impl ToolExecutor for ConfirmToolExecutor { tool_name: "bash".to_string(), summary: "confirmed output".into(), blocks_executed: 1, + filter_stats: None, })) } } @@ -250,6 +254,7 @@ impl ToolExecutor for ExitCodeToolExecutor { tool_name: "bash".to_string(), summary: "[exit code 1] process failed".into(), blocks_executed: 1, + filter_stats: None, })) } } @@ -2154,6 +2159,7 @@ mod self_learning { tool_name: "bash".to_string(), summary: "[error] command failed".into(), blocks_executed: 1, + filter_stats: None, })) } } diff --git a/tests/performance_agent_integration.rs b/tests/performance_agent_integration.rs index fde7176c..657bb7c9 100644 --- a/tests/performance_agent_integration.rs +++ b/tests/performance_agent_integration.rs @@ -100,6 +100,7 @@ impl ToolExecutor for InstrumentedMockExecutor { tool_name: "bash".to_string(), summary: "$ echo test\ntest".to_string(), blocks_executed: 1, + filter_stats: None, })) } else { Ok(None)