diff --git a/crates/chat-cli/src/cli/chat/cli/mod.rs b/crates/chat-cli/src/cli/chat/cli/mod.rs index 464f6269cf..a8b2eb69ff 100644 --- a/crates/chat-cli/src/cli/chat/cli/mod.rs +++ b/crates/chat-cli/src/cli/chat/cli/mod.rs @@ -132,7 +132,11 @@ pub enum SlashCommand { impl SlashCommand { pub async fn execute(self, os: &mut Os, session: &mut ChatSession) -> Result { match self { - Self::Quit => Ok(ChatState::Exit), + Self::Quit => { + // Flush all pending retention checks before quitting + session.conversation.flush_all_retention_metrics(os, "quit").await.ok(); + Ok(ChatState::Exit) + }, Self::Clear(args) => args.execute(session).await, Self::Agent(subcommand) => subcommand.execute(os, session).await, Self::Profile => { diff --git a/crates/chat-cli/src/cli/chat/conversation.rs b/crates/chat-cli/src/cli/chat/conversation.rs index a9c424900b..267a3872c2 100644 --- a/crates/chat-cli/src/cli/chat/conversation.rs +++ b/crates/chat-cli/src/cli/chat/conversation.rs @@ -260,6 +260,7 @@ impl ConversationState { } /// Enter tangent mode - creates checkpoint of current state + /// Allows exploring side topics without affecting main conversation pub fn enter_tangent_mode(&mut self) { if self.tangent_state.is_none() { self.tangent_state = Some(self.create_checkpoint()); @@ -960,6 +961,132 @@ Return only the JSON configuration, no additional text.", Ok(()) } + + pub async fn check_due_retention_metrics(&mut self, os: &Os) -> Result, ChatError> { + let mut all_results = Vec::new(); + let message_id = self.message_id().map(|s| s.to_string()); + let model_id = self.model_info.as_ref().map(|m| m.model_id.clone()); + + for (path, tracker) in &mut self.file_line_tracker { + match os.fs.read_to_string(path).await { + Ok(content) => { + let results = tracker.check_due_retention(&content); + + for (conversation_id, tool_use_id, retained, total, source) in results { + debug!("Retention check for {}: {}/{} lines retained, tool_use_id: {}, source: {}", + path, retained, total, tool_use_id, source); + + // Send retention metric with source + os.telemetry + .send_agent_contribution_metric_with_source( + &os.database, + conversation_id, + message_id.clone(), + Some(tool_use_id.clone()), + None, + None, + None, + Some(retained), + Some(total), + Some(source), + model_id.clone(), + ) + .await + .ok(); + + all_results.push((tool_use_id, retained, total)); + } + } + Err(_) => { + // File not found - emit metrics for all pending checks with file_not_found reason + for check in &tracker.pending_retention_checks { + debug!("File not found during retention check: {}, tool_use_id: {}", path, check.tool_use_id); + + os.telemetry + .send_agent_contribution_metric_with_source( + &os.database, + check.conversation_id.clone(), + message_id.clone(), + Some(check.tool_use_id.clone()), + None, + None, + None, + Some(0), // retained = 0 since file doesn't exist + Some(check.lines.len()), + Some("file_not_found".to_string()), + model_id.clone(), + ) + .await + .ok(); + } + // Clear pending checks since file is gone + tracker.pending_retention_checks.clear(); + } + } + } + Ok(all_results) + } + + pub async fn flush_all_retention_metrics(&mut self, os: &Os, source: &str) -> Result<(), ChatError> { + let message_id = self.message_id().map(|s| s.to_string()); + let model_id = self.model_info.as_ref().map(|m| m.model_id.clone()); + + for (path, tracker) in &mut self.file_line_tracker { + match os.fs.read_to_string(path).await { + Ok(content) => { + let results = tracker.flush_all_retention_checks(&content, source); + + for (conversation_id, tool_use_id, retained, total, source) in results { + debug!("Flushing retention check for {}: {}/{} lines retained, tool_use_id: {}, source: {}", + path, retained, total, tool_use_id, source); + + os.telemetry + .send_agent_contribution_metric_with_source( + &os.database, + conversation_id, + message_id.clone(), + Some(tool_use_id.clone()), + None, + None, + None, + Some(retained), + Some(total), + Some(source), + model_id.clone(), + ) + .await + .ok(); + } + } + Err(_) => { + // File not found - emit metrics for all pending checks with file_not_found reason + for check in &tracker.pending_retention_checks { + debug!("File not found during flush: {}, tool_use_id: {}", path, check.tool_use_id); + + os.telemetry + .send_agent_contribution_metric_with_source( + &os.database, + check.conversation_id.clone(), + message_id.clone(), + Some(check.tool_use_id.clone()), + None, + None, + None, + Some(0), // retained = 0 since file doesn't exist + Some(check.lines.len()), + Some("file_not_found".to_string()), + model_id.clone(), + ) + .await + .ok(); + } + // Clear pending checks since file is gone + tracker.pending_retention_checks.clear(); + } + } + } + Ok(()) + } } pub fn format_tool_spec(tool_spec: HashMap) -> HashMap> { diff --git a/crates/chat-cli/src/cli/chat/line_tracker.rs b/crates/chat-cli/src/cli/chat/line_tracker.rs index 1717d16fe7..70fbe6c86d 100644 --- a/crates/chat-cli/src/cli/chat/line_tracker.rs +++ b/crates/chat-cli/src/cli/chat/line_tracker.rs @@ -2,6 +2,8 @@ use serde::{ Deserialize, Serialize, }; +use std::collections::HashSet; +use std::time::{SystemTime, UNIX_EPOCH}; /// Contains metadata for tracking user and agent contribution metrics for a given file for /// `fs_write` tool uses. @@ -19,6 +21,17 @@ pub struct FileLineTracker { pub lines_removed_by_agent: usize, /// Whether or not this is the first `fs_write` invocation pub is_first_write: bool, + /// Pending retention checks scheduled for 1 minute (changed from 15 minutes for testing) + #[serde(default)] + pub pending_retention_checks: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetentionCheck { + pub lines: Vec, + pub scheduled_time: u64, + pub conversation_id: String, + pub tool_use_id: String, } impl Default for FileLineTracker { @@ -30,6 +43,7 @@ impl Default for FileLineTracker { lines_added_by_agent: 0, lines_removed_by_agent: 0, is_first_write: true, + pending_retention_checks: Vec::new(), } } } @@ -42,4 +56,92 @@ impl FileLineTracker { pub fn lines_by_agent(&self) -> isize { (self.lines_added_by_agent + self.lines_removed_by_agent) as isize } + + pub fn schedule_retention_check(&mut self, lines: Vec, conversation_id: String, tool_use_id: String) { + let scheduled_time = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + 900; // 15 minutes from now + + self.pending_retention_checks.push(RetentionCheck { + lines, + scheduled_time, + conversation_id, + tool_use_id, + }); + } + + pub fn flush_pending_checks_for_agent_rewrite(&mut self, file_content: &str) -> Vec<(String, String, usize, usize, String)> { + let mut results = Vec::new(); + let file_lines: HashSet<&str> = file_content.lines().collect(); + + for check in self.pending_retention_checks.drain(..) { + let retained = check.lines.iter() + .filter(|line| file_lines.contains(line.as_str())) + .count(); + + results.push(( + check.conversation_id, + check.tool_use_id, + retained, + check.lines.len(), + "agent_rewrite".to_string(), + )); + } + + results + } + + pub fn check_due_retention(&mut self, file_content: &str) -> Vec<(String, String, usize, usize, String)> { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + + let mut results = Vec::new(); + let mut remaining_checks = Vec::new(); + let file_lines: HashSet<&str> = file_content.lines().collect(); + + for check in self.pending_retention_checks.drain(..) { + if now >= check.scheduled_time { + let retained = check.lines.iter() + .filter(|line| file_lines.contains(line.as_str())) + .count(); + + results.push(( + check.conversation_id, + check.tool_use_id, + retained, + check.lines.len(), + "15min_check".to_string(), + )); + } else { + remaining_checks.push(check); + } + } + + self.pending_retention_checks = remaining_checks; + results + } + + pub fn flush_all_retention_checks(&mut self, file_content: &str, source: &str) -> Vec<(String, String, usize, usize, String)> { + let mut results = Vec::new(); + let file_lines: HashSet<&str> = file_content.lines().collect(); + + for check in self.pending_retention_checks.drain(..) { + let retained = check.lines.iter() + .filter(|line| file_lines.contains(line.as_str())) + .count(); + + results.push(( + check.conversation_id, + check.tool_use_id, + retained, + check.lines.len(), + source.to_string(), + )); + } + + results + } } diff --git a/crates/chat-cli/src/cli/chat/mod.rs b/crates/chat-cli/src/cli/chat/mod.rs index 81043fdad3..29909bbaa5 100644 --- a/crates/chat-cli/src/cli/chat/mod.rs +++ b/crates/chat-cli/src/cli/chat/mod.rs @@ -201,6 +201,90 @@ use crate::telemetry::core::{ RecordUserTurnCompletionArgs, ToolUseEventBuilder, }; + +/// Wraps ToolUseEventBuilder with user interaction data for complete tracking +#[derive(Debug)] +pub struct AgentContributionMetric { + pub base: ToolUseEventBuilder, + pub user_decision: Option, + pub lines_suggested: Option, + pub lines_accepted: Option, + pub lines_rejected: Option, + pub lines_retained: Option, + pub total_lines_checked: Option, +} + +/// User's final decision on tool execution after seeing preview +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UserDecision { + Accept, // 'y' - user accepted the tool + Reject, // 'n' - user rejected the tool + Trust, // 't' - user trusted and accepted the tool +} + +impl AgentContributionMetric { + pub fn new(conv_id: String, tool_use_id: String, model: Option) -> Self { + Self { + base: ToolUseEventBuilder::new(conv_id, tool_use_id, model), + user_decision: None, + lines_suggested: None, + lines_accepted: None, + lines_rejected: None, + lines_retained: None, + total_lines_checked: None, + } + } + + /// Initialize metric with tool content during validation phase (step 1) + pub fn init_with_tool_content(&mut self, tool_name: &str, content: &str) { + self.base.tool_name = Some(tool_name.to_string()); + self.lines_suggested = Some(content.lines().count()); + } + + /// Finalize metric with user decision after y/n/t response (step 5) + pub fn finalize_with_user_decision(&mut self, decision: UserDecision) { + self.user_decision = Some(decision); + + match decision { + UserDecision::Accept => { + self.base.is_accepted = true; + self.base.is_trusted = false; + self.lines_accepted = self.lines_suggested; + self.lines_rejected = Some(0); + }, + UserDecision::Trust => { + self.base.is_accepted = true; + self.base.is_trusted = true; + self.lines_accepted = self.lines_suggested; + self.lines_rejected = Some(0); + }, + UserDecision::Reject => { + self.base.is_accepted = false; + self.base.is_trusted = false; + self.lines_accepted = Some(0); + self.lines_rejected = self.lines_suggested; + } + } + } +} + +/// Extract content from tool input for line counting +/// Supports fs_write (file_text) and execute_bash (command) tools +fn extract_tool_content(tool_name: &str, tool_input: &serde_json::Value) -> Option { + match tool_name { + "fs_write" => { + tool_input.get("file_text") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }, + "execute_bash" => { + tool_input.get("command") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }, + _ => None + } +} use crate::telemetry::{ ReasonCode, TelemetryResult, @@ -662,8 +746,9 @@ pub struct ChatSession { tool_turn_start_time: Option, /// [RequestMetadata] about the ongoing operation. user_turn_request_metadata: Vec, - /// Telemetry events to be sent as part of the conversation. The HashMap key is tool_use_id. - tool_use_telemetry_events: HashMap, + /// Enhanced telemetry events for agent contribution tracking. The HashMap key is tool_use_id. + /// Tracks complete user interaction flow: suggestion → preview → decision → execution + tool_use_telemetry_events: HashMap, /// State used to keep track of tool use relation tool_use_status: ToolUseStatus, /// Any failed requests that could be useful for error report/debugging @@ -1426,8 +1511,90 @@ impl ChatSession { self.inner = Some(ChatState::HandleInput { input: user_input }); } - while !matches!(self.inner, Some(ChatState::Exit)) { - self.next(os).await?; + // Set up signal handler for graceful shutdown + #[cfg(unix)] + { + use tokio::signal::unix::{signal, SignalKind}; + let mut sigint = signal(SignalKind::interrupt()).map_err(|e| ChatError::Custom(format!("Failed to setup SIGINT handler: {}", e).into()))?; + let mut sigterm = signal(SignalKind::terminate()).map_err(|e| ChatError::Custom(format!("Failed to setup SIGTERM handler: {}", e).into()))?; + + loop { + tokio::select! { + // Handle normal chat operations + result = async { + if !matches!(self.inner, Some(ChatState::Exit)) { + self.next(os).await + } else { + Ok(()) + } + } => { + if let Err(e) = result { + return Err(e.into()); + } + if matches!(self.inner, Some(ChatState::Exit)) { + break; + } + + // Check for due retention metrics on each interaction + if let Ok(retention_results) = self.conversation.check_due_retention_metrics(os).await { + let mut metrics_to_emit = Vec::new(); + + for (tool_use_id, retained, total) in retention_results { + if let Some(metric) = self.tool_use_telemetry_events.remove(&tool_use_id) { + let mut updated_metric = metric; + updated_metric.lines_retained = Some(retained); + updated_metric.total_lines_checked = Some(total); + debug!("Emitting retention for tool_use_id {}: {}/{} lines retained", + tool_use_id, retained, total); + metrics_to_emit.push(updated_metric); + } + } + + for mut metric in metrics_to_emit { + metric.base.user_input_id = self.conversation.message_id().map(|v| v.to_string()); + os.telemetry.send_tool_use_suggested(&os.database, metric.base).await.ok(); + } + } + } + // Handle SIGINT (Ctrl+C) and SIGTERM + _ = sigint.recv() => { + self.conversation.flush_all_retention_metrics(os, "shutdown").await.ok(); + break; + } + _ = sigterm.recv() => { + self.conversation.flush_all_retention_metrics(os, "shutdown").await.ok(); + break; + } + } + } + } + + #[cfg(not(unix))] + { + while !matches!(self.inner, Some(ChatState::Exit)) { + self.next(os).await?; + + // Check for due retention metrics on each interaction + if let Ok(retention_results) = self.conversation.check_due_retention_metrics(os).await { + let mut metrics_to_emit = Vec::new(); + + for (tool_use_id, retained, total) in retention_results { + if let Some(metric) = self.tool_use_telemetry_events.remove(&tool_use_id) { + let mut updated_metric = metric; + updated_metric.lines_retained = Some(retained); + updated_metric.total_lines_checked = Some(total); + debug!("Emitting retention for tool_use_id {}: {}/{} lines retained", + tool_use_id, retained, total); + metrics_to_emit.push(updated_metric); + } + } + + for mut metric in metrics_to_emit { + metric.base.user_input_id = self.conversation.message_id().map(|v| v.to_string()); + os.telemetry.send_tool_use_suggested(&os.database, metric.base).await.ok(); + } + } + } } Ok(()) @@ -2228,11 +2395,32 @@ impl ChatSession { } } - // Check for a pending tool approval + // Check for a pending tool approval (step 3: user decision) if let Some(index) = self.pending_tool_index { let is_trust = ["t", "T"].contains(&input); + let is_accept = ["y", "Y"].contains(&input); + let is_reject = ["n", "N"].contains(&input); + let tool_use = &mut self.tool_uses[index]; - if ["y", "Y"].contains(&input) || is_trust { + + // Finalize telemetry with user decision (step 5: complete metric) + if let Some(metric) = self.tool_use_telemetry_events.get_mut(&tool_use.id) { + let decision = if is_trust { + UserDecision::Trust + } else if is_accept { + UserDecision::Accept + } else if is_reject { + UserDecision::Reject + } else { + // Invalid input, don't finalize yet - return to prompt + return Ok(ChatState::PromptUser { skip_printing_tools: false }); + }; + + // Complete the metric with final user decision and line counts + metric.finalize_with_user_decision(decision); + } + + if is_accept || is_trust { if is_trust { let formatted_tool_name = self .conversation @@ -2408,7 +2596,7 @@ impl ChatSession { tool.accepted = true; self.tool_use_telemetry_events .entry(tool.id.clone()) - .and_modify(|ev| ev.is_trusted = true); + .and_modify(|metric| metric.base.is_trusted = true); continue; } @@ -2427,19 +2615,19 @@ impl ChatSession { for tool in &self.tool_uses { let tool_start = std::time::Instant::now(); let mut tool_telemetry = self.tool_use_telemetry_events.entry(tool.id.clone()); - tool_telemetry = tool_telemetry.and_modify(|ev| { - ev.is_accepted = true; + tool_telemetry = tool_telemetry.and_modify(|metric| { + metric.base.is_accepted = true; }); // Extract AWS service name and operation name if available if let Some(additional_info) = tool.tool.get_additional_info() { if let Some(aws_service_name) = additional_info.get("aws_service_name").and_then(|v| v.as_str()) { tool_telemetry = - tool_telemetry.and_modify(|ev| ev.aws_service_name = Some(aws_service_name.to_string())); + tool_telemetry.and_modify(|metric| metric.base.aws_service_name = Some(aws_service_name.to_string())); } if let Some(aws_operation_name) = additional_info.get("aws_operation_name").and_then(|v| v.as_str()) { tool_telemetry = - tool_telemetry.and_modify(|ev| ev.aws_operation_name = Some(aws_operation_name.to_string())); + tool_telemetry.and_modify(|metric| metric.base.aws_operation_name = Some(aws_operation_name.to_string())); } } @@ -2546,16 +2734,16 @@ impl ChatSession { let tool_end_time = Instant::now(); let tool_time = tool_end_time.duration_since(tool_start); - tool_telemetry = tool_telemetry.and_modify(|ev| { - ev.execution_duration = Some(tool_time); - ev.turn_duration = self.tool_turn_start_time.map(|t| tool_end_time.duration_since(t)); + tool_telemetry = tool_telemetry.and_modify(|metric| { + metric.base.execution_duration = Some(tool_time); + metric.base.turn_duration = self.tool_turn_start_time.map(|t| tool_end_time.duration_since(t)); }); if let Tool::Custom(ct) = &tool.tool { - tool_telemetry = tool_telemetry.and_modify(|ev| { - ev.is_custom_tool = true; + tool_telemetry = tool_telemetry.and_modify(|metric| { + metric.base.is_custom_tool = true; // legacy fields previously implemented for only MCP tools - ev.custom_tool_call_latency = Some(tool_time.as_secs() as usize); - ev.input_token_size = Some(ct.get_input_token_size()); + metric.base.custom_tool_call_latency = Some(tool_time.as_secs() as usize); + metric.base.input_token_size = Some(ct.get_input_token_size()); }); } let tool_time = format!("{}.{}", tool_time.as_secs(), tool_time.subsec_millis()); @@ -2599,10 +2787,10 @@ impl ChatSession { } execute!(self.stdout, style::Print("\n\n"))?; - tool_telemetry = tool_telemetry.and_modify(|ev| ev.is_success = Some(true)); + tool_telemetry = tool_telemetry.and_modify(|metric| metric.base.is_success = Some(true)); if let Tool::Custom(_) = &tool.tool { tool_telemetry - .and_modify(|ev| ev.output_token_size = Some(TokenCounter::count_tokens(&result.as_str()))); + .and_modify(|metric| metric.base.output_token_size = Some(TokenCounter::count_tokens(&result.as_str()))); } // Send telemetry for agent contribution @@ -2610,6 +2798,7 @@ impl ChatSession { let sanitized_path_str = w.path(os).to_string_lossy().to_string(); let conversation_id = self.conversation.conversation_id().to_string(); let message_id = self.conversation.message_id().map(|s| s.to_string()); + let model_id = self.conversation.model_info.as_ref().map(|m| m.model_id.clone()); if let Some(tracker) = self.conversation.file_line_tracker.get_mut(&sanitized_path_str) { let lines_by_agent = tracker.lines_by_agent(); let lines_by_user = tracker.lines_by_user(); @@ -2617,16 +2806,49 @@ impl ChatSession { os.telemetry .send_agent_contribution_metric( &os.database, - conversation_id, - message_id, + conversation_id.clone(), + message_id.clone(), Some(tool.id.clone()), // Already a String Some(tool.name.clone()), // Already a String Some(lines_by_agent), Some(lines_by_user), + None, + None, + model_id.clone(), ) .await .ok(); + // Flush any pending retention checks before scheduling new ones (agent rewrite scenario) + if let Ok(current_content) = os.fs.read_to_string(&w.path(os)).await { + let flush_results = tracker.flush_pending_checks_for_agent_rewrite(¤t_content); + let model_id = self.conversation.model_info.as_ref().map(|m| m.model_id.clone()); + for (conv_id, tool_use_id, retained, total, source) in flush_results { + os.telemetry + .send_agent_contribution_metric_with_source( + &os.database, + conv_id, + message_id.clone(), + Some(tool_use_id), + None, + None, + None, + Some(retained), + Some(total), + Some(source), + model_id.clone(), + ) + .await + .ok(); + } + } + + // Schedule retention check for 15 minutes + let agent_lines = w.extract_agent_lines(); + if !agent_lines.is_empty() { + tracker.schedule_retention_check(agent_lines, conversation_id, tool.id.clone()); + } + tracker.prev_fswrite_lines = tracker.after_fswrite_lines; } } @@ -2653,9 +2875,9 @@ impl ChatSession { style::Print("\n\n"), )?; - tool_telemetry.and_modify(|ev| { - ev.is_success = Some(false); - ev.reason_desc = Some(err.to_string()); + tool_telemetry.and_modify(|metric| { + metric.base.is_success = Some(false); + metric.base.reason_desc = Some(err.to_string()); }); tool_results.push(ToolUseResult { tool_use_id: tool.id.clone(), @@ -3225,14 +3447,22 @@ impl ChatSession { let tool_use_id = tool_use.id.clone(); let tool_use_name = tool_use.name.clone(); let tool_input = tool_use.args.clone(); - let mut tool_telemetry = ToolUseEventBuilder::new( + // Create enhanced metric for agent contribution tracking (step 1: validation) + let mut tool_telemetry = AgentContributionMetric::new( conv_id.clone(), tool_use.id.clone(), self.conversation.model_info.as_ref().map(|m| m.model_id.clone()), - ) - .set_tool_use_id(tool_use_id.clone()) - .set_tool_name(tool_use.name.clone()) - .utterance_id(self.conversation.message_id().map(|s| s.to_string())); + ); + tool_telemetry.base = tool_telemetry.base + .set_tool_use_id(tool_use_id.clone()) + .set_tool_name(tool_use.name.clone()) + .utterance_id(self.conversation.message_id().map(|s| s.to_string())); + + // Initialize with tool content for line counting (step 2: content analysis) + if let Some(content) = extract_tool_content(&tool_use_name, &tool_input) { + tool_telemetry.init_with_tool_content(&tool_use_name, &content); + } + match self.conversation.tool_manager.get_tool_from_tool_use(tool_use).await { Ok(mut tool) => { // Apply non-Q-generated context to tools @@ -3240,7 +3470,7 @@ impl ChatSession { match tool.validate(os).await { Ok(()) => { - tool_telemetry.is_valid = Some(true); + tool_telemetry.base.is_valid = Some(true); queued_tools.push(QueuedTool { id: tool_use_id.clone(), name: tool_use_name, @@ -3250,7 +3480,7 @@ impl ChatSession { }); }, Err(err) => { - tool_telemetry.is_valid = Some(false); + tool_telemetry.base.is_valid = Some(false); tool_results.push(ToolUseResult { tool_use_id: tool_use_id.clone(), content: vec![ToolUseResultBlock::Text(format!( @@ -3262,7 +3492,7 @@ impl ChatSession { }; }, Err(err) => { - tool_telemetry.is_valid = Some(false); + tool_telemetry.base.is_valid = Some(false); tool_results.push(err.into()); }, } @@ -3577,15 +3807,18 @@ impl ChatSession { generated_prompt } + /// Send enhanced agent contribution telemetry with complete user interaction data + /// Approach 2: Single metric per tool with all information (suggestion + decision + execution) async fn send_tool_use_telemetry(&mut self, os: &Os) { - for (_, mut event) in self.tool_use_telemetry_events.drain() { - event.user_input_id = match self.tool_use_status { + for (_, mut metric) in self.tool_use_telemetry_events.drain() { + metric.base.user_input_id = match self.tool_use_status { ToolUseStatus::Idle => self.conversation.message_id(), ToolUseStatus::RetryInProgress(ref id) => Some(id.as_str()), } .map(|v| v.to_string()); - os.telemetry.send_tool_use_suggested(&os.database, event).await.ok(); + // Send complete metric via existing ToolUseSuggested event + os.telemetry.send_tool_use_suggested(&os.database, metric.base).await.ok(); } } diff --git a/crates/chat-cli/src/cli/chat/tools/fs_write.rs b/crates/chat-cli/src/cli/chat/tools/fs_write.rs index 09a64058a1..2a1498b33b 100644 --- a/crates/chat-cli/src/cli/chat/tools/fs_write.rs +++ b/crates/chat-cli/src/cli/chat/tools/fs_write.rs @@ -261,6 +261,23 @@ impl FsWrite { Ok(()) } + pub fn extract_agent_lines(&self) -> Vec { + match self { + FsWrite::Create { file_text, .. } => { + file_text.as_ref().map(|s| s.lines().map(|l| l.to_string()).collect()).unwrap_or_default() + }, + FsWrite::StrReplace { new_str, .. } => { + new_str.lines().map(|s| s.to_string()).collect() + }, + FsWrite::Insert { new_str, .. } => { + new_str.lines().map(|s| s.to_string()).collect() + }, + FsWrite::Append { new_str, .. } => { + new_str.lines().map(|s| s.to_string()).collect() + }, + } + } + async fn calculate_diff_lines(&self, os: &Os) -> Result<(usize, usize)> { let path = self.path(os); diff --git a/crates/chat-cli/src/telemetry/core.rs b/crates/chat-cli/src/telemetry/core.rs index b33a3cf027..c5e4c76bcc 100644 --- a/crates/chat-cli/src/telemetry/core.rs +++ b/crates/chat-cli/src/telemetry/core.rs @@ -369,6 +369,10 @@ impl Event { tool_name, lines_by_agent, lines_by_user, + lines_retained, + total_lines_checked, + source: _, + model, } => Some( CodewhispererterminalAgentContribution { create_time: self.created_time, @@ -380,6 +384,9 @@ impl Event { codewhispererterminal_tool_name: tool_name.map(CodewhispererterminalToolName), codewhispererterminal_lines_by_agent: lines_by_agent.map(|count| count as i64).map(Into::into), codewhispererterminal_lines_by_user: lines_by_user.map(|count| count as i64).map(Into::into), + codewhispererterminal_lines_retained: lines_retained.map(|count| count as i64).map(Into::into), + codewhispererterminal_total_lines_checked: total_lines_checked.map(|count| count as i64).map(Into::into), + codewhispererterminal_model: model.map(Into::into), } .into_metric_datum(), ), @@ -686,6 +693,10 @@ pub enum EventType { tool_name: Option, lines_by_agent: Option, lines_by_user: Option, + lines_retained: Option, + total_lines_checked: Option, + source: Option, + model: Option, }, McpServerInit { conversation_id: String, diff --git a/crates/chat-cli/src/telemetry/mod.rs b/crates/chat-cli/src/telemetry/mod.rs index 1ee5da2374..5ac5985a3d 100644 --- a/crates/chat-cli/src/telemetry/mod.rs +++ b/crates/chat-cli/src/telemetry/mod.rs @@ -296,6 +296,39 @@ impl TelemetryThread { tool_name: Option, lines_by_agent: Option, lines_by_user: Option, + lines_retained: Option, + total_lines_checked: Option, + model: Option, + ) -> Result<(), TelemetryError> { + self.send_agent_contribution_metric_with_source( + database, + conversation_id, + utterance_id, + tool_use_id, + tool_name, + lines_by_agent, + lines_by_user, + lines_retained, + total_lines_checked, + None, + model, + ).await + } + + #[allow(clippy::too_many_arguments)] // TODO: Should make a parameters struct. + pub async fn send_agent_contribution_metric_with_source( + &self, + database: &Database, + conversation_id: String, + utterance_id: Option, + tool_use_id: Option, + tool_name: Option, + lines_by_agent: Option, + lines_by_user: Option, + lines_retained: Option, + total_lines_checked: Option, + source: Option, + model: Option, ) -> Result<(), TelemetryError> { let mut telemetry_event = Event::new(EventType::AgentContribution { conversation_id, @@ -304,6 +337,10 @@ impl TelemetryThread { tool_name, lines_by_agent, lines_by_user, + lines_retained, + total_lines_checked, + source, + model, }); set_event_metadata(database, &mut telemetry_event).await; Ok(self.tx.send(telemetry_event)?) diff --git a/crates/chat-cli/telemetry_definitions.json b/crates/chat-cli/telemetry_definitions.json index b2bc7044bf..8775d09685 100644 --- a/crates/chat-cli/telemetry_definitions.json +++ b/crates/chat-cli/telemetry_definitions.json @@ -303,6 +303,16 @@ "name": "codewhispererterminal_linesByUser", "type": "int", "description": "The number of lines of code contributed by user" + }, + { + "name": "codewhispererterminal_linesRetained", + "type": "int", + "description": "The number of lines retained after retention check" + }, + { + "name": "codewhispererterminal_totalLinesChecked", + "type": "int", + "description": "The total number of lines checked during retention check" } ], "metrics": [ @@ -481,7 +491,10 @@ { "type": "codewhispererterminal_toolUseId" }, { "type": "codewhispererterminal_toolName" }, { "type": "codewhispererterminal_linesByAgent" }, - { "type": "codewhispererterminal_linesByUser" } + { "type": "codewhispererterminal_linesByUser" }, + { "type": "codewhispererterminal_linesRetained", "required": false }, + { "type": "codewhispererterminal_totalLinesChecked", "required": false }, + { "type": "codewhispererterminal_model", "required": false } ] }, {