From b081e39446f048a24d1fabe8296f9ff5e1fa668f Mon Sep 17 00:00:00 2001 From: Dhruvi Gajjar Date: Mon, 3 Nov 2025 17:53:38 -0800 Subject: [PATCH 1/4] added logic to emit Agent Contribution metric for reject code cases too --- crates/chat-cli/src/cli/chat/mod.rs | 177 +++++++++++++++++++++++----- 1 file changed, 145 insertions(+), 32 deletions(-) diff --git a/crates/chat-cli/src/cli/chat/mod.rs b/crates/chat-cli/src/cli/chat/mod.rs index 81043fdad3..80f18d421a 100644 --- a/crates/chat-cli/src/cli/chat/mod.rs +++ b/crates/chat-cli/src/cli/chat/mod.rs @@ -201,6 +201,86 @@ use crate::telemetry::core::{ RecordUserTurnCompletionArgs, ToolUseEventBuilder, }; + +/// Wraps ToolUseEventBuilder with user interaction data for complete tracking +#[derive(Debug)] +pub struct AgentContributionMetric { + pub base: ToolUseEventBuilder, + pub user_decision: Option, + pub lines_suggested: Option, + pub lines_accepted: Option, + pub lines_rejected: Option, +} + +/// User's final decision on tool execution after seeing preview +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UserDecision { + Accept, // 'y' - user accepted the tool + Reject, // 'n' - user rejected the tool + Trust, // 't' - user trusted and accepted the tool +} + +impl AgentContributionMetric { + pub fn new(conv_id: String, tool_use_id: String, model: Option) -> Self { + Self { + base: ToolUseEventBuilder::new(conv_id, tool_use_id, model), + user_decision: None, + lines_suggested: None, + lines_accepted: None, + lines_rejected: None, + } + } + + /// Initialize metric with tool content during validation phase (step 1) + pub fn init_with_tool_content(&mut self, tool_name: &str, content: &str) { + self.base.tool_name = Some(tool_name.to_string()); + self.lines_suggested = Some(content.lines().count()); + } + + /// Finalize metric with user decision after y/n/t response (step 5) + pub fn finalize_with_user_decision(&mut self, decision: UserDecision) { + self.user_decision = Some(decision); + + match decision { + UserDecision::Accept => { + self.base.is_accepted = true; + self.base.is_trusted = false; + self.lines_accepted = self.lines_suggested; + self.lines_rejected = Some(0); + }, + UserDecision::Trust => { + self.base.is_accepted = true; + self.base.is_trusted = true; + self.lines_accepted = self.lines_suggested; + self.lines_rejected = Some(0); + }, + UserDecision::Reject => { + self.base.is_accepted = false; + self.base.is_trusted = false; + self.lines_accepted = Some(0); + self.lines_rejected = self.lines_suggested; + } + } + } +} + +/// Extract content from tool input for line counting +/// Supports fs_write (file_text) and execute_bash (command) tools +fn extract_tool_content(tool_name: &str, tool_input: &serde_json::Value) -> Option { + match tool_name { + "fs_write" => { + tool_input.get("file_text") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }, + "execute_bash" => { + tool_input.get("command") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }, + _ => None + } +} use crate::telemetry::{ ReasonCode, TelemetryResult, @@ -662,8 +742,9 @@ pub struct ChatSession { tool_turn_start_time: Option, /// [RequestMetadata] about the ongoing operation. user_turn_request_metadata: Vec, - /// Telemetry events to be sent as part of the conversation. The HashMap key is tool_use_id. - tool_use_telemetry_events: HashMap, + /// Enhanced telemetry events for agent contribution tracking. The HashMap key is tool_use_id. + /// Tracks complete user interaction flow: suggestion → preview → decision → execution + tool_use_telemetry_events: HashMap, /// State used to keep track of tool use relation tool_use_status: ToolUseStatus, /// Any failed requests that could be useful for error report/debugging @@ -2228,11 +2309,32 @@ impl ChatSession { } } - // Check for a pending tool approval + // Check for a pending tool approval (step 3: user decision) if let Some(index) = self.pending_tool_index { let is_trust = ["t", "T"].contains(&input); + let is_accept = ["y", "Y"].contains(&input); + let is_reject = ["n", "N"].contains(&input); + let tool_use = &mut self.tool_uses[index]; - if ["y", "Y"].contains(&input) || is_trust { + + // Finalize telemetry with user decision (step 5: complete metric) + if let Some(metric) = self.tool_use_telemetry_events.get_mut(&tool_use.id) { + let decision = if is_trust { + UserDecision::Trust + } else if is_accept { + UserDecision::Accept + } else if is_reject { + UserDecision::Reject + } else { + // Invalid input, don't finalize yet - return to prompt + return Ok(ChatState::PromptUser { skip_printing_tools: false }); + }; + + // Complete the metric with final user decision and line counts + metric.finalize_with_user_decision(decision); + } + + if is_accept || is_trust { if is_trust { let formatted_tool_name = self .conversation @@ -2408,7 +2510,7 @@ impl ChatSession { tool.accepted = true; self.tool_use_telemetry_events .entry(tool.id.clone()) - .and_modify(|ev| ev.is_trusted = true); + .and_modify(|metric| metric.base.is_trusted = true); continue; } @@ -2427,19 +2529,19 @@ impl ChatSession { for tool in &self.tool_uses { let tool_start = std::time::Instant::now(); let mut tool_telemetry = self.tool_use_telemetry_events.entry(tool.id.clone()); - tool_telemetry = tool_telemetry.and_modify(|ev| { - ev.is_accepted = true; + tool_telemetry = tool_telemetry.and_modify(|metric| { + metric.base.is_accepted = true; }); // Extract AWS service name and operation name if available if let Some(additional_info) = tool.tool.get_additional_info() { if let Some(aws_service_name) = additional_info.get("aws_service_name").and_then(|v| v.as_str()) { tool_telemetry = - tool_telemetry.and_modify(|ev| ev.aws_service_name = Some(aws_service_name.to_string())); + tool_telemetry.and_modify(|metric| metric.base.aws_service_name = Some(aws_service_name.to_string())); } if let Some(aws_operation_name) = additional_info.get("aws_operation_name").and_then(|v| v.as_str()) { tool_telemetry = - tool_telemetry.and_modify(|ev| ev.aws_operation_name = Some(aws_operation_name.to_string())); + tool_telemetry.and_modify(|metric| metric.base.aws_operation_name = Some(aws_operation_name.to_string())); } } @@ -2546,16 +2648,16 @@ impl ChatSession { let tool_end_time = Instant::now(); let tool_time = tool_end_time.duration_since(tool_start); - tool_telemetry = tool_telemetry.and_modify(|ev| { - ev.execution_duration = Some(tool_time); - ev.turn_duration = self.tool_turn_start_time.map(|t| tool_end_time.duration_since(t)); + tool_telemetry = tool_telemetry.and_modify(|metric| { + metric.base.execution_duration = Some(tool_time); + metric.base.turn_duration = self.tool_turn_start_time.map(|t| tool_end_time.duration_since(t)); }); if let Tool::Custom(ct) = &tool.tool { - tool_telemetry = tool_telemetry.and_modify(|ev| { - ev.is_custom_tool = true; + tool_telemetry = tool_telemetry.and_modify(|metric| { + metric.base.is_custom_tool = true; // legacy fields previously implemented for only MCP tools - ev.custom_tool_call_latency = Some(tool_time.as_secs() as usize); - ev.input_token_size = Some(ct.get_input_token_size()); + metric.base.custom_tool_call_latency = Some(tool_time.as_secs() as usize); + metric.base.input_token_size = Some(ct.get_input_token_size()); }); } let tool_time = format!("{}.{}", tool_time.as_secs(), tool_time.subsec_millis()); @@ -2599,10 +2701,10 @@ impl ChatSession { } execute!(self.stdout, style::Print("\n\n"))?; - tool_telemetry = tool_telemetry.and_modify(|ev| ev.is_success = Some(true)); + tool_telemetry = tool_telemetry.and_modify(|metric| metric.base.is_success = Some(true)); if let Tool::Custom(_) = &tool.tool { tool_telemetry - .and_modify(|ev| ev.output_token_size = Some(TokenCounter::count_tokens(&result.as_str()))); + .and_modify(|metric| metric.base.output_token_size = Some(TokenCounter::count_tokens(&result.as_str()))); } // Send telemetry for agent contribution @@ -2653,9 +2755,9 @@ impl ChatSession { style::Print("\n\n"), )?; - tool_telemetry.and_modify(|ev| { - ev.is_success = Some(false); - ev.reason_desc = Some(err.to_string()); + tool_telemetry.and_modify(|metric| { + metric.base.is_success = Some(false); + metric.base.reason_desc = Some(err.to_string()); }); tool_results.push(ToolUseResult { tool_use_id: tool.id.clone(), @@ -3225,14 +3327,22 @@ impl ChatSession { let tool_use_id = tool_use.id.clone(); let tool_use_name = tool_use.name.clone(); let tool_input = tool_use.args.clone(); - let mut tool_telemetry = ToolUseEventBuilder::new( + // Create enhanced metric for agent contribution tracking (step 1: validation) + let mut tool_telemetry = AgentContributionMetric::new( conv_id.clone(), tool_use.id.clone(), self.conversation.model_info.as_ref().map(|m| m.model_id.clone()), - ) - .set_tool_use_id(tool_use_id.clone()) - .set_tool_name(tool_use.name.clone()) - .utterance_id(self.conversation.message_id().map(|s| s.to_string())); + ); + tool_telemetry.base = tool_telemetry.base + .set_tool_use_id(tool_use_id.clone()) + .set_tool_name(tool_use.name.clone()) + .utterance_id(self.conversation.message_id().map(|s| s.to_string())); + + // Initialize with tool content for line counting (step 2: content analysis) + if let Some(content) = extract_tool_content(&tool_use_name, &tool_input) { + tool_telemetry.init_with_tool_content(&tool_use_name, &content); + } + match self.conversation.tool_manager.get_tool_from_tool_use(tool_use).await { Ok(mut tool) => { // Apply non-Q-generated context to tools @@ -3240,7 +3350,7 @@ impl ChatSession { match tool.validate(os).await { Ok(()) => { - tool_telemetry.is_valid = Some(true); + tool_telemetry.base.is_valid = Some(true); queued_tools.push(QueuedTool { id: tool_use_id.clone(), name: tool_use_name, @@ -3250,7 +3360,7 @@ impl ChatSession { }); }, Err(err) => { - tool_telemetry.is_valid = Some(false); + tool_telemetry.base.is_valid = Some(false); tool_results.push(ToolUseResult { tool_use_id: tool_use_id.clone(), content: vec![ToolUseResultBlock::Text(format!( @@ -3262,7 +3372,7 @@ impl ChatSession { }; }, Err(err) => { - tool_telemetry.is_valid = Some(false); + tool_telemetry.base.is_valid = Some(false); tool_results.push(err.into()); }, } @@ -3577,15 +3687,18 @@ impl ChatSession { generated_prompt } + /// Send enhanced agent contribution telemetry with complete user interaction data + /// Approach 2: Single metric per tool with all information (suggestion + decision + execution) async fn send_tool_use_telemetry(&mut self, os: &Os) { - for (_, mut event) in self.tool_use_telemetry_events.drain() { - event.user_input_id = match self.tool_use_status { + for (_, mut metric) in self.tool_use_telemetry_events.drain() { + metric.base.user_input_id = match self.tool_use_status { ToolUseStatus::Idle => self.conversation.message_id(), ToolUseStatus::RetryInProgress(ref id) => Some(id.as_str()), } .map(|v| v.to_string()); - os.telemetry.send_tool_use_suggested(&os.database, event).await.ok(); + // Send complete metric via existing ToolUseSuggested event + os.telemetry.send_tool_use_suggested(&os.database, metric.base).await.ok(); } } From 1e0f6601a9dbc2ee2db17cac7b25a1b3e529ebbe Mon Sep 17 00:00:00 2001 From: Dhruvi Gajjar Date: Wed, 5 Nov 2025 17:32:24 -0800 Subject: [PATCH 2/4] added logic to emit Agent Contribution metric for reject code cases too --- crates/chat-cli/src/cli/chat/conversation.rs | 28 ++++++++++ crates/chat-cli/src/cli/chat/line_tracker.rs | 54 +++++++++++++++++++ crates/chat-cli/src/cli/chat/mod.rs | 15 +++++- .../chat-cli/src/cli/chat/tools/fs_write.rs | 17 ++++++ crates/chat-cli/src/telemetry/core.rs | 4 ++ crates/chat-cli/src/telemetry/mod.rs | 8 +++ 6 files changed, 125 insertions(+), 1 deletion(-) diff --git a/crates/chat-cli/src/cli/chat/conversation.rs b/crates/chat-cli/src/cli/chat/conversation.rs index a9c424900b..25c9fddf1e 100644 --- a/crates/chat-cli/src/cli/chat/conversation.rs +++ b/crates/chat-cli/src/cli/chat/conversation.rs @@ -960,6 +960,34 @@ Return only the JSON configuration, no additional text.", Ok(()) } + + pub async fn check_due_retention_metrics(&mut self, os: &Os) -> Result<(), ChatError> { + for (path, tracker) in &mut self.file_line_tracker { + if let Ok(content) = os.fs.read_to_string(path).await { + let results = tracker.check_due_retention(&content); + + for (conversation_id, _file_length, retained, total) in results { + debug!("Retention check for {}: {}/{} lines retained", path, retained, total); + + // Send retention metric using AgentContribution event + if let Err(e) = os.telemetry.send_agent_contribution_metric( + &os.database, + conversation_id, + None, // utterance_id + None, // tool_use_id + None, // tool_name + None, // lines_by_agent + None, // lines_by_user + Some(retained), // lines_retained + Some(total), // total_lines + ).await { + debug!("Failed to send retention metric: {}", e); + } + } + } + } + Ok(()) + } } pub fn format_tool_spec(tool_spec: HashMap) -> HashMap> { diff --git a/crates/chat-cli/src/cli/chat/line_tracker.rs b/crates/chat-cli/src/cli/chat/line_tracker.rs index 1717d16fe7..b3c319ea80 100644 --- a/crates/chat-cli/src/cli/chat/line_tracker.rs +++ b/crates/chat-cli/src/cli/chat/line_tracker.rs @@ -2,6 +2,7 @@ use serde::{ Deserialize, Serialize, }; +use std::time::{SystemTime, UNIX_EPOCH}; /// Contains metadata for tracking user and agent contribution metrics for a given file for /// `fs_write` tool uses. @@ -19,6 +20,16 @@ pub struct FileLineTracker { pub lines_removed_by_agent: usize, /// Whether or not this is the first `fs_write` invocation pub is_first_write: bool, + /// Pending retention checks scheduled for 15 minutes + #[serde(default)] + pub pending_retention_checks: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetentionCheck { + pub lines: Vec, + pub scheduled_time: u64, + pub conversation_id: String, } impl Default for FileLineTracker { @@ -30,6 +41,7 @@ impl Default for FileLineTracker { lines_added_by_agent: 0, lines_removed_by_agent: 0, is_first_write: true, + pending_retention_checks: Vec::new(), } } } @@ -42,4 +54,46 @@ impl FileLineTracker { pub fn lines_by_agent(&self) -> isize { (self.lines_added_by_agent + self.lines_removed_by_agent) as isize } + + pub fn schedule_retention_check(&mut self, lines: Vec, conversation_id: String) { + let scheduled_time = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + 900; // 15 minutes from now + + self.pending_retention_checks.push(RetentionCheck { + lines, + scheduled_time, + conversation_id, + }); + } + + pub fn check_due_retention(&mut self, file_content: &str) -> Vec<(String, usize, usize)> { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + + let mut results = Vec::new(); + let mut remaining_checks = Vec::new(); + + for check in self.pending_retention_checks.drain(..) { + if now >= check.scheduled_time { + let retained = check.lines.iter() + .filter(|line| file_content.contains(*line)) + .count(); + + results.push(( + check.conversation_id, + retained, + check.lines.len(), + )); + } else { + remaining_checks.push(check); + } + } + + self.pending_retention_checks = remaining_checks; + results + } } diff --git a/crates/chat-cli/src/cli/chat/mod.rs b/crates/chat-cli/src/cli/chat/mod.rs index 80f18d421a..c58a4de1be 100644 --- a/crates/chat-cli/src/cli/chat/mod.rs +++ b/crates/chat-cli/src/cli/chat/mod.rs @@ -1509,6 +1509,11 @@ impl ChatSession { while !matches!(self.inner, Some(ChatState::Exit)) { self.next(os).await?; + + // Check for due retention metrics on each interaction + if let Err(e) = self.conversation.check_due_retention_metrics(os).await { + debug!("Failed to check retention metrics: {}", e); + } } Ok(()) @@ -2719,16 +2724,24 @@ impl ChatSession { os.telemetry .send_agent_contribution_metric( &os.database, - conversation_id, + conversation_id.clone(), message_id, Some(tool.id.clone()), // Already a String Some(tool.name.clone()), // Already a String Some(lines_by_agent), Some(lines_by_user), + None, // lines_retained - not retention metric + None, // total_lines - not retention metric ) .await .ok(); + // Schedule retention check for 15 minutes + let agent_lines = w.extract_agent_lines(); + if !agent_lines.is_empty() { + tracker.schedule_retention_check(agent_lines, conversation_id); + } + tracker.prev_fswrite_lines = tracker.after_fswrite_lines; } } diff --git a/crates/chat-cli/src/cli/chat/tools/fs_write.rs b/crates/chat-cli/src/cli/chat/tools/fs_write.rs index 09a64058a1..cf6d2ee911 100644 --- a/crates/chat-cli/src/cli/chat/tools/fs_write.rs +++ b/crates/chat-cli/src/cli/chat/tools/fs_write.rs @@ -261,6 +261,23 @@ impl FsWrite { Ok(()) } + pub fn extract_agent_lines(&self) -> Vec { + match self { + FsWrite::Create { file_text, .. } => { + file_text.lines().map(|s| s.to_string()).collect() + }, + FsWrite::StrReplace { new_str, .. } => { + new_str.lines().map(|s| s.to_string()).collect() + }, + FsWrite::Insert { new_str, .. } => { + new_str.lines().map(|s| s.to_string()).collect() + }, + FsWrite::Append { new_str, .. } => { + new_str.lines().map(|s| s.to_string()).collect() + }, + } + } + async fn calculate_diff_lines(&self, os: &Os) -> Result<(usize, usize)> { let path = self.path(os); diff --git a/crates/chat-cli/src/telemetry/core.rs b/crates/chat-cli/src/telemetry/core.rs index b33a3cf027..a9c9bc0bf8 100644 --- a/crates/chat-cli/src/telemetry/core.rs +++ b/crates/chat-cli/src/telemetry/core.rs @@ -369,6 +369,8 @@ impl Event { tool_name, lines_by_agent, lines_by_user, + lines_retained: _, + total_lines: _, } => Some( CodewhispererterminalAgentContribution { create_time: self.created_time, @@ -686,6 +688,8 @@ pub enum EventType { tool_name: Option, lines_by_agent: Option, lines_by_user: Option, + lines_retained: Option, + total_lines: Option, }, McpServerInit { conversation_id: String, diff --git a/crates/chat-cli/src/telemetry/mod.rs b/crates/chat-cli/src/telemetry/mod.rs index 1ee5da2374..585a0afca5 100644 --- a/crates/chat-cli/src/telemetry/mod.rs +++ b/crates/chat-cli/src/telemetry/mod.rs @@ -296,6 +296,8 @@ impl TelemetryThread { tool_name: Option, lines_by_agent: Option, lines_by_user: Option, + lines_retained: Option, + total_lines: Option, ) -> Result<(), TelemetryError> { let mut telemetry_event = Event::new(EventType::AgentContribution { conversation_id, @@ -304,6 +306,8 @@ impl TelemetryThread { tool_name, lines_by_agent, lines_by_user, + lines_retained, + total_lines, }); set_event_metadata(database, &mut telemetry_event).await; Ok(self.tx.send(telemetry_event)?) @@ -648,6 +652,8 @@ impl TelemetryClient { conversation_id, utterance_id, lines_by_agent, + lines_retained, + total_lines, .. } => { let user_context = self.user_context().unwrap(); @@ -671,6 +677,8 @@ impl TelemetryClient { ?event, ?user_context, telemetry_enabled = self.telemetry_enabled, + lines_retained = ?lines_retained, + total_lines = ?total_lines, "Sending cw telemetry event" ); if let Err(err) = codewhisperer_client From 4443c53f4a96293e59ab11ca8c9b3dd8326cc1f5 Mon Sep 17 00:00:00 2001 From: Dhruvi Gajjar Date: Thu, 13 Nov 2025 13:28:45 -0800 Subject: [PATCH 3/4] added graceful shutdown, file not found erros, and one agent Contribution Metric instead of tool use suggested --- crates/chat-cli/src/cli/chat/cli/mod.rs | 6 +- crates/chat-cli/src/cli/chat/conversation.rs | 131 +++++++++++++++--- crates/chat-cli/src/cli/chat/line_tracker.rs | 50 ++++++- crates/chat-cli/src/cli/chat/mod.rs | 105 ++++++++++++-- .../chat-cli/src/cli/chat/tools/fs_write.rs | 2 +- crates/chat-cli/src/telemetry/core.rs | 10 +- crates/chat-cli/src/telemetry/mod.rs | 37 ++++- crates/chat-cli/telemetry_definitions.json | 14 +- 8 files changed, 312 insertions(+), 43 deletions(-) diff --git a/crates/chat-cli/src/cli/chat/cli/mod.rs b/crates/chat-cli/src/cli/chat/cli/mod.rs index 464f6269cf..a8b2eb69ff 100644 --- a/crates/chat-cli/src/cli/chat/cli/mod.rs +++ b/crates/chat-cli/src/cli/chat/cli/mod.rs @@ -132,7 +132,11 @@ pub enum SlashCommand { impl SlashCommand { pub async fn execute(self, os: &mut Os, session: &mut ChatSession) -> Result { match self { - Self::Quit => Ok(ChatState::Exit), + Self::Quit => { + // Flush all pending retention checks before quitting + session.conversation.flush_all_retention_metrics(os, "quit").await.ok(); + Ok(ChatState::Exit) + }, Self::Clear(args) => args.execute(session).await, Self::Agent(subcommand) => subcommand.execute(os, session).await, Self::Profile => { diff --git a/crates/chat-cli/src/cli/chat/conversation.rs b/crates/chat-cli/src/cli/chat/conversation.rs index 25c9fddf1e..1e604e4d44 100644 --- a/crates/chat-cli/src/cli/chat/conversation.rs +++ b/crates/chat-cli/src/cli/chat/conversation.rs @@ -260,6 +260,7 @@ impl ConversationState { } /// Enter tangent mode - creates checkpoint of current state + /// Allows exploring side topics without affecting main conversation pub fn enter_tangent_mode(&mut self) { if self.tangent_state.is_none() { self.tangent_state = Some(self.create_checkpoint()); @@ -961,28 +962,120 @@ Return only the JSON configuration, no additional text.", Ok(()) } - pub async fn check_due_retention_metrics(&mut self, os: &Os) -> Result<(), ChatError> { + pub async fn check_due_retention_metrics(&mut self, os: &Os) -> Result, ChatError> { + let mut all_results = Vec::new(); + let message_id = self.message_id().map(|s| s.to_string()); + for (path, tracker) in &mut self.file_line_tracker { - if let Ok(content) = os.fs.read_to_string(path).await { - let results = tracker.check_due_retention(&content); - - for (conversation_id, _file_length, retained, total) in results { - debug!("Retention check for {}: {}/{} lines retained", path, retained, total); + match os.fs.read_to_string(path).await { + Ok(content) => { + let results = tracker.check_due_retention(&content); - // Send retention metric using AgentContribution event - if let Err(e) = os.telemetry.send_agent_contribution_metric( - &os.database, - conversation_id, - None, // utterance_id - None, // tool_use_id - None, // tool_name - None, // lines_by_agent - None, // lines_by_user - Some(retained), // lines_retained - Some(total), // total_lines - ).await { - debug!("Failed to send retention metric: {}", e); + for (conversation_id, tool_use_id, retained, total, source) in results { + debug!("Retention check for {}: {}/{} lines retained, tool_use_id: {}, source: {}", + path, retained, total, tool_use_id, source); + + // Send retention metric with source + os.telemetry + .send_agent_contribution_metric_with_source( + &os.database, + conversation_id, + message_id.clone(), + Some(tool_use_id.clone()), + None, + None, + None, + Some(retained), + Some(total), + Some(source), + ) + .await + .ok(); + + all_results.push((tool_use_id, retained, total)); + } + } + Err(_) => { + // File not found - emit metrics for all pending checks with file_not_found reason + for check in &tracker.pending_retention_checks { + debug!("File not found during retention check: {}, tool_use_id: {}", path, check.tool_use_id); + + os.telemetry + .send_agent_contribution_metric_with_source( + &os.database, + check.conversation_id.clone(), + message_id.clone(), + Some(check.tool_use_id.clone()), + None, + None, + None, + Some(0), // retained = 0 since file doesn't exist + Some(check.lines.len()), + Some("file_not_found".to_string()), + ) + .await + .ok(); + } + // Clear pending checks since file is gone + tracker.pending_retention_checks.clear(); + } + } + } + Ok(all_results) + } + + pub async fn flush_all_retention_metrics(&mut self, os: &Os, source: &str) -> Result<(), ChatError> { + let message_id = self.message_id().map(|s| s.to_string()); + + for (path, tracker) in &mut self.file_line_tracker { + match os.fs.read_to_string(path).await { + Ok(content) => { + let results = tracker.flush_all_retention_checks(&content, source); + + for (conversation_id, tool_use_id, retained, total, source) in results { + debug!("Flushing retention check for {}: {}/{} lines retained, tool_use_id: {}, source: {}", + path, retained, total, tool_use_id, source); + + os.telemetry + .send_agent_contribution_metric_with_source( + &os.database, + conversation_id, + message_id.clone(), + Some(tool_use_id.clone()), + None, + None, + None, + Some(retained), + Some(total), + Some(source), + ) + .await + .ok(); + } + } + Err(_) => { + // File not found - emit metrics for all pending checks with file_not_found reason + for check in &tracker.pending_retention_checks { + debug!("File not found during flush: {}, tool_use_id: {}", path, check.tool_use_id); + + os.telemetry + .send_agent_contribution_metric_with_source( + &os.database, + check.conversation_id.clone(), + message_id.clone(), + Some(check.tool_use_id.clone()), + None, + None, + None, + Some(0), // retained = 0 since file doesn't exist + Some(check.lines.len()), + Some("file_not_found".to_string()), + ) + .await + .ok(); } + // Clear pending checks since file is gone + tracker.pending_retention_checks.clear(); } } } diff --git a/crates/chat-cli/src/cli/chat/line_tracker.rs b/crates/chat-cli/src/cli/chat/line_tracker.rs index b3c319ea80..98d0baa1e2 100644 --- a/crates/chat-cli/src/cli/chat/line_tracker.rs +++ b/crates/chat-cli/src/cli/chat/line_tracker.rs @@ -20,7 +20,7 @@ pub struct FileLineTracker { pub lines_removed_by_agent: usize, /// Whether or not this is the first `fs_write` invocation pub is_first_write: bool, - /// Pending retention checks scheduled for 15 minutes + /// Pending retention checks scheduled for 1 minute (changed from 15 minutes for testing) #[serde(default)] pub pending_retention_checks: Vec, } @@ -30,6 +30,7 @@ pub struct RetentionCheck { pub lines: Vec, pub scheduled_time: u64, pub conversation_id: String, + pub tool_use_id: String, } impl Default for FileLineTracker { @@ -55,7 +56,7 @@ impl FileLineTracker { (self.lines_added_by_agent + self.lines_removed_by_agent) as isize } - pub fn schedule_retention_check(&mut self, lines: Vec, conversation_id: String) { + pub fn schedule_retention_check(&mut self, lines: Vec, conversation_id: String, tool_use_id: String) { let scheduled_time = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -65,10 +66,31 @@ impl FileLineTracker { lines, scheduled_time, conversation_id, + tool_use_id, }); } - pub fn check_due_retention(&mut self, file_content: &str) -> Vec<(String, usize, usize)> { + pub fn flush_pending_checks_for_agent_rewrite(&mut self, file_content: &str) -> Vec<(String, String, usize, usize, String)> { + let mut results = Vec::new(); + + for check in self.pending_retention_checks.drain(..) { + let retained = check.lines.iter() + .filter(|line| file_content.contains(*line)) + .count(); + + results.push(( + check.conversation_id, + check.tool_use_id, + retained, + check.lines.len(), + "agent_rewrite".to_string(), + )); + } + + results + } + + pub fn check_due_retention(&mut self, file_content: &str) -> Vec<(String, String, usize, usize, String)> { let now = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -85,8 +107,10 @@ impl FileLineTracker { results.push(( check.conversation_id, + check.tool_use_id, retained, check.lines.len(), + "15min_check".to_string(), )); } else { remaining_checks.push(check); @@ -96,4 +120,24 @@ impl FileLineTracker { self.pending_retention_checks = remaining_checks; results } + + pub fn flush_all_retention_checks(&mut self, file_content: &str, source: &str) -> Vec<(String, String, usize, usize, String)> { + let mut results = Vec::new(); + + for check in self.pending_retention_checks.drain(..) { + let retained = check.lines.iter() + .filter(|line| file_content.contains(*line)) + .count(); + + results.push(( + check.conversation_id, + check.tool_use_id, + retained, + check.lines.len(), + source.to_string(), + )); + } + + results + } } diff --git a/crates/chat-cli/src/cli/chat/mod.rs b/crates/chat-cli/src/cli/chat/mod.rs index c58a4de1be..adc73b945a 100644 --- a/crates/chat-cli/src/cli/chat/mod.rs +++ b/crates/chat-cli/src/cli/chat/mod.rs @@ -210,6 +210,8 @@ pub struct AgentContributionMetric { pub lines_suggested: Option, pub lines_accepted: Option, pub lines_rejected: Option, + pub lines_retained: Option, + pub total_lines_checked: Option, } /// User's final decision on tool execution after seeing preview @@ -228,6 +230,8 @@ impl AgentContributionMetric { lines_suggested: None, lines_accepted: None, lines_rejected: None, + lines_retained: None, + total_lines_checked: None, } } @@ -1507,12 +1511,73 @@ impl ChatSession { self.inner = Some(ChatState::HandleInput { input: user_input }); } - while !matches!(self.inner, Some(ChatState::Exit)) { - self.next(os).await?; - - // Check for due retention metrics on each interaction - if let Err(e) = self.conversation.check_due_retention_metrics(os).await { - debug!("Failed to check retention metrics: {}", e); + // Set up signal handler for graceful shutdown + #[cfg(unix)] + { + use tokio::signal::unix::{signal, SignalKind}; + let mut sigint = signal(SignalKind::interrupt()).map_err(|e| ChatError::Custom(format!("Failed to setup SIGINT handler: {}", e).into()))?; + let mut sigterm = signal(SignalKind::terminate()).map_err(|e| ChatError::Custom(format!("Failed to setup SIGTERM handler: {}", e).into()))?; + + loop { + tokio::select! { + // Handle normal chat operations + result = async { + if !matches!(self.inner, Some(ChatState::Exit)) { + self.next(os).await + } else { + Ok(()) + } + } => { + if let Err(e) = result { + return Err(e.into()); + } + if matches!(self.inner, Some(ChatState::Exit)) { + break; + } + + // Check for due retention metrics on each interaction + if let Ok(retention_results) = self.conversation.check_due_retention_metrics(os).await { + // Update existing telemetry events with retention data + for (tool_use_id, retained, total) in retention_results { + if let Some(metric) = self.tool_use_telemetry_events.get_mut(&tool_use_id) { + metric.lines_retained = Some(retained); + metric.total_lines_checked = Some(total); + debug!("Updated retention for tool_use_id {}: {}/{} lines retained", + tool_use_id, retained, total); + } + } + } + } + // Handle SIGINT (Ctrl+C) and SIGTERM + _ = sigint.recv() => { + self.conversation.flush_all_retention_metrics(os, "shutdown").await.ok(); + break; + } + _ = sigterm.recv() => { + self.conversation.flush_all_retention_metrics(os, "shutdown").await.ok(); + break; + } + } + } + } + + #[cfg(not(unix))] + { + while !matches!(self.inner, Some(ChatState::Exit)) { + self.next(os).await?; + + // Check for due retention metrics on each interaction + if let Ok(retention_results) = self.conversation.check_due_retention_metrics(os).await { + // Update existing telemetry events with retention data + for (tool_use_id, retained, total) in retention_results { + if let Some(metric) = self.tool_use_telemetry_events.get_mut(&tool_use_id) { + metric.lines_retained = Some(retained); + metric.total_lines_checked = Some(total); + debug!("Updated retention for tool_use_id {}: {}/{} lines retained", + tool_use_id, retained, total); + } + } + } } } @@ -2730,16 +2795,38 @@ impl ChatSession { Some(tool.name.clone()), // Already a String Some(lines_by_agent), Some(lines_by_user), - None, // lines_retained - not retention metric - None, // total_lines - not retention metric + None, + None, ) .await .ok(); + // Flush any pending retention checks before scheduling new ones (agent rewrite scenario) + if let Ok(current_content) = os.fs.read_to_string(&w.path(os)).await { + let flush_results = tracker.flush_pending_checks_for_agent_rewrite(¤t_content); + for (conv_id, tool_use_id, retained, total, source) in flush_results { + os.telemetry + .send_agent_contribution_metric_with_source( + &os.database, + conv_id, + message_id.clone(), + Some(tool_use_id), + None, + None, + None, + Some(retained), + Some(total), + Some(source), + ) + .await + .ok(); + } + } + // Schedule retention check for 15 minutes let agent_lines = w.extract_agent_lines(); if !agent_lines.is_empty() { - tracker.schedule_retention_check(agent_lines, conversation_id); + tracker.schedule_retention_check(agent_lines, conversation_id, tool.id.clone()); } tracker.prev_fswrite_lines = tracker.after_fswrite_lines; diff --git a/crates/chat-cli/src/cli/chat/tools/fs_write.rs b/crates/chat-cli/src/cli/chat/tools/fs_write.rs index cf6d2ee911..2a1498b33b 100644 --- a/crates/chat-cli/src/cli/chat/tools/fs_write.rs +++ b/crates/chat-cli/src/cli/chat/tools/fs_write.rs @@ -264,7 +264,7 @@ impl FsWrite { pub fn extract_agent_lines(&self) -> Vec { match self { FsWrite::Create { file_text, .. } => { - file_text.lines().map(|s| s.to_string()).collect() + file_text.as_ref().map(|s| s.lines().map(|l| l.to_string()).collect()).unwrap_or_default() }, FsWrite::StrReplace { new_str, .. } => { new_str.lines().map(|s| s.to_string()).collect() diff --git a/crates/chat-cli/src/telemetry/core.rs b/crates/chat-cli/src/telemetry/core.rs index a9c9bc0bf8..c653726eee 100644 --- a/crates/chat-cli/src/telemetry/core.rs +++ b/crates/chat-cli/src/telemetry/core.rs @@ -369,8 +369,9 @@ impl Event { tool_name, lines_by_agent, lines_by_user, - lines_retained: _, - total_lines: _, + lines_retained, + total_lines_checked, + source: _, } => Some( CodewhispererterminalAgentContribution { create_time: self.created_time, @@ -382,6 +383,8 @@ impl Event { codewhispererterminal_tool_name: tool_name.map(CodewhispererterminalToolName), codewhispererterminal_lines_by_agent: lines_by_agent.map(|count| count as i64).map(Into::into), codewhispererterminal_lines_by_user: lines_by_user.map(|count| count as i64).map(Into::into), + codewhispererterminal_lines_retained: lines_retained.map(|count| count as i64).map(Into::into), + codewhispererterminal_total_lines_checked: total_lines_checked.map(|count| count as i64).map(Into::into), } .into_metric_datum(), ), @@ -689,7 +692,8 @@ pub enum EventType { lines_by_agent: Option, lines_by_user: Option, lines_retained: Option, - total_lines: Option, + total_lines_checked: Option, + source: Option, }, McpServerInit { conversation_id: String, diff --git a/crates/chat-cli/src/telemetry/mod.rs b/crates/chat-cli/src/telemetry/mod.rs index 585a0afca5..bf692efe31 100644 --- a/crates/chat-cli/src/telemetry/mod.rs +++ b/crates/chat-cli/src/telemetry/mod.rs @@ -297,7 +297,35 @@ impl TelemetryThread { lines_by_agent: Option, lines_by_user: Option, lines_retained: Option, - total_lines: Option, + total_lines_checked: Option, + ) -> Result<(), TelemetryError> { + self.send_agent_contribution_metric_with_source( + database, + conversation_id, + utterance_id, + tool_use_id, + tool_name, + lines_by_agent, + lines_by_user, + lines_retained, + total_lines_checked, + None, + ).await + } + + #[allow(clippy::too_many_arguments)] // TODO: Should make a parameters struct. + pub async fn send_agent_contribution_metric_with_source( + &self, + database: &Database, + conversation_id: String, + utterance_id: Option, + tool_use_id: Option, + tool_name: Option, + lines_by_agent: Option, + lines_by_user: Option, + lines_retained: Option, + total_lines_checked: Option, + source: Option, ) -> Result<(), TelemetryError> { let mut telemetry_event = Event::new(EventType::AgentContribution { conversation_id, @@ -307,7 +335,8 @@ impl TelemetryThread { lines_by_agent, lines_by_user, lines_retained, - total_lines, + total_lines_checked, + source, }); set_event_metadata(database, &mut telemetry_event).await; Ok(self.tx.send(telemetry_event)?) @@ -652,8 +681,6 @@ impl TelemetryClient { conversation_id, utterance_id, lines_by_agent, - lines_retained, - total_lines, .. } => { let user_context = self.user_context().unwrap(); @@ -677,8 +704,6 @@ impl TelemetryClient { ?event, ?user_context, telemetry_enabled = self.telemetry_enabled, - lines_retained = ?lines_retained, - total_lines = ?total_lines, "Sending cw telemetry event" ); if let Err(err) = codewhisperer_client diff --git a/crates/chat-cli/telemetry_definitions.json b/crates/chat-cli/telemetry_definitions.json index b2bc7044bf..70e25ba936 100644 --- a/crates/chat-cli/telemetry_definitions.json +++ b/crates/chat-cli/telemetry_definitions.json @@ -303,6 +303,16 @@ "name": "codewhispererterminal_linesByUser", "type": "int", "description": "The number of lines of code contributed by user" + }, + { + "name": "codewhispererterminal_linesRetained", + "type": "int", + "description": "The number of lines retained after retention check" + }, + { + "name": "codewhispererterminal_totalLinesChecked", + "type": "int", + "description": "The total number of lines checked during retention check" } ], "metrics": [ @@ -481,7 +491,9 @@ { "type": "codewhispererterminal_toolUseId" }, { "type": "codewhispererterminal_toolName" }, { "type": "codewhispererterminal_linesByAgent" }, - { "type": "codewhispererterminal_linesByUser" } + { "type": "codewhispererterminal_linesByUser" }, + { "type": "codewhispererterminal_linesRetained", "required": false }, + { "type": "codewhispererterminal_totalLinesChecked", "required": false } ] }, { From 49a8b4930b71cb830941faf4ebc13df3b3b7f443 Mon Sep 17 00:00:00 2001 From: Dhruvi Gajjar Date: Mon, 24 Nov 2025 13:11:15 -0800 Subject: [PATCH 4/4] added model info in agent contribution metric --- crates/chat-cli/src/cli/chat/conversation.rs | 6 +++ crates/chat-cli/src/cli/chat/line_tracker.rs | 10 +++-- crates/chat-cli/src/cli/chat/mod.rs | 42 +++++++++++++++----- crates/chat-cli/src/telemetry/core.rs | 3 ++ crates/chat-cli/src/telemetry/mod.rs | 4 ++ crates/chat-cli/telemetry_definitions.json | 3 +- 6 files changed, 53 insertions(+), 15 deletions(-) diff --git a/crates/chat-cli/src/cli/chat/conversation.rs b/crates/chat-cli/src/cli/chat/conversation.rs index 1e604e4d44..267a3872c2 100644 --- a/crates/chat-cli/src/cli/chat/conversation.rs +++ b/crates/chat-cli/src/cli/chat/conversation.rs @@ -965,6 +965,7 @@ Return only the JSON configuration, no additional text.", pub async fn check_due_retention_metrics(&mut self, os: &Os) -> Result, ChatError> { let mut all_results = Vec::new(); let message_id = self.message_id().map(|s| s.to_string()); + let model_id = self.model_info.as_ref().map(|m| m.model_id.clone()); for (path, tracker) in &mut self.file_line_tracker { match os.fs.read_to_string(path).await { @@ -988,6 +989,7 @@ Return only the JSON configuration, no additional text.", Some(retained), Some(total), Some(source), + model_id.clone(), ) .await .ok(); @@ -1012,6 +1014,7 @@ Return only the JSON configuration, no additional text.", Some(0), // retained = 0 since file doesn't exist Some(check.lines.len()), Some("file_not_found".to_string()), + model_id.clone(), ) .await .ok(); @@ -1026,6 +1029,7 @@ Return only the JSON configuration, no additional text.", pub async fn flush_all_retention_metrics(&mut self, os: &Os, source: &str) -> Result<(), ChatError> { let message_id = self.message_id().map(|s| s.to_string()); + let model_id = self.model_info.as_ref().map(|m| m.model_id.clone()); for (path, tracker) in &mut self.file_line_tracker { match os.fs.read_to_string(path).await { @@ -1048,6 +1052,7 @@ Return only the JSON configuration, no additional text.", Some(retained), Some(total), Some(source), + model_id.clone(), ) .await .ok(); @@ -1070,6 +1075,7 @@ Return only the JSON configuration, no additional text.", Some(0), // retained = 0 since file doesn't exist Some(check.lines.len()), Some("file_not_found".to_string()), + model_id.clone(), ) .await .ok(); diff --git a/crates/chat-cli/src/cli/chat/line_tracker.rs b/crates/chat-cli/src/cli/chat/line_tracker.rs index 98d0baa1e2..70fbe6c86d 100644 --- a/crates/chat-cli/src/cli/chat/line_tracker.rs +++ b/crates/chat-cli/src/cli/chat/line_tracker.rs @@ -2,6 +2,7 @@ use serde::{ Deserialize, Serialize, }; +use std::collections::HashSet; use std::time::{SystemTime, UNIX_EPOCH}; /// Contains metadata for tracking user and agent contribution metrics for a given file for @@ -72,10 +73,11 @@ impl FileLineTracker { pub fn flush_pending_checks_for_agent_rewrite(&mut self, file_content: &str) -> Vec<(String, String, usize, usize, String)> { let mut results = Vec::new(); + let file_lines: HashSet<&str> = file_content.lines().collect(); for check in self.pending_retention_checks.drain(..) { let retained = check.lines.iter() - .filter(|line| file_content.contains(*line)) + .filter(|line| file_lines.contains(line.as_str())) .count(); results.push(( @@ -98,11 +100,12 @@ impl FileLineTracker { let mut results = Vec::new(); let mut remaining_checks = Vec::new(); + let file_lines: HashSet<&str> = file_content.lines().collect(); for check in self.pending_retention_checks.drain(..) { if now >= check.scheduled_time { let retained = check.lines.iter() - .filter(|line| file_content.contains(*line)) + .filter(|line| file_lines.contains(line.as_str())) .count(); results.push(( @@ -123,10 +126,11 @@ impl FileLineTracker { pub fn flush_all_retention_checks(&mut self, file_content: &str, source: &str) -> Vec<(String, String, usize, usize, String)> { let mut results = Vec::new(); + let file_lines: HashSet<&str> = file_content.lines().collect(); for check in self.pending_retention_checks.drain(..) { let retained = check.lines.iter() - .filter(|line| file_content.contains(*line)) + .filter(|line| file_lines.contains(line.as_str())) .count(); results.push(( diff --git a/crates/chat-cli/src/cli/chat/mod.rs b/crates/chat-cli/src/cli/chat/mod.rs index adc73b945a..29909bbaa5 100644 --- a/crates/chat-cli/src/cli/chat/mod.rs +++ b/crates/chat-cli/src/cli/chat/mod.rs @@ -1537,15 +1537,23 @@ impl ChatSession { // Check for due retention metrics on each interaction if let Ok(retention_results) = self.conversation.check_due_retention_metrics(os).await { - // Update existing telemetry events with retention data + let mut metrics_to_emit = Vec::new(); + for (tool_use_id, retained, total) in retention_results { - if let Some(metric) = self.tool_use_telemetry_events.get_mut(&tool_use_id) { - metric.lines_retained = Some(retained); - metric.total_lines_checked = Some(total); - debug!("Updated retention for tool_use_id {}: {}/{} lines retained", + if let Some(metric) = self.tool_use_telemetry_events.remove(&tool_use_id) { + let mut updated_metric = metric; + updated_metric.lines_retained = Some(retained); + updated_metric.total_lines_checked = Some(total); + debug!("Emitting retention for tool_use_id {}: {}/{} lines retained", tool_use_id, retained, total); + metrics_to_emit.push(updated_metric); } } + + for mut metric in metrics_to_emit { + metric.base.user_input_id = self.conversation.message_id().map(|v| v.to_string()); + os.telemetry.send_tool_use_suggested(&os.database, metric.base).await.ok(); + } } } // Handle SIGINT (Ctrl+C) and SIGTERM @@ -1568,15 +1576,23 @@ impl ChatSession { // Check for due retention metrics on each interaction if let Ok(retention_results) = self.conversation.check_due_retention_metrics(os).await { - // Update existing telemetry events with retention data + let mut metrics_to_emit = Vec::new(); + for (tool_use_id, retained, total) in retention_results { - if let Some(metric) = self.tool_use_telemetry_events.get_mut(&tool_use_id) { - metric.lines_retained = Some(retained); - metric.total_lines_checked = Some(total); - debug!("Updated retention for tool_use_id {}: {}/{} lines retained", + if let Some(metric) = self.tool_use_telemetry_events.remove(&tool_use_id) { + let mut updated_metric = metric; + updated_metric.lines_retained = Some(retained); + updated_metric.total_lines_checked = Some(total); + debug!("Emitting retention for tool_use_id {}: {}/{} lines retained", tool_use_id, retained, total); + metrics_to_emit.push(updated_metric); } } + + for mut metric in metrics_to_emit { + metric.base.user_input_id = self.conversation.message_id().map(|v| v.to_string()); + os.telemetry.send_tool_use_suggested(&os.database, metric.base).await.ok(); + } } } } @@ -2782,6 +2798,7 @@ impl ChatSession { let sanitized_path_str = w.path(os).to_string_lossy().to_string(); let conversation_id = self.conversation.conversation_id().to_string(); let message_id = self.conversation.message_id().map(|s| s.to_string()); + let model_id = self.conversation.model_info.as_ref().map(|m| m.model_id.clone()); if let Some(tracker) = self.conversation.file_line_tracker.get_mut(&sanitized_path_str) { let lines_by_agent = tracker.lines_by_agent(); let lines_by_user = tracker.lines_by_user(); @@ -2790,13 +2807,14 @@ impl ChatSession { .send_agent_contribution_metric( &os.database, conversation_id.clone(), - message_id, + message_id.clone(), Some(tool.id.clone()), // Already a String Some(tool.name.clone()), // Already a String Some(lines_by_agent), Some(lines_by_user), None, None, + model_id.clone(), ) .await .ok(); @@ -2804,6 +2822,7 @@ impl ChatSession { // Flush any pending retention checks before scheduling new ones (agent rewrite scenario) if let Ok(current_content) = os.fs.read_to_string(&w.path(os)).await { let flush_results = tracker.flush_pending_checks_for_agent_rewrite(¤t_content); + let model_id = self.conversation.model_info.as_ref().map(|m| m.model_id.clone()); for (conv_id, tool_use_id, retained, total, source) in flush_results { os.telemetry .send_agent_contribution_metric_with_source( @@ -2817,6 +2836,7 @@ impl ChatSession { Some(retained), Some(total), Some(source), + model_id.clone(), ) .await .ok(); diff --git a/crates/chat-cli/src/telemetry/core.rs b/crates/chat-cli/src/telemetry/core.rs index c653726eee..c5e4c76bcc 100644 --- a/crates/chat-cli/src/telemetry/core.rs +++ b/crates/chat-cli/src/telemetry/core.rs @@ -372,6 +372,7 @@ impl Event { lines_retained, total_lines_checked, source: _, + model, } => Some( CodewhispererterminalAgentContribution { create_time: self.created_time, @@ -385,6 +386,7 @@ impl Event { codewhispererterminal_lines_by_user: lines_by_user.map(|count| count as i64).map(Into::into), codewhispererterminal_lines_retained: lines_retained.map(|count| count as i64).map(Into::into), codewhispererterminal_total_lines_checked: total_lines_checked.map(|count| count as i64).map(Into::into), + codewhispererterminal_model: model.map(Into::into), } .into_metric_datum(), ), @@ -694,6 +696,7 @@ pub enum EventType { lines_retained: Option, total_lines_checked: Option, source: Option, + model: Option, }, McpServerInit { conversation_id: String, diff --git a/crates/chat-cli/src/telemetry/mod.rs b/crates/chat-cli/src/telemetry/mod.rs index bf692efe31..5ac5985a3d 100644 --- a/crates/chat-cli/src/telemetry/mod.rs +++ b/crates/chat-cli/src/telemetry/mod.rs @@ -298,6 +298,7 @@ impl TelemetryThread { lines_by_user: Option, lines_retained: Option, total_lines_checked: Option, + model: Option, ) -> Result<(), TelemetryError> { self.send_agent_contribution_metric_with_source( database, @@ -310,6 +311,7 @@ impl TelemetryThread { lines_retained, total_lines_checked, None, + model, ).await } @@ -326,6 +328,7 @@ impl TelemetryThread { lines_retained: Option, total_lines_checked: Option, source: Option, + model: Option, ) -> Result<(), TelemetryError> { let mut telemetry_event = Event::new(EventType::AgentContribution { conversation_id, @@ -337,6 +340,7 @@ impl TelemetryThread { lines_retained, total_lines_checked, source, + model, }); set_event_metadata(database, &mut telemetry_event).await; Ok(self.tx.send(telemetry_event)?) diff --git a/crates/chat-cli/telemetry_definitions.json b/crates/chat-cli/telemetry_definitions.json index 70e25ba936..8775d09685 100644 --- a/crates/chat-cli/telemetry_definitions.json +++ b/crates/chat-cli/telemetry_definitions.json @@ -493,7 +493,8 @@ { "type": "codewhispererterminal_linesByAgent" }, { "type": "codewhispererterminal_linesByUser" }, { "type": "codewhispererterminal_linesRetained", "required": false }, - { "type": "codewhispererterminal_totalLinesChecked", "required": false } + { "type": "codewhispererterminal_totalLinesChecked", "required": false }, + { "type": "codewhispererterminal_model", "required": false } ] }, {